From f3bcc7b942360117a00dfa24902768f9d3027da5 Mon Sep 17 00:00:00 2001 From: genjix Date: Sat, 21 Apr 2012 01:52:17 +0100 Subject: [PATCH] Super fast C++ payment_history (not working with mempool yet) --- backends/libbitcoin/h1.py | 3 +- backends/libbitcoin/history.cpp | 331 +++++++++++++++++++++++++++++- backends/libbitcoin/history1/__init__.py | 9 +- 3 files changed, 332 insertions(+), 11 deletions(-) diff --git a/backends/libbitcoin/h1.py b/backends/libbitcoin/h1.py index 1a90b28..659a3cf 100644 --- a/backends/libbitcoin/h1.py +++ b/backends/libbitcoin/h1.py @@ -4,8 +4,9 @@ import history1 as history def blockchain_started(ec, chain): print "Blockchain initialisation:", ec -def finish(ec): +def finish(ec, result): print "Finish:", ec + print result a = bitcoin.async_service(1) chain = bitcoin.bdb_blockchain(a, "/home/genjix/libbitcoin/database", diff --git a/backends/libbitcoin/history.cpp b/backends/libbitcoin/history.cpp index 9cc5cd5..d146a86 100644 --- a/backends/libbitcoin/history.cpp +++ b/backends/libbitcoin/history.cpp @@ -1,4 +1,5 @@ #include +#include #include using namespace libbitcoin; @@ -8,10 +9,61 @@ namespace python = boost::python; #include "/home/genjix/python-bitcoin/src/primitive.h" -typedef std::function finish_handler; - namespace ph = std::placeholders; +struct info_unit +{ + typedef std::vector string_list; + + // block_hash == null_hash if mempool tx + hash_digest tx_hash, block_hash; + string_list inputs, outputs; + size_t index; + uint64_t value; + size_t height; + uint64_t timestamp; + bool is_input; + // set to empty if unused + data_chunk raw_output_script; +}; + +typedef std::shared_ptr info_unit_ptr; + +bool inputs_all_loaded(info_unit::string_list inputs) +{ + for (const auto& input: inputs) + if (input.empty()) + return false; + return true; +} + +struct payment_entry +{ + bool is_loaded() + { + if (!loaded_output) + return false; + if (input_exists && !loaded_input) + return false; + return true; + } + + message::output_point outpoint; + info_unit_ptr loaded_output; + // indeterminate until we know whether this output has a spend + boost::tribool input_exists; + message::input_point inpoint; + // Ignore if input does not exist + info_unit_ptr loaded_input; +}; + +typedef std::shared_ptr payment_entry_ptr; + +typedef std::vector statement_entry; + +typedef std::function< + void (const std::error_code&, const statement_entry&)> finish_handler; + class query_history : public std::enable_shared_from_this { @@ -40,10 +92,230 @@ private: stopped_ = true; } + bool stop_on_error(const std::error_code& ec) + { + if (ec) + { + handle_finish_(ec, statement_entry()); + stop(); + } + return stopped_; + } + void start_loading(const std::error_code& ec, const message::output_point_list& outpoints) { - handle_finish_(ec); + if (stop_on_error(ec)) + return; + for (auto outpoint: outpoints) + { + auto entry = std::make_shared(); + entry->outpoint = outpoint; + statement_.push_back(entry); + chain_->fetch_spend(outpoint, + strand_.wrap(std::bind(&query_history::load_spend, + shared_from_this(), ph::_1, ph::_2, entry))); + load_tx_info(outpoint, entry, false); + } + } + + void load_spend(const std::error_code& ec, + const message::input_point inpoint, payment_entry_ptr entry) + { + // Need a custom self.stop_on_error(...) as a missing spend + // is not an error in this case. + if (ec && ec != error::unspent_output) + stop_on_error(ec); + if (stopped_) + return; + + if (ec == error::unspent_output) + { + // This particular entry.output_point + // has not been spent yet + entry->input_exists = false; + finish_if_done(); + } + else + { + // Has been spent + entry->input_exists = true; + entry->inpoint = inpoint; + load_tx_info(inpoint, entry, true); + } + } + + void finish_if_done() + { + for (auto entry: statement_) + if (!entry->is_loaded()) + return; + // Finish up + for (auto entry: statement_) + { + if (entry->input_exists) + { + BITCOIN_ASSERT(entry->loaded_input && entry->loaded_output); + // value of the input is simply the inverse of + // the corresponding output + entry->loaded_input->value = -entry->loaded_output->value; + // Unspent outputs have a raw_output_script field + // Blank this field as it isn't used + entry->loaded_output->raw_output_script.clear(); + } + else + { + // Unspent outputs have a raw_output_script field + } + } + handle_finish_(std::error_code(), statement_); + stop(); + } + + template + void load_tx_info(const Point& point, payment_entry_ptr entry, + bool is_input) + { + auto info = std::make_shared(); + info->tx_hash = point.hash; + info->index = point.index; + info->is_input = is_input; + // Before loading the transaction, Stratum requires the hash + // of the parent block, so we load the block depth and then + // fetch the block header and hash it. + chain_->fetch_transaction_index(point.hash, + strand_.wrap(std::bind(&query_history::tx_index, + shared_from_this(), ph::_1, ph::_2, ph::_3, entry, info))); + } + + void tx_index(const std::error_code& ec, size_t block_depth, + size_t offset, payment_entry_ptr entry, info_unit_ptr info) + { + if (stop_on_error(ec)) + return; + info->height = block_depth; + // And now for the block hash + chain_->fetch_block_header(block_depth, + strand_.wrap(std::bind(&query_history::block_header, + shared_from_this(), ph::_1, ph::_2, entry, info))); + } + + void block_header(const std::error_code& ec, + const message::block blk_head, + payment_entry_ptr entry, info_unit_ptr info) + { + if (stop_on_error(ec)) + return; + info->timestamp = blk_head.timestamp; + info->block_hash = hash_block_header(blk_head); + // Now load the actual main transaction for this input or output + chain_->fetch_transaction(info->tx_hash, + strand_.wrap(std::bind(&query_history::load_chain_tx, + shared_from_this(), ph::_1, ph::_2, entry, info))); + } + + void load_tx(const message::transaction& tx, info_unit_ptr info) + { + // List of output addresses + for (const auto& tx_out: tx.outputs) + { + payment_address addr; + // Attempt to extract address from output script + if (extract(addr, tx_out.output_script)) + info->outputs.push_back(addr.encoded()); + else + info->outputs.push_back("Unknown"); + } + // For the inputs, we need the originator address which has to + // be looked up in the blockchain. + // Create list of empty strings and then populate it. + // Loading has finished when list is no longer all empty strings. + for (const auto& tx_in: tx.inputs) + info->inputs.push_back(""); + // If this transaction was loaded for an input, then we already + // have a source address for at least one input. + if (info->is_input) + { + BITCOIN_ASSERT(info->index < info->inputs.size()); + info->inputs[info->index] = address_.encoded(); + } + } + + void load_chain_tx(const std::error_code& ec, + const message::transaction& tx, + payment_entry_ptr entry, info_unit_ptr info) + { + if (stop_on_error(ec)) + return; + load_tx(tx, info); + if (!info->is_input) + { + BITCOIN_ASSERT(info->index < tx.outputs.size()); + const auto& our_output = tx.outputs[info->index]; + info->value = our_output.value; + // Save serialised output script in case this output is unspent + info->raw_output_script = save_script(our_output.output_script); + } + // If all the inputs are loaded + if (inputs_all_loaded(info->inputs)) + { + // We are the sole input + BITCOIN_ASSERT(info->is_input); + entry->loaded_input = info; + finish_if_done(); + } + + // ********* + // fetch_input_txs + + // Load the previous_output for every input so we can get + // the output address + for (size_t input_index = 0; + input_index < tx.inputs.size(); ++input_index) + { + if (info->is_input && info->index == input_index) + continue; + const auto& prevout = tx.inputs[input_index].previous_output; + chain_->fetch_transaction(prevout.hash, + strand_.wrap(std::bind(&query_history::load_input_chain_tx, + shared_from_this(), ph::_1, ph::_2, prevout.index, + entry, info, input_index))); + } + } + + void load_input_tx(const message::transaction& tx, size_t output_index, + info_unit_ptr info, size_t input_index) + { + // For our input, we load the previous tx so we can get the + // corresponding output. + // We need the output to extract the address. + BITCOIN_ASSERT(output_index < tx.outputs.size()); + const auto& out_script = tx.outputs[output_index].output_script; + payment_address addr; + BITCOIN_ASSERT(input_index < info->inputs.size()); + if (extract(addr, out_script)) + info->inputs[input_index] = addr.encoded(); + else + info->inputs[input_index] = "Unknown"; + } + + void load_input_chain_tx(const std::error_code& ec, + const message::transaction& tx, size_t output_index, + payment_entry_ptr entry, info_unit_ptr info, size_t input_index) + { + if (stop_on_error(ec)) + return; + load_input_tx(tx, output_index, info, input_index); + // If all the inputs are loaded, then we have finished loading + // the info for this input-output entry pair + if (inputs_all_loaded(info->inputs)) + { + if (info->is_input) + entry->loaded_input = info; + else + entry->loaded_output = info; + } + finish_if_done(); } io_service::strand strand_; @@ -52,17 +324,64 @@ private: transaction_pool_ptr txpool_; bool stopped_; + statement_entry statement_; payment_address address_; finish_handler handle_finish_; }; typedef std::shared_ptr query_history_ptr; +void write_xputs_strings(std::stringstream& ss, const char* xput_name, + const info_unit::string_list& xputs) +{ + ss << '"' << xput_name << "\": ["; + for (auto it = xputs.begin(); it != xputs.end(); ++it) + { + if (it != xputs.begin()) + ss << ","; + ss << '"' << *it << '"'; + } + ss << "]"; +} + +void write_info(std::string& json, info_unit_ptr info) +{ + std::stringstream ss; + ss << "{" + << "\"tx_hash\": \"" << pretty_hex(info->tx_hash) << "\"," + << "\"block_hash\": \"" << pretty_hex(info->block_hash) << "\"," + << "\"index\": " << info->index << "," + << "\"value\": " << info->value << "," + << "\"height\": " << info->height << "," + << "\"timestamp\": " << info->timestamp << "," + << "\"is_input\": " << info->is_input << ","; + write_xputs_strings(ss, "inputs", info->inputs); + ss << ","; + write_xputs_strings(ss, "outputs", info->outputs); + if (!info->raw_output_script.empty()) + ss << "," + << "\"raw_output_script\": \"" + << pretty_hex(info->raw_output_script) << "\""; + ss << "}"; + json += ss.str(); +} + void keep_query_alive_proxy(const std::error_code& ec, + const statement_entry& statement, python::object handle_finish, query_history_ptr history) { - pyfunction f(handle_finish); - f(ec); + std::string json = "["; + for (auto it = statement.begin(); it != statement.end(); ++it) + { + if (it != statement.begin()) + json += ","; + auto entry = *it; + BITCOIN_ASSERT(entry->loaded_output); + write_info(json, entry->loaded_output); + } + json += "]"; + pyfunction f(handle_finish); + f(ec, json); } void payment_history(async_service_ptr service, blockchain_ptr chain, @@ -72,7 +391,7 @@ void payment_history(async_service_ptr service, blockchain_ptr chain, query_history_ptr history = std::make_shared(*service, chain, txpool); history->start(address, - std::bind(keep_query_alive_proxy, ph::_1, + std::bind(keep_query_alive_proxy, ph::_1, ph::_2, handle_finish, history)); } diff --git a/backends/libbitcoin/history1/__init__.py b/backends/libbitcoin/history1/__init__.py index 9fc479f..f257a17 100644 --- a/backends/libbitcoin/history1/__init__.py +++ b/backends/libbitcoin/history1/__init__.py @@ -1,11 +1,12 @@ import _history -from bitcoin import bind, _1 +from bitcoin import bind, _1, _2 +import json -def wrap_finish(handle_finish, ec): - handle_finish(ec) +def wrap_finish(handle_finish, ec, result_json): + handle_finish(ec, json.loads(result_json)) def payment_history(service, chain, txpool, address, finish): _history.payment_history(service.internal_ptr, chain.internal_ptr, txpool.internal_ptr, address, - bind(wrap_finish, finish, _1)) + bind(wrap_finish, finish, _1, _2)) -- 1.7.1