X-Git-Url: https://git.novaco.in/?p=electrum-server.git;a=blobdiff_plain;f=backends%2Fabe%2F__init__.py;h=745db879b7fa57ebbe0acf13fe9a37356d8ed58a;hp=33ac9a7834a61fb1667576380632f4bcd6dc0cb3;hb=5f39f3e7c91ad135ad86c8cf964d2bef235719b7;hpb=09f0e2f27acc188f8786a1340f1374539c59dfd3 diff --git a/backends/abe/__init__.py b/backends/abe/__init__.py index 33ac9a7..745db87 100644 --- a/backends/abe/__init__.py +++ b/backends/abe/__init__.py @@ -1,4 +1,4 @@ -from Abe.abe import hash_to_address, decode_check_address +from Abe.util import hash_to_address, decode_check_address from Abe.DataStore import DataStore as Datastore_class from Abe import DataStore, readconf, BCDataStream, deserialize, util, base58 @@ -7,7 +7,9 @@ import binascii import thread, traceback, sys, urllib, operator from json import dumps, loads from Queue import Queue -import time +import time, threading + + class AbeStore(Datastore_class): @@ -24,8 +26,9 @@ class AbeStore(Datastore_class): Datastore_class.__init__(self,args) + self.sql_limit = int( config.get('database','limit') ) + self.tx_cache = {} - self.mempool_keys = {} self.bitcoind_url = 'http://%s:%s@%s:%s/' % ( config.get('bitcoind','user'), config.get('bitcoind','password'), config.get('bitcoind','host'), config.get('bitcoind','port')) self.address_queue = Queue() @@ -53,6 +56,10 @@ class AbeStore(Datastore_class): inrows = self.get_tx_inputs(txid, False) for row in inrows: _hash = self.binout(row[6]) + if not _hash: + #print "WARNING: missing tx_in for tx", txid + continue + address = hash_to_address(chr(0), _hash) if self.tx_cache.has_key(address): print "cache: invalidating", address @@ -62,6 +69,10 @@ class AbeStore(Datastore_class): outrows = self.get_tx_outputs(txid, False) for row in outrows: _hash = self.binout(row[6]) + if not _hash: + #print "WARNING: missing tx_out for tx", txid + continue + address = hash_to_address(chr(0), _hash) if self.tx_cache.has_key(address): print "cache: invalidating", address @@ -69,15 +80,20 @@ class AbeStore(Datastore_class): self.address_queue.put(address) def safe_sql(self,sql, params=(), lock=True): + + error = False try: if lock: self.dblock.acquire() ret = self.selectall(sql,params) except: - print "sql error", sql - ret = [] + error = True + traceback.print_exc(file=sys.stdout) finally: if lock: self.dblock.release() + if error: + raise BaseException('sql error') + return ret @@ -116,8 +132,9 @@ class AbeStore(Datastore_class): ORDER BY txin.txin_pos """%(tx_id,), (), lock) + def get_address_out_rows(self, dbhash): - return self.safe_sql(""" SELECT + out = self.safe_sql(""" SELECT b.block_nTime, cc.chain_id, b.block_height, @@ -135,10 +152,15 @@ class AbeStore(Datastore_class): JOIN txout prevout ON (txin.txout_id = prevout.txout_id) JOIN pubkey ON (pubkey.pubkey_id = prevout.pubkey_id) WHERE pubkey.pubkey_hash = ? - AND cc.in_longest = 1""", (dbhash,)) + AND cc.in_longest = 1 + LIMIT ? """, (dbhash,self.sql_limit)) + + if len(out)==self.sql_limit: + raise BaseException('limit reached') + return out def get_address_out_rows_memorypool(self, dbhash): - return self.safe_sql(""" SELECT + out = self.safe_sql(""" SELECT 1, tx.tx_hash, tx.tx_id, @@ -148,10 +170,15 @@ class AbeStore(Datastore_class): JOIN txin ON (txin.tx_id = tx.tx_id) JOIN txout prevout ON (txin.txout_id = prevout.txout_id) JOIN pubkey ON (pubkey.pubkey_id = prevout.pubkey_id) - WHERE pubkey.pubkey_hash = ? """, (dbhash,)) + WHERE pubkey.pubkey_hash = ? + LIMIT ? """, (dbhash,self.sql_limit)) + + if len(out)==self.sql_limit: + raise BaseException('limit reached') + return out def get_address_in_rows(self, dbhash): - return self.safe_sql(""" SELECT + out = self.safe_sql(""" SELECT b.block_nTime, cc.chain_id, b.block_height, @@ -168,10 +195,15 @@ class AbeStore(Datastore_class): JOIN txout ON (txout.tx_id = tx.tx_id) JOIN pubkey ON (pubkey.pubkey_id = txout.pubkey_id) WHERE pubkey.pubkey_hash = ? - AND cc.in_longest = 1""", (dbhash,)) + AND cc.in_longest = 1 + LIMIT ? """, (dbhash,self.sql_limit)) + + if len(out)==self.sql_limit: + raise BaseException('limit reached') + return out def get_address_in_rows_memorypool(self, dbhash): - return self.safe_sql( """ SELECT + out = self.safe_sql( """ SELECT 0, tx.tx_hash, tx.tx_id, @@ -180,10 +212,15 @@ class AbeStore(Datastore_class): FROM tx JOIN txout ON (txout.tx_id = tx.tx_id) JOIN pubkey ON (pubkey.pubkey_id = txout.pubkey_id) - WHERE pubkey.pubkey_hash = ? """, (dbhash,)) + WHERE pubkey.pubkey_hash = ? + LIMIT ? """, (dbhash,self.sql_limit)) + + if len(out)==self.sql_limit: + raise BaseException('limit reached') + return out def get_history(self, addr): - + cached_version = self.tx_cache.get( addr ) if cached_version is not None: return cached_version @@ -231,6 +268,9 @@ class AbeStore(Datastore_class): rows += self.get_address_out_rows_memorypool( dbhash ) address_has_mempool = False + current_id = self.safe_sql("""SELECT last_value FROM tx_seq""") + current_id = current_id[0][0] + for row in rows: is_in, tx_hash, tx_id, pos, value = row tx_hash = self.hashout_hex(tx_hash) @@ -240,10 +280,11 @@ class AbeStore(Datastore_class): # this means that pending transactions were added to the db, even if they are not returned by getmemorypool address_has_mempool = True - # this means pending transactions are returned by getmemorypool - if tx_hash not in self.mempool_keys: + # fixme: we need to detect transactions that became invalid + if current_id - tx_id > 10000: continue + #print "mempool", tx_hash txpoint = { "timestamp": 0, @@ -265,6 +306,9 @@ class AbeStore(Datastore_class): inrows = self.get_tx_inputs(tx_id) for row in inrows: _hash = self.binout(row[6]) + if not _hash: + #print "WARNING: missing tx_in for tx", tx_id, addr + continue address = hash_to_address(chr(0), _hash) txinputs.append(address) txpoint['inputs'] = txinputs @@ -272,6 +316,9 @@ class AbeStore(Datastore_class): outrows = self.get_tx_outputs(tx_id) for row in outrows: _hash = self.binout(row[6]) + if not _hash: + #print "WARNING: missing tx_out for tx", tx_id, addr + continue address = hash_to_address(chr(0), _hash) txoutputs.append(address) txpoint['outputs'] = txoutputs @@ -290,6 +337,7 @@ class AbeStore(Datastore_class): if not row[4]: txpoint['raw_output_script'] = row[1] # cache result + # do not cache mempool results because statuses are ambiguous if not address_has_mempool: self.tx_cache[addr] = txpoints @@ -314,9 +362,6 @@ class AbeStore(Datastore_class): def memorypool_update(store): ds = BCDataStream.BCDataStream() - previous_transactions = store.mempool_keys - store.mempool_keys = [] - postdata = dumps({"method": 'getmemorypool', 'params': [], 'id':'jsonrpc'}) respdata = urllib.urlopen(store.bitcoind_url, postdata).read() @@ -332,12 +377,12 @@ class AbeStore(Datastore_class): tx['hash'] = util.double_sha256(tx['tx']) tx_hash = store.hashin(tx['hash']) - store.mempool_keys.append(tx_hash) if store.tx_find_id_and_value(tx): pass else: tx_id = store.import_tx(tx, False) store.update_tx_cache(tx_id) + #print tx_hash store.commit() @@ -355,22 +400,13 @@ class AbeStore(Datastore_class): def main_iteration(store): - try: - store.dblock.acquire() + with store.dblock: store.catch_up() store.memorypool_update() block_number = store.get_block_number(1) + return block_number - except IOError: - print "IOError: cannot reach bitcoind" - block_number = 0 - except: - traceback.print_exc(file=sys.stdout) - block_number = 0 - finally: - store.dblock.release() - return block_number def catch_up(store): @@ -395,30 +431,50 @@ class BlockchainProcessor(Processor): self.store = AbeStore(config) self.block_number = -1 self.watched_addresses = [] + threading.Timer(10, self.run_store_iteration).start() def process(self, request): + #print "abe process", request + message_id = request['id'] method = request['method'] params = request.get('params',[]) - result = '' + result = None + error = None + if method == 'blockchain.numblocks.subscribe': result = self.block_number + elif method == 'blockchain.address.subscribe': - address = params[0] - self.watch_address(address) - status = self.store.get_status(address) - result = status + try: + address = params[0] + result = self.store.get_status(address) + self.watch_address(address) + except BaseException, e: + error = str(e) + ': ' + address + print "error:", error + elif method == 'blockchain.address.get_history': - address = params[0] - result = self.store.get_history( address ) + try: + address = params[0] + result = self.store.get_history( address ) + except BaseException, e: + error = str(e) + ': ' + address + print "error:", error + elif method == 'blockchain.transaction.broadcast': txo = self.store.send_tx(params[0]) print "sent tx:", txo result = txo + else: - print "unknown method", request + error = "unknown method:%s"%method + - if result != '': + if error: + response = { 'id':message_id, 'error':error } + self.push_response(response) + elif result != '': response = { 'id':message_id, 'result':result } self.push_response(response) @@ -428,27 +484,33 @@ class BlockchainProcessor(Processor): self.watched_addresses.append(addr) - def run(self): + def run_store_iteration(self): - old_block_number = None - while not self.shared.stopped(): - self.block_number = self.store.main_iteration() - - if self.block_number != old_block_number: - old_block_number = self.block_number - print "block number:", self.block_number - self.push_response({ 'method':'blockchain.numblocks.subscribe', 'params':[self.block_number] }) - - while True: - try: - addr = self.store.address_queue.get(False) - except: - break - if addr in self.watched_addresses: - status = self.store.get_status( addr ) - self.push_response({ 'method':'blockchain.address.subscribe', 'params':[addr, status] }) - - time.sleep(10) + try: + block_number = self.store.main_iteration() + except: + traceback.print_exc(file=sys.stdout) + print "terminating" + self.shared.stop() + + if self.shared.stopped(): + print "exit timer" + return + + if self.block_number != block_number: + self.block_number = block_number + print "block number:", self.block_number + self.push_response({ 'method':'blockchain.numblocks.subscribe', 'params':[self.block_number] }) + + while True: + try: + addr = self.store.address_queue.get(False) + except: + break + if addr in self.watched_addresses: + status = self.store.get_status( addr ) + self.push_response({ 'method':'blockchain.address.subscribe', 'params':[addr, status] }) + threading.Timer(10, self.run_store_iteration).start()