X-Git-Url: https://git.novaco.in/?a=blobdiff_plain;f=backends%2Fabe%2F__init__.py;h=7a2cb02f1fe794aa2f5683ae32ca6e8df7f05ce9;hb=42321d7bc69ab6d6f87e97d4295b13844243ba70;hp=c60688bf261ab06206e4ae699448633ef2b6ca0b;hpb=c293b5a5d4e71f555981ba934ec53dedcce36fea;p=electrum-server.git diff --git a/backends/abe/__init__.py b/backends/abe/__init__.py index c60688b..7a2cb02 100644 --- a/backends/abe/__init__.py +++ b/backends/abe/__init__.py @@ -49,18 +49,18 @@ class AbeStore(Datastore_class): coin = config.get('server', 'coin') self.addrtype = 0 if coin == 'litecoin': - print 'Litecoin settings:' + print_log ('Litecoin settings:') datadir = config.get('server','datadir') - print ' datadir = ' + datadir + print_log (' datadir = ' + datadir) args.datadir = [{"dirname":datadir,"chain":"Litecoin","code3":"LTC","address_version":"\u0030"}] - print ' addrtype = 48' + print_log (' addrtype = 48') self.addrtype = 48 Datastore_class.__init__(self,args) # Use 1 (Bitcoin) if chain_id is not sent self.chain_id = self.datadirs[0]["chain_id"] or 1 - print 'Coin chain_id = %d' % self.chain_id + print_log ('Coin chain_id = %d' % self.chain_id) self.sql_limit = int( config.get('database','limit') ) @@ -71,7 +71,8 @@ class AbeStore(Datastore_class): self.address_queue = Queue() - self.lock = threading.Lock() + self.lock = threading.Lock() # for the database + self.cache_lock = threading.Lock() # for the cache self.last_tx_id = 0 self.known_mempool_hashes = [] @@ -86,7 +87,7 @@ class AbeStore(Datastore_class): def import_block(self, b, chain_ids=frozenset()): - #print "import block" + #print_log ("import block") block_id = super(AbeStore, self).import_block(b, chain_ids) for pos in xrange(len(b['transactions'])): tx = b['transactions'][pos] @@ -96,7 +97,7 @@ class AbeStore(Datastore_class): if tx_id: self.update_tx_cache(tx_id) else: - print "error: import_block: no tx_id" + print_log ("error: import_block: no tx_id") return block_id @@ -105,26 +106,30 @@ class AbeStore(Datastore_class): for row in inrows: _hash = self.binout(row[6]) if not _hash: - #print "WARNING: missing tx_in for tx", txid + #print_log ("WARNING: missing tx_in for tx", txid) continue address = hash_to_address(chr(self.addrtype), _hash) - if self.tx_cache.has_key(address): - print "cache: invalidating", address - self.tx_cache.pop(address) + with self.cache_lock: + if self.tx_cache.has_key(address): + print_log ("cache: invalidating", address) + self.tx_cache.pop(address) + self.address_queue.put(address) outrows = self.get_tx_outputs(txid, False) for row in outrows: _hash = self.binout(row[6]) if not _hash: - #print "WARNING: missing tx_out for tx", txid + #print_log ("WARNING: missing tx_out for tx", txid) continue address = hash_to_address(chr(self.addrtype), _hash) - if self.tx_cache.has_key(address): - print "cache: invalidating", address - self.tx_cache.pop(address) + with self.cache_lock: + if self.tx_cache.has_key(address): + print_log ("cache: invalidating", address) + self.tx_cache.pop(address) + self.address_queue.put(address) def safe_sql(self,sql, params=(), lock=True): @@ -269,13 +274,16 @@ class AbeStore(Datastore_class): raise BaseException('limit reached') return out - def get_history(self, addr): - with self.lock: + + def get_history(self, addr, cache_only=False): + with self.cache_lock: cached_version = self.tx_cache.get( addr ) if cached_version is not None: return cached_version + if cache_only: return -1 + version, binaddr = decode_check_address(addr) if binaddr is None: return None @@ -292,7 +300,7 @@ class AbeStore(Datastore_class): try: nTime, chain_id, height, is_in, blk_hash, tx_hash, tx_id, pos, value = row except: - print "cannot unpack row", row + print_log ("cannot unpack row", row) break tx_hash = self.hashout_hex(tx_hash) txpoint = { @@ -327,13 +335,13 @@ class AbeStore(Datastore_class): # discard transactions that are too old if self.last_tx_id - tx_id > 50000: - print "discarding tx id", tx_id + print_log ("discarding tx id", tx_id) continue # this means that pending transactions were added to the db, even if they are not returned by getmemorypool address_has_mempool = True - #print "mempool", tx_hash + #print_log ("mempool", tx_hash) txpoint = { "timestamp": 0, "height": 0, @@ -355,7 +363,7 @@ class AbeStore(Datastore_class): for row in inrows: _hash = self.binout(row[6]) if not _hash: - #print "WARNING: missing tx_in for tx", tx_id, addr + #print_log ("WARNING: missing tx_in for tx", tx_id, addr) continue address = hash_to_address(chr(self.addrtype), _hash) txinputs.append(address) @@ -365,7 +373,7 @@ class AbeStore(Datastore_class): for row in outrows: _hash = self.binout(row[6]) if not _hash: - #print "WARNING: missing tx_out for tx", tx_id, addr + #print_log ("WARNING: missing tx_out for tx", tx_id, addr) continue address = hash_to_address(chr(self.addrtype), _hash) txoutputs.append(address) @@ -386,42 +394,27 @@ class AbeStore(Datastore_class): txpoint.pop('tx_id') + + txpoints = map(lambda x: {'tx_hash':x['tx_hash'], 'height':x['height']}, txpoints) + out = [] + for item in txpoints: + if item not in out: out.append(item) + # cache result - # do not cache mempool results because statuses are ambiguous - if not address_has_mempool: - with self.lock: - self.tx_cache[addr] = txpoints + ## do not cache mempool results because statuses are ambiguous + #if not address_has_mempool: + with self.cache_lock: + self.tx_cache[addr] = out - return txpoints - - def get_history2(self, addr): - h = self.get_history(addr) - out = map(lambda x: {'tx_hash':x['tx_hash'], 'height':x['height']}, h) - out2 = [] - for item in out: - if item not in out2: out2.append(item) - return out2 - - - def get_status(self,addr): - # get address status, i.e. the last block for that address. - tx_points = self.get_history(addr) - if not tx_points: - status = None - else: - lastpoint = tx_points[-1] - status = lastpoint['block_hash'] - # this is a temporary hack; move it up once old clients have disappeared - if status == 'mempool': # and session['version'] != "old": - status = status + ':%d'% len(tx_points) - return status - - def get_status2(self,addr): + return out + + + def get_status(self, addr, cache_only=False): # for 0.5 clients - tx_points = self.get_history2(addr) - if not tx_points: - return None + tx_points = self.get_history(addr, cache_only) + if cache_only and tx_points == -1: return -1 + if not tx_points: return None status = '' for tx in tx_points: status += tx.get('tx_hash') + ':%d:' % tx.get('height') @@ -454,7 +447,7 @@ class AbeStore(Datastore_class): def get_chunk(self, index): - with self.lock: + with self.cache_lock: msg = self.chunk_cache.get(index) if msg: return msg @@ -470,7 +463,7 @@ class AbeStore(Datastore_class): prev_block_hash, block_height FROM chain_summary - WHERE block_height >= %d AND block_height< %d AND in_longest = 1"""%(index*2016, (index+1)*2016) + WHERE block_height >= %d AND block_height< %d AND in_longest = 1 ORDER BY block_height"""%(index*2016, (index+1)*2016) out = self.safe_sql(sql) msg = '' @@ -483,12 +476,12 @@ class AbeStore(Datastore_class): if h.get('block_height')==0: h['prev_block_hash'] = "0"*64 msg += header_to_string(h) - #print "hash", encode(Hash(msg.decode('hex'))) + #print_log ("hash", encode(Hash(msg.decode('hex')))) #if h.get('block_height')==1:break - with self.lock: + with self.cache_lock: self.chunk_cache[index] = msg - print "get_chunk", index, len(msg) + print_log ("get_chunk", index, len(msg)) return msg @@ -567,16 +560,20 @@ class AbeStore(Datastore_class): ds = BCDataStream.BCDataStream() postdata = dumps({"method": 'getrawmempool', 'params': [], 'id':'jsonrpc'}) respdata = urllib.urlopen(store.bitcoind_url, postdata).read() + r = loads(respdata) if r['error'] != None: - print r['error'] + print_log (r['error']) return mempool_hashes = r.get('result') + num_new_tx = 0 + for tx_hash in mempool_hashes: if tx_hash in store.known_mempool_hashes: continue store.known_mempool_hashes.append(tx_hash) + num_new_tx += 1 postdata = dumps({"method": 'getrawtransaction', 'params': [tx_hash], 'id':'jsonrpc'}) respdata = urllib.urlopen(store.bitcoind_url, postdata).read() @@ -594,10 +591,11 @@ class AbeStore(Datastore_class): else: tx_id = store.import_tx(tx, False) store.update_tx_cache(tx_id) - #print tx_hash + #print_log (tx_hash) store.commit() store.known_mempool_hashes = mempool_hashes + return num_new_tx def send_tx(self,tx): @@ -614,14 +612,22 @@ class AbeStore(Datastore_class): def main_iteration(self): with self.lock: + t1 = time.time() self.catch_up() - self.memorypool_update() + t2 = time.time() + time_catch_up = t2 - t1 + n = self.memorypool_update() + time_mempool = time.time() - t2 height = self.get_block_number( self.chain_id ) - try: self.chunk_cache.pop(height/2016) - except: pass + + with self.cache_lock: + try: + self.chunk_cache.pop(height/2016) + except: + pass block_header = self.get_block_header( height ) - return block_header + return block_header, time_catch_up, time_mempool, n @@ -639,24 +645,32 @@ class AbeStore(Datastore_class): -from processor import Processor +from processor import Processor, print_log class BlockchainProcessor(Processor): - def __init__(self, config): + def __init__(self, config, shared): Processor.__init__(self) self.store = AbeStore(config) self.watched_addresses = [] + self.shared = shared # catch_up first - self.block_header = self.store.main_iteration() + self.block_header, time_catch_up, time_mempool, n = self.store.main_iteration() self.block_number = self.block_header.get('block_height') - print "blockchain: %d blocks"%self.block_number + print_log ("blockchain: %d blocks"%self.block_number) threading.Timer(10, self.run_store_iteration).start() - def process(self, request): - #print "abe process", request + + def add_request(self, request): + # see if we can get if from cache. if not, add to queue + if self.process( request, cache_only = True) == -1: + self.queue.put(request) + + + def process(self, request, cache_only = False): + #print_log ("abe process", request) message_id = request['id'] method = request['method'] @@ -673,66 +687,58 @@ class BlockchainProcessor(Processor): elif method == 'blockchain.address.subscribe': try: address = params[0] - result = self.store.get_status(address) - self.watch_address(address) - except BaseException, e: - error = str(e) + ': ' + address - print "error:", error - - elif method == 'blockchain.address.subscribe2': - try: - address = params[0] - result = self.store.get_status2(address) + result = self.store.get_status(address, cache_only) self.watch_address(address) except BaseException, e: error = str(e) + ': ' + address - print "error:", error + print_log ("error:", error) elif method == 'blockchain.address.get_history': try: address = params[0] - result = self.store.get_history( address ) + result = self.store.get_history( address, cache_only ) except BaseException, e: error = str(e) + ': ' + address - print "error:", error - - elif method == 'blockchain.address.get_history2': - try: - address = params[0] - result = self.store.get_history2( address ) - except BaseException, e: - error = str(e) + ': ' + address - print "error:", error + print_log ("error:", error) elif method == 'blockchain.block.get_header': - try: - height = params[0] - result = self.store.get_block_header( height ) - except BaseException, e: - error = str(e) + ': %d'% height - print "error:", error - + if cache_only: + result = -1 + else: + try: + height = params[0] + result = self.store.get_block_header( height ) + except BaseException, e: + error = str(e) + ': %d'% height + print_log ("error:", error) + elif method == 'blockchain.block.get_chunk': - try: - index = params[0] - result = self.store.get_chunk( index ) - except BaseException, e: - error = str(e) + ': %d'% index - print "error:", error - + if cache_only: + result = -1 + else: + try: + index = params[0] + result = self.store.get_chunk( index ) + except BaseException, e: + error = str(e) + ': %d'% index + print_log ("error:", error) + elif method == 'blockchain.transaction.broadcast': txo = self.store.send_tx(params[0]) - print "sent tx:", txo + print_log ("sent tx:", txo) result = txo elif method == 'blockchain.transaction.get_merkle': - try: - tx_hash = params[0] - result = self.store.get_tx_merkle(tx_hash ) - except BaseException, e: - error = str(e) + ': ' + tx_hash - print "error:", error - + if cache_only: + result = -1 + else: + try: + tx_hash = params[0] + result = self.store.get_tx_merkle(tx_hash ) + except BaseException, e: + error = str(e) + ': ' + tx_hash + print_log ("error:", error) + elif method == 'blockchain.transaction.get': try: tx_hash = params[0] @@ -740,11 +746,12 @@ class BlockchainProcessor(Processor): result = self.store.get_raw_tx(tx_hash, height ) except BaseException, e: error = str(e) + ': ' + tx_hash - print "error:", error + print_log ("error:", error) else: error = "unknown method:%s"%method + if cache_only and result == -1: return -1 if error: response = { 'id':message_id, 'error':error } @@ -762,21 +769,21 @@ class BlockchainProcessor(Processor): def run_store_iteration(self): try: - t1 = time.time() - block_header = self.store.main_iteration() - t2 = time.time() - t1 + block_header, time_catch_up, time_mempool, n = self.store.main_iteration() except: traceback.print_exc(file=sys.stdout) - print "terminating" + print_log ("terminating") self.shared.stop() if self.shared.stopped(): - print "exit timer" + print_log ("exit timer") return + #print_log ("block number: %d (%.3fs) mempool:%d (%.3fs)"%(self.block_number, time_catch_up, n, time_mempool)) + if self.block_number != block_header.get('block_height'): self.block_number = block_header.get('block_height') - print "block number: %d (%.3f seconds)"%(self.block_number, t2) + print_log ("block number: %d (%.3fs)"%(self.block_number, time_catch_up)) self.push_response({ 'id': None, 'method':'blockchain.numblocks.subscribe', 'params':[self.block_number] }) if self.block_header != block_header: @@ -790,10 +797,11 @@ class BlockchainProcessor(Processor): except: break if addr in self.watched_addresses: - status = self.store.get_status( addr ) - status2 = self.store.get_status2( addr ) - self.push_response({ 'id': None, 'method':'blockchain.address.subscribe', 'params':[addr, status] }) - self.push_response({ 'id': None, 'method':'blockchain.address.subscribe2', 'params':[addr, status2] }) + try: + status = self.store.get_status( addr ) + self.push_response({ 'id': None, 'method':'blockchain.address.subscribe', 'params':[addr, status] }) + except: + break threading.Timer(10, self.run_store_iteration).start()