X-Git-Url: https://git.novaco.in/?a=blobdiff_plain;f=backends%2Fbitcoind%2Fblockchain_processor.py;h=220be097ddea98c69cfbe1c708c891d51d3070de;hb=251b29f82b9d49ba45c06e5a13358afd1a5a052f;hp=10b81c8e0f81e21173e184918404a2a5dc12cce1;hpb=a3a2df3a14ab265c5bd5c2dbbc4ba299d3f8489c;p=electrum-server.git diff --git a/backends/bitcoind/blockchain_processor.py b/backends/bitcoind/blockchain_processor.py index 10b81c8..220be09 100644 --- a/backends/bitcoind/blockchain_processor.py +++ b/backends/bitcoind/blockchain_processor.py @@ -1,7 +1,6 @@ import ast import hashlib from json import dumps, loads -import leveldb import os from Queue import Queue import random @@ -24,7 +23,12 @@ class BlockchainProcessor(Processor): self.shared = shared self.config = config self.up_to_date = False - self.watched_addresses = [] + + self.watch_lock = threading.Lock() + self.watch_blocks = [] + self.watch_headers = [] + self.watched_addresses = {} + self.history_cache = {} self.chunk_cache = {} self.cache_lock = threading.Lock() @@ -32,17 +36,31 @@ class BlockchainProcessor(Processor): self.mempool_addresses = {} self.mempool_hist = {} - self.mempool_hashes = [] + self.mempool_hashes = set([]) self.mempool_lock = threading.Lock() self.address_queue = Queue() - self.dbpath = config.get('leveldb', 'path') + + try: + self.use_plyvel = config.getboolean('leveldb', 'use_plyvel') + except: + self.use_plyvel = False + print_log('use_plyvel:', self.use_plyvel) + + # don't use the same database for plyvel, because python-leveldb uses snappy compression + self.dbpath = config.get('leveldb', 'path_plyvel' if self.use_plyvel else 'path') + self.pruning_limit = config.getint('leveldb', 'pruning_limit') self.db_version = 1 # increase this when database needs to be updated self.dblock = threading.Lock() try: - self.db = leveldb.LevelDB(self.dbpath, paranoid_checks=True) + if self.use_plyvel: + import plyvel + self.db = plyvel.DB(self.dbpath, create_if_missing=True, paranoid_checks=None, compression=None) + else: + import leveldb + self.db = leveldb.LevelDB(self.dbpath, paranoid_checks=False) except: traceback.print_exc(file=sys.stdout) self.shared.stop() @@ -68,7 +86,7 @@ class BlockchainProcessor(Processor): self.sent_header = None try: - hist = self.deserialize(self.db.Get('height')) + hist = self.deserialize(self.db_get('height')) self.last_hash, self.height, db_version = hist[0] print_log("Database version", self.db_version) print_log("Blockchain height", self.height) @@ -103,6 +121,35 @@ class BlockchainProcessor(Processor): threading.Timer(10, self.main_iteration).start() + + def db_get(self, key): + if self.use_plyvel: + return self.db.get(key) + else: + try: + return self.db.Get(key) + except KeyError: + return None + + def batch_put(self, batch, key, value): + if self.use_plyvel: + batch.put(key, value) + else: + batch.Put(key, value) + + def batch_delete(self, batch, key): + if self.use_plyvel: + batch.delete(key) + else: + batch.Delete(key) + + def batch_write(self, batch, sync): + if self.use_plyvel: + batch.write()#, sync=sync) + else: + self.db.Write(batch, sync=sync) + + def bitcoind(self, method, params=[]): postdata = dumps({"method": method, 'params': params, 'id': 'jsonrpc'}) try: @@ -272,9 +319,14 @@ class BlockchainProcessor(Processor): with self.dblock: try: - hist = self.deserialize(self.db.Get(addr)) + hist = self.deserialize(self.db_get(str((addr)))) is_known = True except: + self.shared.stop() + raise + if hist: + is_known = True + else: hist = [] is_known = False @@ -473,21 +525,21 @@ class BlockchainProcessor(Processor): return tx_hashes, txdict def get_undo_info(self, height): - s = self.db.Get("undo%d" % (height % 100)) + s = self.db_get("undo%d" % (height % 100)) return eval(s) def write_undo_info(self, batch, height, undo_info): if self.is_test or height > self.bitcoind_height - 100: - batch.Put("undo%d" % (height % 100), repr(undo_info)) + self.batch_put(batch, "undo%d" % (height % 100), repr(undo_info)) def import_block(self, block, block_hash, block_height, sync, revert=False): self.batch_list = {} # address -> history self.batch_txio = {} # transaction i/o -> address - block_inputs = [] - block_outputs = [] - addr_to_read = [] + block_inputs = set([]) + block_outputs = set([]) + addr_to_read = set([]) # deserialize transactions t0 = time.time() @@ -507,51 +559,51 @@ class BlockchainProcessor(Processor): for tx in txdict.values(): for x in tx.get('inputs'): txi = (x.get('prevout_hash') + int_to_hex(x.get('prevout_n'), 4)).decode('hex') - block_inputs.append(txi) + block_inputs.add(txi) - block_inputs.sort() - for txi in block_inputs: + #block_inputs.sort() + for txi in sorted(block_inputs): try: - addr = self.db.Get(txi) - except KeyError: - # the input could come from the same block - continue + addr = self.db_get(txi) + if addr is None: + # the input could come from the same block + continue except: traceback.print_exc(file=sys.stdout) self.shared.stop() raise self.batch_txio[txi] = addr - addr_to_read.append(addr) + addr_to_read.add(addr) else: for txid, tx in txdict.items(): for x in tx.get('outputs'): txo = (txid + int_to_hex(x.get('index'), 4)).decode('hex') - block_outputs.append(txo) - addr_to_read.append( x.get('address') ) + block_outputs.add(txo) + addr_to_read.add( x.get('address') ) undo = undo_info.get(txid) for i, x in enumerate(tx.get('inputs')): addr = undo['prev_addr'][i] - addr_to_read.append(addr) - - + addr_to_read.add(addr) + #time spent reading txio + t000 = time.time() # read histories of addresses for txid, tx in txdict.items(): for x in tx.get('outputs'): - addr_to_read.append(x.get('address')) + addr_to_read.add(x.get('address')) - addr_to_read.sort() - for addr in addr_to_read: + #addr_to_read.sort() + for addr in sorted(addr_to_read): try: - self.batch_list[addr] = self.db.Get(addr) - except KeyError: - self.batch_list[addr] = '' + h = self.db_get(addr) + self.batch_list[addr] = '' if h is None else h except: + print "db get error", addr traceback.print_exc(file=sys.stdout) self.shared.stop() raise @@ -620,9 +672,14 @@ class BlockchainProcessor(Processor): max_addr = '' t2 = time.time() - batch = leveldb.WriteBatch() + if self.use_plyvel: + batch = self.db.write_batch() + else: + import leveldb + batch = leveldb.WriteBatch() + for addr, serialized_hist in self.batch_list.items(): - batch.Put(addr, serialized_hist) + self.batch_put(batch, addr, serialized_hist) l = len(serialized_hist)/80 if l > max_len: max_len = l @@ -631,47 +688,90 @@ class BlockchainProcessor(Processor): if not revert: # add new created outputs for txio, addr in self.batch_txio.items(): - batch.Put(txio, addr) + self.batch_put(batch, txio, addr) # delete spent inputs for txi in block_inputs: - batch.Delete(txi) + self.batch_delete(batch, txi) # add undo info self.write_undo_info(batch, block_height, undo_info) else: # restore spent inputs for txio, addr in self.batch_txio.items(): # print "restoring spent input", repr(txio) - batch.Put(txio, addr) + self.batch_put(batch, txio, addr) # delete spent outputs for txo in block_outputs: - batch.Delete(txo) + self.batch_delete(batch, txo) # add the max - batch.Put('height', self.serialize([(block_hash, block_height, self.db_version)])) + self.batch_put(batch, 'height', self.serialize([(block_hash, block_height, self.db_version)])) # actual write - self.db.Write(batch, sync=sync) + self.batch_write(batch, sync) t3 = time.time() - if t3 - t0 > 10 and not sync: - print_log("block", block_height, - "parse:%0.2f " % (t00 - t0), - "read:%0.2f " % (t1 - t00), - "proc:%.2f " % (t2-t1), + if t3 - t0 > 0 and not sync: + print_log("block %d "%block_height, + "total:%0.2f " % (t3 - t0), + #"parse:%0.2f " % (t00 - t0), + "read_txio[%4d]:%0.2f " % (len(block_inputs), t000 - t00), + "read_addr[%4d]:%0.2f " % (len(addr_to_read), t1 - t000), + #"proc:%.2f " % (t2-t1), "write:%.2f " % (t3-t2), "max:", max_len, max_addr) for addr in self.batch_list.keys(): self.invalidate_cache(addr) - def add_request(self, request): + def add_request(self, session, request): # see if we can get if from cache. if not, add to queue - if self.process(request, cache_only=True) == -1: - self.queue.put(request) + if self.process(session, request, cache_only=True) == -1: + self.queue.put((session, request)) + - def process(self, request, cache_only=False): - #print "abe process", request + def do_subscribe(self, method, params, session): + with self.watch_lock: + if method == 'blockchain.numblocks.subscribe': + if session not in self.watch_blocks: + self.watch_blocks.append(session) + elif method == 'blockchain.headers.subscribe': + if session not in self.watch_headers: + self.watch_headers.append(session) + + elif method == 'blockchain.address.subscribe': + address = params[0] + l = self.watched_addresses.get(address) + if l is None: + self.watched_addresses[address] = [session] + elif session not in l: + l.append(session) + + + def do_unsubscribe(self, method, params, session): + with self.watch_lock: + if method == 'blockchain.numblocks.subscribe': + if session in self.watch_blocks: + self.watch_blocks.remove(session) + elif method == 'blockchain.headers.subscribe': + if session in self.watch_headers: + self.watch_headers.remove(session) + elif method == "blockchain.address.subscribe": + addr = params[0] + l = self.watched_addresses.get(addr) + if not l: + return + if session in l: + l.remove(session) + if session in l: + print "error rc!!" + self.shared.stop() + if l == []: + self.watched_addresses.pop(addr) + + + def process(self, session, request, cache_only=False): + message_id = request['id'] method = request['method'] params = request.get('params', []) @@ -688,22 +788,6 @@ class BlockchainProcessor(Processor): try: address = params[0] result = self.get_status(address, cache_only) - self.watch_address(address) - except BaseException, e: - error = str(e) + ': ' + address - print_log("error:", error) - - elif method == 'blockchain.address.unsubscribe': - try: - password = params[0] - address = params[1] - if password == self.config.get('server', 'password'): - self.watched_addresses.remove(address) - # print_log('unsubscribed', address) - result = "ok" - else: - print_log('incorrect password') - result = "authentication error" except BaseException, e: error = str(e) + ': ' + address print_log("error:", error) @@ -774,13 +858,10 @@ class BlockchainProcessor(Processor): return -1 if error: - self.push_response({'id': message_id, 'error': error}) + self.push_response(session, {'id': message_id, 'error': error}) elif result != '': - self.push_response({'id': message_id, 'result': result}) + self.push_response(session, {'id': message_id, 'result': result}) - def watch_address(self, addr): - if addr not in self.watched_addresses: - self.watched_addresses.append(addr) def getfullblock(self, block_hash): block = self.bitcoind('getblock', [block_hash]) @@ -806,7 +887,9 @@ class BlockchainProcessor(Processor): rawtxdata = [] for ir in r: if ir['error'] is not None: - raise BaseException(r['error']) + self.shared.stop() + print_log("Error: make sure you run bitcoind with txindex=1; use -reindex if needed.") + raise BaseException(ir['error']) rawtxdata.append(ir['result']) block['tx'] = rawtxdata return block @@ -859,10 +942,15 @@ class BlockchainProcessor(Processor): self.header = self.block2header(self.bitcoind('getblock', [self.last_hash])) + if self.shared.stopped() and self.use_plyvel: + print_log( "closing database" ) + self.db.close() + + def memorypool_update(self): - mempool_hashes = self.bitcoind('getrawmempool') + mempool_hashes = set(self.bitcoind('getrawmempool')) + touched_addresses = set([]) - touched_addresses = [] for tx_hash in mempool_hashes: if tx_hash in self.mempool_hashes: continue @@ -877,16 +965,16 @@ class BlockchainProcessor(Processor): addr = x.get('address') if addr and addr not in mpa: mpa.append(addr) - touched_addresses.append(addr) + touched_addresses.add(addr) for x in tx.get('outputs'): addr = x.get('address') if addr and addr not in mpa: mpa.append(addr) - touched_addresses.append(addr) + touched_addresses.add(addr) self.mempool_addresses[tx_hash] = mpa - self.mempool_hashes.append(tx_hash) + self.mempool_hashes.add(tx_hash) # remove older entries from mempool_hashes self.mempool_hashes = mempool_hashes @@ -896,7 +984,7 @@ class BlockchainProcessor(Processor): if tx_hash not in self.mempool_hashes: self.mempool_addresses.pop(tx_hash) for addr in addresses: - touched_addresses.append(addr) + touched_addresses.add(addr) # rebuild mempool histories new_mempool_hist = {} @@ -921,13 +1009,18 @@ class BlockchainProcessor(Processor): print_log("cache: invalidating", address) self.history_cache.pop(address) - if address in self.watched_addresses: + with self.watch_lock: + sessions = self.watched_addresses.get(address) + + if sessions: # TODO: update cache here. if new value equals cached value, do not send notification - self.address_queue.put(address) + self.address_queue.put((address,sessions)) def main_iteration(self): if self.shared.stopped(): print_log("blockchain processor terminating") + if self.use_plyvel: + self.db.close() return with self.dblock: @@ -939,33 +1032,36 @@ class BlockchainProcessor(Processor): if self.sent_height != self.height: self.sent_height = self.height - self.push_response({ - 'id': None, - 'method': 'blockchain.numblocks.subscribe', - 'params': [self.height], - }) + for session in self.watch_blocks: + self.push_response(session, { + 'id': None, + 'method': 'blockchain.numblocks.subscribe', + 'params': [self.height], + }) if self.sent_header != self.header: print_log("blockchain: %d (%.3fs)" % (self.height, t2 - t1)) self.sent_header = self.header - self.push_response({ - 'id': None, - 'method': 'blockchain.headers.subscribe', - 'params': [self.header], - }) + for session in self.watch_headers: + self.push_response(session, { + 'id': None, + 'method': 'blockchain.headers.subscribe', + 'params': [self.header], + }) while True: try: - addr = self.address_queue.get(False) + addr, sessions = self.address_queue.get(False) except: break - if addr in self.watched_addresses: - status = self.get_status(addr) - self.push_response({ - 'id': None, - 'method': 'blockchain.address.subscribe', - 'params': [addr, status], - }) + + status = self.get_status(addr) + for session in sessions: + self.push_response(session, { + 'id': None, + 'method': 'blockchain.address.subscribe', + 'params': [addr, status], + }) if not self.shared.stopped(): threading.Timer(10, self.main_iteration).start()