X-Git-Url: https://git.novaco.in/?a=blobdiff_plain;f=backends%2Fbitcoind%2Fstorage.py;h=098d12e61484e5ba90e9516d5c4916df2ab020c3;hb=d0439647dfc60efce32e0a13b0ec02428ba412cf;hp=12c8609f77acf550fd8c75df64765ba8dd0303f5;hpb=477dadc5c491755794725de3abcf3b6ae4542c80;p=electrum-server.git diff --git a/backends/bitcoind/storage.py b/backends/bitcoind/storage.py index 12c8609..098d12e 100644 --- a/backends/bitcoind/storage.py +++ b/backends/bitcoind/storage.py @@ -25,14 +25,15 @@ class Storage(object): self.test_reorgs = test_reorgs try: + self.db_utxo = plyvel.DB(os.path.join(self.dbpath,'utxo'), create_if_missing=True, compression=None) self.db_addr = plyvel.DB(os.path.join(self.dbpath,'addr'), create_if_missing=True, compression=None) - self.db = plyvel.DB(os.path.join(self.dbpath,'utxo'), create_if_missing=True, compression=None) + self.db_hist = plyvel.DB(os.path.join(self.dbpath,'hist'), create_if_missing=True, compression=None) self.db_undo = plyvel.DB(os.path.join(self.dbpath,'undo'), create_if_missing=True, compression=None) except: traceback.print_exc(file=sys.stdout) self.shared.stop() - self.db_version = 2 # increase this when database needs to be updated + self.db_version = 3 # increase this when database needs to be updated try: self.last_hash, self.height, db_version = ast.literal_eval(self.db_undo.get('height')) print_log("Database version", self.db_version) @@ -41,7 +42,7 @@ class Storage(object): #traceback.print_exc(file=sys.stdout) print_log('initializing database') self.height = 0 - self.last_hash = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' + self.last_hash = '00000a060336cbb72fe969666d337b87198b1add2abaa59cca226820b32933a4' db_version = self.db_version # write root self.put_node('', {}) @@ -53,6 +54,11 @@ class Storage(object): return + # compute root hash + d = self.get_node('') + self.root_hash, v = self.get_node_hash('',d,None) + print_log("UTXO tree root hash:", self.root_hash.encode('hex')) + print_log("Coins in database:", v) # convert between bitcoin addresses and 20 bytes keys used for storage. def address_to_key(self, addr): @@ -62,23 +68,83 @@ class Storage(object): return hash_160_to_bc_address(addr) + def get_proof(self, addr): + key = self.address_to_key(addr) + i = self.db_utxo.iterator(start=key) + k, _ = i.next() + + p = self.get_path(k) + p.append(k) + + out = [] + for item in p: + v = self.db_utxo.get(item) + out.append((item.encode('hex'), v.encode('hex'))) + + return out + + + def get_balance(self, addr): + key = self.address_to_key(addr) + i = self.db_utxo.iterator(start=key) + k, _ = i.next() + if not k.startswith(key): + return 0 + p = self.get_parent(k) + d = self.get_node(p) + letter = k[len(p)] + return d[letter][1] + + + def listunspent(self, addr): + key = self.address_to_key(addr) + + out = [] + for k, v in self.db_utxo.iterator(start=key): + if not k.startswith(key): + break + if len(k) == KEYLENGTH: + txid = k[20:52].encode('hex') + txpos = hex_to_int(k[52:56]) + h = hex_to_int(v[8:12]) + v = hex_to_int(v[0:8]) + out.append({'tx_hash': txid, 'tx_pos':txpos, 'height': h, 'value':v}) + + out.sort(key=lambda x:x['height']) + return out + + def get_history(self, addr): - addr = self.address_to_key(addr) - x = self.db_addr.get(addr) - if x is None: - return '' - try: - _hash, v, h = x - return h - except: - traceback.print_exc(file=sys.stdout) - self.shared.stop() - raise + out = [] + + o = self.listunspent(addr) + for item in o: + out.append((item['tx_hash'], item['height'])) + + h = self.db_hist.get(addr) + + while h: + item = h[0:80] + h = h[80:] + txi = item[0:32].encode('hex') + hi = hex_to_int(item[36:40]) + txo = item[40:72].encode('hex') + ho = hex_to_int(item[76:80]) + out.append((txi, hi)) + out.append((txo, ho)) + + # sort + out.sort(key=lambda x:x[1]) + + # uniqueness + out = set(out) + + return map(lambda x: {'tx_hash':x[0], 'height':x[1]}, out) + def get_address(self, txi): - addr = self.db.get(txi) - return self.key_to_address(addr) if addr else None + return self.db_addr.get(txi) def get_undo_info(self, height): @@ -123,12 +189,12 @@ class Storage(object): if batch: batch.put(key, out) else: - self.db_addr.put(key, out) + self.db_utxo.put(key, out) def get_node(self, key): - s = self.db_addr.get(key) + s = self.db_utxo.get(key) if s is None: return @@ -149,13 +215,13 @@ class Storage(object): return d - def add_address(self, target, value): + def add_address(self, target, value, height): assert len(target) == KEYLENGTH word = target key = '' path = [ '' ] - i = self.db_addr.iterator() + i = self.db_utxo.iterator() while key != target: @@ -202,8 +268,8 @@ class Storage(object): break # write - s = int_to_hex(value, 8).decode('hex') - self.db_addr.put(target, s) + s = (int_to_hex(value, 8) + int_to_hex(height,4)).decode('hex') + self.db_utxo.put(target, s) # the hash of a node is the txid _hash = target[20:52] self.update_node_hash(target, path, _hash, value) @@ -262,7 +328,7 @@ class Storage(object): # batch write modified nodes - batch = self.db_addr.write_batch() + batch = self.db_utxo.write_batch() for k, v in nodes.items(): self.put_node(k, v, batch) batch.write() @@ -292,7 +358,7 @@ class Storage(object): word = target key = '' path = [ '' ] - i = self.db_addr.iterator(start='') + i = self.db_utxo.iterator(start='') while key != target: @@ -315,7 +381,7 @@ class Storage(object): assert key not in path path.append(key) else: - print_log('not in tree', self.db_addr.get(key+word[0]), new_key.encode('hex')) + print_log('not in tree', self.db_utxo.get(key+word[0]), new_key.encode('hex')) return False else: assert key in path @@ -327,34 +393,36 @@ class Storage(object): def delete_address(self, leaf): path = self.get_path(leaf) if path is False: - print_log("addr not in tree", leaf.encode('hex'), self.key_to_address(leaf[0:20]), self.db_addr.get(leaf)) + print_log("addr not in tree", leaf.encode('hex'), self.key_to_address(leaf[0:20]), self.db_utxo.get(leaf)) raise - self.db_addr.delete(leaf) + s = self.db_utxo.get(leaf) + + self.db_utxo.delete(leaf) if leaf in self.hash_list: self.hash_list.pop(leaf) parent = path[-1] letter = leaf[len(parent)] items = self.get_node(parent) - vv = items.pop(letter) + items.pop(letter) # remove key if it has a single child if len(items) == 1: letter, v = items.items()[0] - self.db_addr.delete(parent) + self.db_utxo.delete(parent) if parent in self.hash_list: self.hash_list.pop(parent) # we need the exact length for the iteration - i = self.db_addr.iterator() + i = self.db_utxo.iterator() i.seek(parent+letter) k, v = i.next() # note: k is not necessarily a leaf if len(k) == KEYLENGTH: - _hash, value = k[20:52], hex_to_int(v) + _hash, value = k[20:52], hex_to_int(v[0:8]) else: _hash, value = None, None @@ -365,11 +433,11 @@ class Storage(object): _hash, value = None, None self.update_node_hash(parent, path[:-1], _hash, value) - return vv + return s def get_children(self, x): - i = self.db_addr.iterator() + i = self.db_utxo.iterator() l = 0 while l <256: i.seek(x+chr(l)) @@ -388,7 +456,7 @@ class Storage(object): def get_parent(self, x): """ return parent and skip string""" - i = self.db_addr.iterator() + i = self.db_utxo.iterator() for j in range(len(x)): p = x[0:-j-1] i.seek(p) @@ -409,8 +477,9 @@ class Storage(object): def close(self): + self.db_utxo.close() self.db_addr.close() - self.db.close() + self.db_hist.close() self.db_undo.close() @@ -419,10 +488,10 @@ class Storage(object): txo = (tx_hash + int_to_hex(tx_pos, 4)).decode('hex') # write the new history - self.add_address(key + txo, value) + self.add_address(key + txo, value, tx_height) # backlink - self.db.put(txo, key) + self.db_addr.put(txo, addr) @@ -434,19 +503,36 @@ class Storage(object): self.delete_address(key + txo) # backlink - self.db.delete(txo) + self.db_addr.delete(txo) + def get_utxo_value(self, addr, txi): + key = self.address_to_key(addr) + leaf = key + txi + s = self.db_utxo.get(leaf) + value = hex_to_int(s[0:8]) + return value + def set_spent(self, addr, txi, txid, index, height, undo): key = self.address_to_key(addr) leaf = key + txi - _hash, value = self.delete_address(leaf) - undo[leaf] = value + s = self.delete_address(leaf) + value = hex_to_int(s[0:8]) + in_height = hex_to_int(s[8:12]) + undo[leaf] = value, in_height # delete backlink txi-> addr - self.db.delete(txi) + self.db_addr.delete(txi) + + # add to history + s = self.db_hist.get(addr) + if s is None: s = '' + txo = (txid + int_to_hex(index,4) + int_to_hex(height,4)).decode('hex') + s += txi + int_to_hex(in_height,4).decode('hex') + txo + s = s[ -80*self.pruning_limit:] + self.db_hist.put(addr, s) @@ -455,11 +541,20 @@ class Storage(object): leaf = key + txi # restore backlink - self.db.put(txi, key) + self.db_addr.put(txi, addr) - v = undo.pop(leaf) - self.add_address(leaf, v) + v, height = undo.pop(leaf) + self.add_address(leaf, v, height) + # revert add to history + s = self.db_hist.get(addr) + # s might be empty if pruning limit was reached + if not s: + return + + assert s[-80:-44] == txi + s = s[:-80] + self.db_hist.put(addr, s) @@ -493,14 +588,14 @@ class Storage(object): def revert_transaction(self, txid, tx, block_height, touched_addr, undo): #print_log("revert tx", txid) - for x in tx.get('outputs'): + for x in reversed(tx.get('outputs')): addr = x.get('address') if addr is None: continue self.revert_add_to_history(addr, txid, x.get('index'), x.get('value'), block_height) touched_addr.add(addr) prev_addr = undo.pop('prev_addr') - for i, x in enumerate(tx.get('inputs')): + for i, x in reversed(list(enumerate(tx.get('inputs')))): addr = prev_addr[i] if addr is not None: txi = (x.get('prevout_hash') + int_to_hex(x.get('prevout_n'), 4)).decode('hex')