X-Git-Url: https://git.novaco.in/?p=electrum-server.git;a=blobdiff_plain;f=backends%2Fbitcoind%2Fstorage.py;h=cca9645eb141b9d161df4e84c5ec095876269bb3;hp=12c8609f77acf550fd8c75df64765ba8dd0303f5;hb=b9d74456c47ed3a54c73185d449ff7f23cb6edf4;hpb=477dadc5c491755794725de3abcf3b6ae4542c80 diff --git a/backends/bitcoind/storage.py b/backends/bitcoind/storage.py index 12c8609..cca9645 100644 --- a/backends/bitcoind/storage.py +++ b/backends/bitcoind/storage.py @@ -25,8 +25,9 @@ class Storage(object): self.test_reorgs = test_reorgs try: + self.db_utxo = plyvel.DB(os.path.join(self.dbpath,'utxo'), create_if_missing=True, compression=None) self.db_addr = plyvel.DB(os.path.join(self.dbpath,'addr'), create_if_missing=True, compression=None) - self.db = plyvel.DB(os.path.join(self.dbpath,'utxo'), create_if_missing=True, compression=None) + self.db_hist = plyvel.DB(os.path.join(self.dbpath,'hist'), create_if_missing=True, compression=None) self.db_undo = plyvel.DB(os.path.join(self.dbpath,'undo'), create_if_missing=True, compression=None) except: traceback.print_exc(file=sys.stdout) @@ -53,6 +54,11 @@ class Storage(object): return + # compute root hash + d = self.get_node('') + self.root_hash, v = self.get_node_hash('',d,None) + print_log("UTXO tree root hash:", self.root_hash.encode('hex')) + print_log("Coins in database:", v) # convert between bitcoin addresses and 20 bytes keys used for storage. def address_to_key(self, addr): @@ -62,22 +68,83 @@ class Storage(object): return hash_160_to_bc_address(addr) + def get_proof(self, addr): + key = self.address_to_key(addr) + i = self.db_utxo.iterator(start=key) + k, _ = i.next() + + p = self.get_path(k) + p.append(k) + + out = [] + for item in p: + v = self.db_utxo.get(item) + out.append((item.encode('hex'), v.encode('hex'))) + + return out + + + def get_balance(self, addr): + key = self.address_to_key(addr) + i = self.db_utxo.iterator(start=key) + k, _ = i.next() + if not k.startswith(key): + return 0 + p = self.get_parent(k) + d = self.get_node(p) + letter = k[len(p)] + return d[letter][1] + + + def listunspent(self, addr): + key = self.address_to_key(addr) + + out = [] + for k, v in self.db_utxo.iterator(start=key): + if not k.startswith(key): + break + if len(k) == KEYLENGTH: + txid = k[20:52].encode('hex') + txpos = hex_to_int(k[52:56]) + h = hex_to_int(v[8:12]) + v = hex_to_int(v[0:8]) + out.append({'tx_hash': txid, 'tx_pos':txpos, 'height': h, 'value':v}) + + out.sort(key=lambda x:x['height']) + return out + + def get_history(self, addr): - addr = self.address_to_key(addr) - x = self.db_addr.get(addr) - if x is None: - return '' - try: - _hash, v, h = x - return h - except: - traceback.print_exc(file=sys.stdout) - self.shared.stop() - raise + out = [] + + o = self.listunspent(addr) + for item in o: + out.append((item['tx_hash'], item['height'])) + + h = self.db_hist.get(addr) + + while h: + item = h[0:80] + h = h[80:] + txi = item[0:32].encode('hex') + hi = hex_to_int(item[36:40]) + txo = item[40:72].encode('hex') + ho = hex_to_int(item[76:80]) + out.append((txi, hi)) + out.append((txo, ho)) + + # sort + out.sort(key=lambda x:x[1]) + + # uniqueness + out = set(out) + + return map(lambda x: {'tx_hash':x[0], 'height':x[1]}, out) + def get_address(self, txi): - addr = self.db.get(txi) + addr = self.db_addr.get(txi) return self.key_to_address(addr) if addr else None @@ -123,12 +190,12 @@ class Storage(object): if batch: batch.put(key, out) else: - self.db_addr.put(key, out) + self.db_utxo.put(key, out) def get_node(self, key): - s = self.db_addr.get(key) + s = self.db_utxo.get(key) if s is None: return @@ -149,13 +216,13 @@ class Storage(object): return d - def add_address(self, target, value): + def add_address(self, target, value, height): assert len(target) == KEYLENGTH word = target key = '' path = [ '' ] - i = self.db_addr.iterator() + i = self.db_utxo.iterator() while key != target: @@ -202,8 +269,8 @@ class Storage(object): break # write - s = int_to_hex(value, 8).decode('hex') - self.db_addr.put(target, s) + s = (int_to_hex(value, 8) + int_to_hex(height,4)).decode('hex') + self.db_utxo.put(target, s) # the hash of a node is the txid _hash = target[20:52] self.update_node_hash(target, path, _hash, value) @@ -262,7 +329,7 @@ class Storage(object): # batch write modified nodes - batch = self.db_addr.write_batch() + batch = self.db_utxo.write_batch() for k, v in nodes.items(): self.put_node(k, v, batch) batch.write() @@ -292,7 +359,7 @@ class Storage(object): word = target key = '' path = [ '' ] - i = self.db_addr.iterator(start='') + i = self.db_utxo.iterator(start='') while key != target: @@ -315,7 +382,7 @@ class Storage(object): assert key not in path path.append(key) else: - print_log('not in tree', self.db_addr.get(key+word[0]), new_key.encode('hex')) + print_log('not in tree', self.db_utxo.get(key+word[0]), new_key.encode('hex')) return False else: assert key in path @@ -327,34 +394,36 @@ class Storage(object): def delete_address(self, leaf): path = self.get_path(leaf) if path is False: - print_log("addr not in tree", leaf.encode('hex'), self.key_to_address(leaf[0:20]), self.db_addr.get(leaf)) + print_log("addr not in tree", leaf.encode('hex'), self.key_to_address(leaf[0:20]), self.db_utxo.get(leaf)) raise - self.db_addr.delete(leaf) + s = self.db_utxo.get(leaf) + + self.db_utxo.delete(leaf) if leaf in self.hash_list: self.hash_list.pop(leaf) parent = path[-1] letter = leaf[len(parent)] items = self.get_node(parent) - vv = items.pop(letter) + items.pop(letter) # remove key if it has a single child if len(items) == 1: letter, v = items.items()[0] - self.db_addr.delete(parent) + self.db_utxo.delete(parent) if parent in self.hash_list: self.hash_list.pop(parent) # we need the exact length for the iteration - i = self.db_addr.iterator() + i = self.db_utxo.iterator() i.seek(parent+letter) k, v = i.next() # note: k is not necessarily a leaf if len(k) == KEYLENGTH: - _hash, value = k[20:52], hex_to_int(v) + _hash, value = k[20:52], hex_to_int(v[0:8]) else: _hash, value = None, None @@ -365,11 +434,11 @@ class Storage(object): _hash, value = None, None self.update_node_hash(parent, path[:-1], _hash, value) - return vv + return s def get_children(self, x): - i = self.db_addr.iterator() + i = self.db_utxo.iterator() l = 0 while l <256: i.seek(x+chr(l)) @@ -388,7 +457,7 @@ class Storage(object): def get_parent(self, x): """ return parent and skip string""" - i = self.db_addr.iterator() + i = self.db_utxo.iterator() for j in range(len(x)): p = x[0:-j-1] i.seek(p) @@ -409,8 +478,9 @@ class Storage(object): def close(self): + self.db_utxo.close() self.db_addr.close() - self.db.close() + self.db_hist.close() self.db_undo.close() @@ -419,10 +489,10 @@ class Storage(object): txo = (tx_hash + int_to_hex(tx_pos, 4)).decode('hex') # write the new history - self.add_address(key + txo, value) + self.add_address(key + txo, value, tx_height) # backlink - self.db.put(txo, key) + self.db_addr.put(txo, key) @@ -434,19 +504,36 @@ class Storage(object): self.delete_address(key + txo) # backlink - self.db.delete(txo) + self.db_addr.delete(txo) + def get_utxo_value(self, addr, txi): + key = self.address_to_key(addr) + leaf = key + txi + s = self.db_utxo.get(leaf) + value = hex_to_int(s[0:8]) + return value + def set_spent(self, addr, txi, txid, index, height, undo): key = self.address_to_key(addr) leaf = key + txi - _hash, value = self.delete_address(leaf) - undo[leaf] = value + s = self.delete_address(leaf) + value = hex_to_int(s[0:8]) + in_height = hex_to_int(s[8:12]) + undo[leaf] = value, in_height # delete backlink txi-> addr - self.db.delete(txi) + self.db_addr.delete(txi) + + # add to history + s = self.db_hist.get(addr) + if s is None: s = '' + txo = (txid + int_to_hex(index,4) + int_to_hex(height,4)).decode('hex') + s += txi + int_to_hex(in_height,4).decode('hex') + txo + s = s[ -80*self.pruning_limit:] + self.db_hist.put(addr, s) @@ -455,11 +542,20 @@ class Storage(object): leaf = key + txi # restore backlink - self.db.put(txi, key) + self.db_addr.put(txi, key) - v = undo.pop(leaf) - self.add_address(leaf, v) + v, height = undo.pop(leaf) + self.add_address(leaf, v, height) + # revert add to history + s = self.db_hist.get(addr) + # s might be empty if pruning limit was reached + if not s: + return + + assert s[-80:-44] == txi + s = s[:-80] + self.db_hist.put(addr, s) @@ -493,14 +589,14 @@ class Storage(object): def revert_transaction(self, txid, tx, block_height, touched_addr, undo): #print_log("revert tx", txid) - for x in tx.get('outputs'): + for x in reversed(tx.get('outputs')): addr = x.get('address') if addr is None: continue self.revert_add_to_history(addr, txid, x.get('index'), x.get('value'), block_height) touched_addr.add(addr) prev_addr = undo.pop('prev_addr') - for i, x in enumerate(tx.get('inputs')): + for i, x in reversed(list(enumerate(tx.get('inputs')))): addr = prev_addr[i] if addr is not None: txi = (x.get('prevout_hash') + int_to_hex(x.get('prevout_n'), 4)).decode('hex')