self.addresses = config.get('addresses', []) # receiving addresses visible for user
self.change_addresses = config.get('change_addresses', []) # addresses used as change
self.seed = config.get('seed', '') # encrypted
- self.history = config.get('history',{})
self.labels = config.get('labels',{}) # labels for addresses and transactions
self.aliases = config.get('aliases', {}) # aliases for addresses
self.authorities = config.get('authorities', {}) # trusted addresses
self.receipts = config.get('receipts',{}) # signed URIs
self.addressbook = config.get('contacts', []) # outgoing addresses, for payments
self.imported_keys = config.get('imported_keys',{})
+ self.history = config.get('addr_history',{}) # address -> list(txid, height)
+ self.transactions = config.get('transactions',{}) # txid -> deserialised
# not saved
+ self.prevout_values = {}
+ self.spent_outputs = []
self.receipt = None # next receipt
- self.tx_history = {}
- self.was_updated = True
- self.blocks = -1
self.banner = ''
+ # spv
+ self.verifier = None
+
# there is a difference between wallet.up_to_date and interface.is_up_to_date()
# interface.is_up_to_date() returns true when all requests have been answered and processed
# wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
self.lock = threading.Lock()
self.tx_event = threading.Event()
- self.update_tx_history()
if self.seed_version != SEED_VERSION:
raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
+ for tx_hash in self.transactions.keys():
+ self.update_tx_outputs(tx_hash)
+
+
def init_up_to_date(self):
self.up_to_date_event.clear()
self.up_to_date = False
+
def import_key(self, keypair, password):
address, key = keypair.split(':')
if not self.is_valid(address):
if value >= self.gap_limit:
self.gap_limit = value
self.save()
- self.interface.poke()
+ self.interface.poke('synchronizer')
return True
elif value >= self.min_acceptable_gap():
if len(self.addresses) < n:
new_addresses.append( self.create_new_address(False) )
continue
- if map( lambda a: self.history.get(a), self.addresses[-n:] ) == n*[[]]:
+ if map( lambda a: self.history.get(a, []), self.addresses[-n:] ) == n*[[]]:
break
else:
new_addresses.append( self.create_new_address(False) )
return (len(self.change_addresses) > 1 ) or ( len(self.addresses) > self.gap_limit )
def fill_addressbook(self):
- for tx in self.tx_history.values():
- if tx['value']<0:
- for i in tx['outputs']:
- if not self.is_mine(i) and i not in self.addressbook:
- self.addressbook.append(i)
+ for tx_hash, tx in self.transactions.items():
+ if self.get_tx_value(tx_hash)<0:
+ for o in tx['outputs']:
+ addr = o.get('address')
+ if not self.is_mine(addr) and addr not in self.addressbook:
+ self.addressbook.append(addr)
# redo labels
- self.update_tx_labels()
+ # self.update_tx_labels()
def get_address_flags(self, addr):
return flags
+ def get_tx_value(self, tx_hash, addresses = None):
+ # return the balance for that tx
+ if addresses is None: addresses = self.all_addresses()
+ with self.lock:
+ v = 0
+ d = self.transactions.get(tx_hash)
+ if not d: return 0
+ for item in d.get('inputs'):
+ addr = item.get('address')
+ if addr in addresses:
+ key = item['prevout_hash'] + ':%d'%item['prevout_n']
+ value = self.prevout_values.get( key )
+ if value is None: continue
+ v -= value
+ for item in d.get('outputs'):
+ addr = item.get('address')
+ if addr in addresses:
+ value = item.get('value')
+ v += value
+ return v
+
+
+
+ def update_tx_outputs(self, tx_hash):
+ tx = self.transactions.get(tx_hash)
+ for item in tx.get('outputs'):
+ value = item.get('value')
+ key = tx_hash+ ':%d'%item.get('index')
+ with self.lock:
+ self.prevout_values[key] = value
+
+ for item in tx.get('inputs'):
+ if self.is_mine(item.get('address')):
+ key = item['prevout_hash'] + ':%d'%item['prevout_n']
+ self.spent_outputs.append(key)
+
+
def get_addr_balance(self, addr):
assert self.is_mine(addr)
h = self.history.get(addr,[])
+ if h == ['*']: return 0,0
c = u = 0
- for item in h:
- v = item['value']
- if item['height']:
+ for tx_hash, tx_height in h:
+ v = self.get_tx_value(tx_hash, [addr])
+ if tx_height:
c += v
else:
u += v
if i in domain: domain.remove(i)
for addr in domain:
- h = self.history.get(addr)
- if h is None: continue
- for item in h:
- if item.get('raw_output_script'):
- coins.append( (addr,item))
-
- coins = sorted( coins, key = lambda x: x[1]['timestamp'] )
+ h = self.history.get(addr, [])
+ if h == ['*']: continue
+ for tx_hash, tx_height in h:
+ tx = self.transactions.get(tx_hash)
+ for output in tx.get('outputs'):
+ if output.get('address') != addr: continue
+ key = tx_hash + ":%d" % output.get('index')
+ if key in self.spent_outputs: continue
+ output['tx_hash'] = tx_hash
+ coins.append(output)
+
+ #coins = sorted( coins, key = lambda x: x[1]['timestamp'] )
for addr in self.prioritized_addresses:
- h = self.history.get(addr)
- if h is None: continue
- for item in h:
- if item.get('raw_output_script'):
- prioritized_coins.append( (addr,item))
-
- prioritized_coins = sorted( prioritized_coins, key = lambda x: x[1]['timestamp'] )
+ h = self.history.get(addr, [])
+ if h == ['*']: continue
+ for tx_hash, tx_height in h:
+ for output in tx.get('outputs'):
+ if output.get('address') != addr: continue
+ key = tx_hash + ":%d" % output.get('index')
+ if key in self.spent_outputs: continue
+ output['tx_hash'] = tx_hash
+ prioritized_coins.append(output)
+
+ #prioritized_coins = sorted( prioritized_coins, key = lambda x: x[1]['timestamp'] )
inputs = []
coins = prioritized_coins + coins
- for c in coins:
- addr, item = c
+ for item in coins:
+ addr = item.get('address')
v = item.get('value')
total += v
inputs.append((addr, v, item['tx_hash'], item['index'], item['raw_output_script'], None, None) )
else:
return s
- def get_status(self, address):
+
+ def get_history(self, address):
with self.lock:
- h = self.history.get(address)
- if not h:
- status = None
- else:
- lastpoint = h[-1]
- status = lastpoint['block_hash']
- if status == 'mempool':
- status = status + ':%d'% len(h)
- return status
+ return self.history.get(address)
+
+ def get_status(self, h):
+ if not h: return None
+ if h == ['*']: return '*'
+ status = ''
+ for tx_hash, height in h:
+ status += tx_hash + ':%d:' % height
+ return hashlib.sha256( status ).digest().encode('hex')
+
+
+
+ def receive_tx_callback(self, tx_hash, tx):
+
+ if not self.check_new_tx(tx_hash, tx):
+ raise BaseException("error: received transaction is not consistent with history"%tx_hash)
+
+ with self.lock:
+ self.transactions[tx_hash] = tx
+
+ tx_height = tx.get('height')
+ if tx_height>0: self.verifier.add(tx_hash, tx_height)
+
+ self.update_tx_outputs(tx_hash)
+
+ self.save()
- def receive_history_callback(self, addr, data):
- #print "updating history for", addr
+ def receive_history_callback(self, addr, hist):
+
+ if hist != ['*']:
+ if not self.check_new_history(addr, hist):
+ raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
+
with self.lock:
- self.history[addr] = data
- self.update_tx_history()
+ self.history[addr] = hist
self.save()
+ if hist != ['*']:
+ for tx_hash, tx_height in hist:
+ if tx_height>0:
+ # add it in case it was previously unconfirmed
+ self.verifier.add(tx_hash, tx_height)
+ # set the height in case it changed
+ tx = self.transactions.get(tx_hash)
+ if tx:
+ if tx.get('height') != tx_height:
+ print_error( "changing height for tx", tx_hash )
+ tx['height'] = tx_height
+
+
def get_tx_history(self):
with self.lock:
- lines = self.tx_history.values()
+ lines = self.transactions.values()
+
lines = sorted(lines, key=operator.itemgetter("timestamp"))
return lines
- def get_tx_hashes(self):
- with self.lock:
- hashes = self.tx_history.keys()
- return hashes
-
def get_transactions_at_height(self, height):
with self.lock:
- values = self.tx_history.values()[:]
+ values = self.transactions.values()[:]
out = []
for tx in values:
out.append(tx['tx_hash'])
return out
- def update_tx_history(self):
- self.tx_history= {}
- for addr in self.all_addresses():
- h = self.history.get(addr)
- if h is None: continue
- for tx in h:
- tx_hash = tx['tx_hash']
- line = self.tx_history.get(tx_hash)
- if not line:
- self.tx_history[tx_hash] = copy.copy(tx)
- line = self.tx_history.get(tx_hash)
- else:
- line['value'] += tx['value']
- if line['height'] == 0:
- line['timestamp'] = 1e12
- self.update_tx_labels()
- def update_tx_labels(self):
- for tx in self.tx_history.values():
+ def get_label(self, tx_hash):
+ label = self.labels.get(tx_hash)
+ is_default = (label == '') or (label is None)
+ if is_default: label = self.get_default_label(tx_hash)
+ return label, is_default
+
+ def get_default_label(self, tx_hash):
+ tx = self.transactions.get(tx_hash)
+ if tx:
default_label = ''
- if tx['value']<0:
- for o_addr in tx['outputs']:
+ if self.get_tx_value(tx_hash)<0:
+ for o in tx['outputs']:
+ o_addr = o.get('address')
if not self.is_mine(o_addr):
try:
default_label = self.labels[o_addr]
except KeyError:
default_label = o_addr
else:
- for o_addr in tx['outputs']:
+ for o in tx['outputs']:
+ o_addr = o.get('address')
if self.is_mine(o_addr) and not self.is_change(o_addr):
break
else:
- for o_addr in tx['outputs']:
+ for o in tx['outputs']:
+ o_addr = o.get('address')
if self.is_mine(o_addr):
break
else:
except KeyError:
default_label = o_addr
- tx['default_label'] = default_label
+ return default_label
+
def mktx(self, to_address, amount, label, password, fee=None, change_addr=None, from_addr= None):
if not self.is_valid(to_address):
# asynchronous
self.tx_event.clear()
tx_hash = Hash(tx.decode('hex') )[::-1].encode('hex')
- self.interface.send([('blockchain.transaction.broadcast', [tx])])
+ self.interface.send([('blockchain.transaction.broadcast', [tx])], 'synchronizer')
return tx_hash
def receive_tx(self,tx_hash):
'seed': self.seed,
'addresses': self.addresses,
'change_addresses': self.change_addresses,
- 'history': self.history,
+ 'addr_history': self.history,
'labels': self.labels,
'contacts': self.addressbook,
'imported_keys': self.imported_keys,
'frozen_addresses': self.frozen_addresses,
'prioritized_addresses': self.prioritized_addresses,
'gap_limit': self.gap_limit,
+ 'transactions': self.transactions,
}
for k, v in s.items():
self.config.set_key(k,v)
self.config.save()
+ def set_verifier(self, verifier):
+ self.verifier = verifier
+
+ # review transactions (they might not all be in history)
+ for tx_hash, tx in self.transactions.items():
+ tx_height = tx.get('height')
+ if tx_height <1:
+ print_error( "skipping", tx_hash, tx_height )
+ continue
+
+ if tx_height>0:
+ self.verifier.add(tx_hash, tx_height)
+
+ # set the timestamp for transactions that need it
+ if tx and not tx.get('timestamp'):
+ timestamp = self.verifier.get_timestamp(tx_height)
+ if timestamp:
+ self.set_tx_timestamp(tx_hash, timestamp)
+
+
+ # review existing history
+ for addr, hist in self.history.items():
+ if hist == ['*']: continue
+ for tx_hash, tx_height in hist:
+ if tx_height>0:
+ # add it in case it was previously unconfirmed
+ self.verifier.add(tx_hash, tx_height)
+ # set the height in case it changed
+ tx = self.transactions.get(tx_hash)
+ if tx:
+ if tx.get('height') != tx_height:
+ print_error( "changing height for tx", tx_hash )
+ tx['height'] = tx_height
+
+
+
+ def set_tx_timestamp(self, tx_hash, timestamp):
+ with self.lock:
+ self.transactions[tx_hash]['timestamp'] = timestamp
+
+
+
+ def is_addr_in_tx(self, addr, tx):
+ found = False
+ for txin in tx.get('inputs'):
+ if addr == txin.get('address'):
+ found = True
+ break
+ for txout in tx.get('outputs'):
+ if addr == txout.get('address'):
+ found = True
+ break
+ return found
+
+
+ def check_new_history(self, addr, hist):
+ # - check that all tx in hist are relevant
+ for tx_hash, height in hist:
+ tx = self.transactions.get(tx_hash)
+ if not tx: continue
+ if not self.is_addr_in_tx(addr,tx):
+ return False
+ # todo: check that we are not "orphaning" a transaction
+ # if we are, reject tx if unconfirmed, else reject the server
+
+ return True
+
+
+
+ def check_new_tx(self, tx_hash, tx):
+ # 1 check that tx is referenced in addr_history.
+ addresses = []
+ for addr, hist in self.history.items():
+ if hist == ['*']:continue
+ for txh, height in hist:
+ if txh == tx_hash:
+ addresses.append(addr)
+
+ if not addresses:
+ return False
+
+ # 2 check that referencing addresses are in the tx
+ for addr in addresses:
+ if not self.is_addr_in_tx(addr, tx):
+ return False
+
+ return True
self.interface.register_channel('synchronizer')
self.wallet.interface.register_callback('connected', self.wallet.init_up_to_date)
self.wallet.interface.register_callback('connected', lambda: self.interface.send([('server.banner',[])],'synchronizer') )
+ self.was_updated = True
def synchronize_wallet(self):
new_addresses = self.wallet.synchronize()
if new_addresses:
self.subscribe_to_addresses(new_addresses)
+ self.wallet.up_to_date = False
+ return
- if self.interface.is_up_to_date('synchronizer'):
- if not self.wallet.up_to_date:
- self.wallet.up_to_date = True
- self.wallet.was_updated = True
- self.wallet.up_to_date_event.set()
- else:
+ if not self.interface.is_up_to_date('synchronizer'):
if self.wallet.up_to_date:
self.wallet.up_to_date = False
- self.wallet.was_updated = True
-
+ self.was_updated = True
+ return
+ self.wallet.up_to_date = True
+ self.was_updated = True
+ self.wallet.up_to_date_event.set()
+
def subscribe_to_addresses(self, addresses):
messages = []
for addr in addresses:
def run(self):
+ requested_tx = []
+ missing_tx = []
+ requested_histories = {}
+
+ # request any missing transactions
+ for history in self.wallet.history.values():
+ if history == ['*']: continue
+ for tx_hash, tx_height in history:
+ if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
+ missing_tx.append( (tx_hash, tx_height) )
+ print_error("missing tx", missing_tx)
# wait until we are connected, in case the user is not connected
while not self.interface.is_connected:
self.interface.send([('server.banner',[])],'synchronizer')
# subscriptions
- self.interface.send([('blockchain.numblocks.subscribe',[])], 'synchronizer')
- self.interface.send([('server.peers.subscribe',[])],'synchronizer')
self.subscribe_to_addresses(self.wallet.all_addresses())
while True:
# 1. send new requests
self.synchronize_wallet()
- if self.wallet.was_updated:
+ for tx_hash, tx_height in missing_tx:
+ if (tx_hash, tx_height) not in requested_tx:
+ self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
+ requested_tx.append( (tx_hash, tx_height) )
+ missing_tx = []
+
+ if self.was_updated:
self.interface.trigger_callback('updated')
- self.wallet.was_updated = False
+ self.was_updated = False
# 2. get a response
r = self.interface.get_response('synchronizer')
- if not r: continue
+ if not r:
+ continue
# 3. handle response
method = r['method']
params = r['params']
- result = r['result']
+ result = r.get('result')
+ error = r.get('error')
+ if error:
+ print "error", r
+ continue
if method == 'blockchain.address.subscribe':
addr = params[0]
- if self.wallet.get_status(addr) != result:
- self.interface.send([('blockchain.address.get_history', [address] )])
-
+ if self.wallet.get_status(self.wallet.get_history(addr)) != result:
+ self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
+ requested_histories[addr] = result
+
elif method == 'blockchain.address.get_history':
addr = params[0]
- self.wallet.receive_history_callback(addr, result)
- self.wallet.was_updated = True
+ print_error("receiving history", addr, result)
+ if result == ['*']:
+ assert requested_histories.pop(addr) == '*'
+ self.wallet.receive_history_callback(addr, result)
+ else:
+ hist = []
+ # check that txids are unique
+ txids = []
+ for item in result:
+ tx_hash = item['tx_hash']
+ if tx_hash not in txids:
+ txids.append(tx_hash)
+ hist.append( (tx_hash, item['height']) )
+
+ if len(hist) != len(result):
+ raise BaseException("error: server sent history with non-unique txid", result)
+
+ # check that the status corresponds to what was announced
+ rs = requested_histories.pop(addr)
+ if self.wallet.get_status(hist) != rs:
+ raise BaseException("error: status mismatch: %s"%addr)
+
+ # store received history
+ self.wallet.receive_history_callback(addr, hist)
+
+ # request transactions that we don't have
+ for tx_hash, tx_height in hist:
+ if self.wallet.transactions.get(tx_hash) is None:
+ if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
+ missing_tx.append( (tx_hash, tx_height) )
+ else:
+ timestamp = self.wallet.verifier.get_timestamp(tx_height)
+ self.wallet.set_tx_timestamp(tx_hash, timestamp)
+
+ elif method == 'blockchain.transaction.get':
+ tx_hash = params[0]
+ tx_height = params[1]
+ d = self.deserialize_tx(tx_hash, tx_height, result)
+ self.wallet.receive_tx_callback(tx_hash, d)
+ self.was_updated = True
+ requested_tx.remove( (tx_hash, tx_height) )
+ print_error("received tx:", d)
elif method == 'blockchain.transaction.broadcast':
self.wallet.tx_result = result
self.wallet.tx_event.set()
- elif method == 'blockchain.numblocks.subscribe':
- self.wallet.blocks = result
- self.wallet.was_updated = True
-
- elif method == 'server.version':
- pass
-
- elif method == 'server.peers.subscribe':
- servers = []
- for item in result:
- s = []
- host = item[1]
- ports = []
- version = None
- if len(item) > 2:
- for v in item[2]:
- if re.match("[stgh]\d+", v):
- ports.append((v[0], v[1:]))
- if re.match("v(.?)+", v):
- version = v[1:]
- if ports and version:
- servers.append((host, ports))
- self.interface.servers = servers
- self.interface.trigger_callback('peers')
-
elif method == 'server.banner':
self.wallet.banner = result
- self.wallet.was_updated = True
+ self.was_updated = True
else:
print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
- if self.wallet.was_updated:
+ if self.was_updated and not requested_tx:
self.interface.trigger_callback('updated')
- self.wallet.was_updated = False
+ self.was_updated = False
-encode = lambda x: x[::-1].encode('hex')
-decode = lambda x: x.decode('hex')[::-1]
-from bitcoin import Hash, rev_hex, int_to_hex
+ def deserialize_tx(self, tx_hash, tx_height, raw_tx):
-class WalletVerifier(threading.Thread):
+ assert tx_hash == hash_encode(Hash(raw_tx.decode('hex')))
+ import deserialize
+ vds = deserialize.BCDataStream()
+ vds.write(raw_tx.decode('hex'))
+ d = deserialize.parse_Transaction(vds)
+ d['height'] = tx_height
+ d['tx_hash'] = tx_hash
+ d['timestamp'] = self.wallet.verifier.get_timestamp(tx_height)
+ return d
- def __init__(self, wallet, config):
- threading.Thread.__init__(self)
- self.daemon = True
- self.config = config
- self.wallet = wallet
- self.interface = self.wallet.interface
- self.interface.register_channel('verifier')
- self.validated = config.get('verified_tx',[])
- self.merkle_roots = config.get('merkle_roots',{})
- self.headers = config.get('block_headers',{})
- self.lock = threading.Lock()
- self.saved = True
-
- def run(self):
- requested = []
-
- while True:
- txlist = self.wallet.get_tx_hashes()
-
- for tx in txlist:
- if tx not in self.validated:
- if tx not in requested:
- requested.append(tx)
- self.request_merkle(tx)
- self.saved = False
- break
-
- try:
- r = self.interface.get_response('verifier',timeout=1)
- except Queue.Empty:
- if len(self.validated) == len(txlist) and not self.saved:
- print "saving verified transactions"
- self.config.set_key('verified_tx', self.validated, True)
- self.saved = True
- continue
-
- # 3. handle response
- method = r['method']
- params = r['params']
- result = r['result']
-
- if method == 'blockchain.transaction.get_merkle':
- tx_hash = params[0]
- tx_height = result.get('block_height')
- self.merkle_roots[tx_hash] = self.hash_merkle_root(result['merkle'], tx_hash)
- # if we already have the header, check merkle root directly
- header = self.headers.get(tx_height)
- if header:
- self.validated.append(tx_hash)
- assert header.get('merkle_root') == self.merkle_roots[tx_hash]
- self.request_headers(tx_height)
-
- elif method == 'blockchain.block.get_header':
- self.validate_header(result)
-
-
- def request_merkle(self, tx_hash):
- self.interface.send([ ('blockchain.transaction.get_merkle',[tx_hash]) ], 'verifier')
-
-
- def request_headers(self, tx_height, delta=10):
- headers_requests = []
- for height in range(tx_height-delta,tx_height+delta): # we might can request blocks that do not exist yet
- if height not in self.headers:
- headers_requests.append( ('blockchain.block.get_header',[height]) )
- self.interface.send(headers_requests,'verifier')
-
-
- def validate_header(self, header):
- """ if there is a previous or a next block in the list, check the hash"""
- height = header.get('block_height')
- with self.lock:
- self.headers[height] = header # detect conflicts
- prev_header = next_header = None
- if height-1 in self.headers:
- prev_header = self.headers[height-1]
- if height+1 in self.headers:
- next_header = self.headers[height+1]
-
- if prev_header:
- prev_hash = self.hash_header(prev_header)
- assert prev_hash == header.get('prev_block_hash')
- if next_header:
- _hash = self.hash_header(header)
- assert _hash == next_header.get('prev_block_hash')
-
- # check if there are transactions at that height
- for tx_hash in self.wallet.get_transactions_at_height(height):
- if tx_hash in self.validated: continue
- # check if we already have the merkle root
- merkle_root = self.merkle_roots.get(tx_hash)
- if merkle_root:
- self.validated.append(tx_hash)
- assert header.get('merkle_root') == merkle_root
-
- def hash_header(self, res):
- header = int_to_hex(res.get('version'),4) \
- + rev_hex(res.get('prev_block_hash')) \
- + rev_hex(res.get('merkle_root')) \
- + int_to_hex(int(res.get('timestamp')),4) \
- + int_to_hex(int(res.get('bits')),4) \
- + int_to_hex(int(res.get('nonce')),4)
- return rev_hex(Hash(header.decode('hex')).encode('hex'))
-
- def hash_merkle_root(self, merkle_s, target_hash):
- h = decode(target_hash)
- for item in merkle_s:
- is_left = item[0] == 'L'
- h = Hash( h + decode(item[1:]) ) if is_left else Hash( decode(item[1:]) + h )
- return encode(h)