lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import ecdsa
  31 import Queue
  32 import time
  33
  34 from ecdsa.util import string_to_number, number_to_string
  35 from util import print_error, user_dir, format_satoshis
  36 from bitcoin import *
  37
  38 # URL decode
  39 _ud = re.compile('%([0-9a-hA-H]{2})', re.MULTILINE)
  40 urldecode = lambda x: _ud.sub(lambda m: chr(int(m.group(1), 16)), x)
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46
  47 from version import ELECTRUM_VERSION, SEED_VERSION
  48
  49
  50 class Wallet:
  51     def __init__(self, config={}):
  52
  53         self.config = config
  54         self.electrum_version = ELECTRUM_VERSION
  55
  56         # saved fields
  57         self.seed_version          = config.get('seed_version', SEED_VERSION)
  58         self.gap_limit             = config.get('gap_limit', 5)
  59         self.use_change            = config.get('use_change',True)
  60         self.fee                   = int(config.get('fee',100000))
  61         self.num_zeros             = int(config.get('num_zeros',0))
  62         self.master_public_key     = config.get('master_public_key','')
  63         self.use_encryption        = config.get('use_encryption', False)
  64         self.addresses             = config.get('addresses', [])          # receiving addresses visible for user
  65         self.change_addresses      = config.get('change_addresses', [])   # addresses used as change
  66         self.seed                  = config.get('seed', '')               # encrypted
  67         self.labels                = config.get('labels',{})              # labels for addresses and transactions
  68         self.aliases               = config.get('aliases', {})            # aliases for addresses
  69         self.authorities           = config.get('authorities', {})        # trusted addresses
  70         self.frozen_addresses      = config.get('frozen_addresses',[])
  71         self.prioritized_addresses = config.get('prioritized_addresses',[])
  72         self.receipts              = config.get('receipts',{})            # signed URIs
  73         self.addressbook           = config.get('contacts', [])           # outgoing addresses, for payments
  74         self.imported_keys         = config.get('imported_keys',{})
  75         self.history               = config.get('addr_history',{})        # address -> list(txid, height)
  76         self.transactions          = config.get('transactions',{})        # txid -> deserialised
  77
  78         self.requested_amounts     = config.get('requested_amounts',{})   # txid -> deserialised
  79
  80         # not saved
  81         self.prevout_values = {}     # my own transaction outputs
  82         self.spent_outputs = []
  83         self.receipt = None          # next receipt
  84         self.banner = ''
  85
  86         # spv
  87         self.verifier = None
  88
  89         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
  90         # interface.is_up_to_date() returns true when all requests have been answered and processed
  91         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
  92
  93         self.up_to_date = False
  94         self.lock = threading.Lock()
  95         self.tx_event = threading.Event()
  96
  97         if self.seed_version != SEED_VERSION:
  98             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
  99
 100         for tx_hash in self.transactions.keys():
 101             self.update_tx_outputs(tx_hash)
 102
 103
 104     def set_up_to_date(self,b):
 105         with self.lock: self.up_to_date = b
 106
 107     def is_up_to_date(self):
 108         with self.lock: return self.up_to_date
 109
 110     def update(self):
 111         self.up_to_date = False
 112         self.interface.poke('synchronizer')
 113         while not self.is_up_to_date(): time.sleep(0.1)
 114
 115     def import_key(self, keypair, password):
 116         address, key = keypair.split(':')
 117         if not self.is_valid(address):
 118             raise BaseException('Invalid Bitcoin address')
 119         if address in self.all_addresses():
 120             raise BaseException('Address already in wallet')
 121         b = ASecretToSecret( key )
 122         if not b:
 123             raise BaseException('Unsupported key format')
 124         secexp = int( b.encode('hex'), 16)
 125         private_key = ecdsa.SigningKey.from_secret_exponent( secexp, curve=SECP256k1 )
 126         # sanity check
 127         public_key = private_key.get_verifying_key()
 128         if not address == public_key_to_bc_address( '04'.decode('hex') + public_key.to_string() ):
 129             raise BaseException('Address does not match private key')
 130         self.imported_keys[address] = self.pw_encode( key, password )
 131
 132
 133     def new_seed(self, password):
 134         seed = "%032x"%ecdsa.util.randrange( pow(2,128) )
 135         #self.init_mpk(seed)
 136         # encrypt
 137         self.seed = self.pw_encode( seed, password )
 138
 139
 140     def init_mpk(self,seed):
 141         # public key
 142         curve = SECP256k1
 143         secexp = self.stretch_key(seed)
 144         master_private_key = ecdsa.SigningKey.from_secret_exponent( secexp, curve = SECP256k1 )
 145         self.master_public_key = master_private_key.get_verifying_key().to_string().encode('hex')
 146
 147     def all_addresses(self):
 148         return self.addresses + self.change_addresses + self.imported_keys.keys()
 149
 150     def is_mine(self, address):
 151         return address in self.all_addresses()
 152
 153     def is_change(self, address):
 154         return address in self.change_addresses
 155
 156     def is_valid(self,addr):
 157         ADDRESS_RE = re.compile('[1-9A-HJ-NP-Za-km-z]{26,}\\Z')
 158         if not ADDRESS_RE.match(addr): return False
 159         try:
 160             h = bc_address_to_hash_160(addr)
 161         except:
 162             return False
 163         return addr == hash_160_to_bc_address(h)
 164
 165     def stretch_key(self,seed):
 166         oldseed = seed
 167         for i in range(100000):
 168             seed = hashlib.sha256(seed + oldseed).digest()
 169         return string_to_number( seed )
 170
 171     def get_sequence(self,n,for_change):
 172         return string_to_number( Hash( "%d:%d:"%(n,for_change) + self.master_public_key.decode('hex') ) )
 173
 174     def get_private_key_base58(self, address, password):
 175         pk = self.get_private_key(address, password)
 176         if pk is None: return None
 177         return SecretToASecret( pk )
 178
 179     def get_private_key(self, address, password):
 180         """  Privatekey(type,n) = Master_private_key + H(n|S|type)  """
 181         order = generator_secp256k1.order()
 182
 183         if address in self.imported_keys.keys():
 184             b = self.pw_decode( self.imported_keys[address], password )
 185             if not b: return None
 186             b = ASecretToSecret( b )
 187             secexp = int( b.encode('hex'), 16)
 188         else:
 189             if address in self.addresses:
 190                 n = self.addresses.index(address)
 191                 for_change = False
 192             elif address in self.change_addresses:
 193                 n = self.change_addresses.index(address)
 194                 for_change = True
 195             else:
 196                 raise BaseException("unknown address")
 197             try:
 198                 seed = self.pw_decode( self.seed, password)
 199             except:
 200                 raise BaseException("Invalid password")
 201             if not seed: return None
 202             secexp = self.stretch_key(seed)
 203             secexp = ( secexp + self.get_sequence(n,for_change) ) % order
 204
 205         pk = number_to_string(secexp,order)
 206         return pk
 207
 208     def msg_magic(self, message):
 209         return "\x18Bitcoin Signed Message:\n" + chr( len(message) ) + message
 210
 211     def sign_message(self, address, message, password):
 212         private_key = ecdsa.SigningKey.from_string( self.get_private_key(address, password), curve = SECP256k1 )
 213         public_key = private_key.get_verifying_key()
 214         signature = private_key.sign_digest( Hash( self.msg_magic( message ) ), sigencode = ecdsa.util.sigencode_string )
 215         assert public_key.verify_digest( signature, Hash( self.msg_magic( message ) ), sigdecode = ecdsa.util.sigdecode_string)
 216         for i in range(4):
 217             sig = base64.b64encode( chr(27+i) + signature )
 218             try:
 219                 self.verify_message( address, sig, message)
 220                 return sig
 221             except:
 222                 continue
 223         else:
 224             raise BaseException("error: cannot sign message")
 225
 226
 227     def verify_message(self, address, signature, message):
 228         """ See http://www.secg.org/download/aid-780/sec1-v2.pdf for the math """
 229         from ecdsa import numbertheory, ellipticcurve, util
 230         import msqr
 231         curve = curve_secp256k1
 232         G = generator_secp256k1
 233         order = G.order()
 234         # extract r,s from signature
 235         sig = base64.b64decode(signature)
 236         if len(sig) != 65: raise BaseException("Wrong encoding")
 237         r,s = util.sigdecode_string(sig[1:], order)
 238         nV = ord(sig[0])
 239         if nV < 27 or nV >= 35:
 240             raise BaseException("Bad encoding")
 241         if nV >= 31:
 242             compressed = True
 243             nV -= 4
 244         else:
 245             compressed = False
 246
 247         recid = nV - 27
 248         # 1.1
 249         x = r + (recid/2) * order
 250         # 1.3
 251         alpha = ( x * x * x  + curve.a() * x + curve.b() ) % curve.p()
 252         beta = msqr.modular_sqrt(alpha, curve.p())
 253         y = beta if (beta - recid) % 2 == 0 else curve.p() - beta
 254         # 1.4 the constructor checks that nR is at infinity
 255         R = ellipticcurve.Point(curve, x, y, order)
 256         # 1.5 compute e from message:
 257         h = Hash( self.msg_magic( message ) )
 258         e = string_to_number(h)
 259         minus_e = -e % order
 260         # 1.6 compute Q = r^-1 (sR - eG)
 261         inv_r = numbertheory.inverse_mod(r,order)
 262         Q = inv_r * ( s * R + minus_e * G )
 263         public_key = ecdsa.VerifyingKey.from_public_point( Q, curve = SECP256k1 )
 264         # check that Q is the public key
 265         public_key.verify_digest( sig[1:], h, sigdecode = ecdsa.util.sigdecode_string)
 266         # check that we get the original signing address
 267         addr = public_key_to_bc_address( encode_point(public_key, compressed) )
 268         if address != addr:
 269             raise BaseException("Bad signature")
 270
 271
 272     def create_new_address(self, for_change):
 273         n = len(self.change_addresses) if for_change else len(self.addresses)
 274         address = self.get_new_address(n, for_change)
 275         if for_change:
 276             self.change_addresses.append(address)
 277         else:
 278             self.addresses.append(address)
 279         self.history[address] = []
 280         return address
 281
 282     def get_new_address(self, n, for_change):
 283         """   Publickey(type,n) = Master_public_key + H(n|S|type)*point  """
 284         curve = SECP256k1
 285         z = self.get_sequence(n, for_change)
 286         master_public_key = ecdsa.VerifyingKey.from_string( self.master_public_key.decode('hex'), curve = SECP256k1 )
 287         pubkey_point = master_public_key.pubkey.point + z*curve.generator
 288         public_key2 = ecdsa.VerifyingKey.from_public_point( pubkey_point, curve = SECP256k1 )
 289         address = public_key_to_bc_address( '04'.decode('hex') + public_key2.to_string() )
 290         print address
 291         return address
 292
 293
 294     def change_gap_limit(self, value):
 295         if value >= self.gap_limit:
 296             self.gap_limit = value
 297             self.save()
 298             self.interface.poke('synchronizer')
 299             return True
 300
 301         elif value >= self.min_acceptable_gap():
 302             k = self.num_unused_trailing_addresses()
 303             n = len(self.addresses) - k + value
 304             self.addresses = self.addresses[0:n]
 305             self.gap_limit = value
 306             self.save()
 307             return True
 308         else:
 309             return False
 310
 311     def num_unused_trailing_addresses(self):
 312         k = 0
 313         for a in self.addresses[::-1]:
 314             if self.history.get(a):break
 315             k = k + 1
 316         return k
 317
 318     def min_acceptable_gap(self):
 319         # fixme: this assumes wallet is synchronized
 320         n = 0
 321         nmax = 0
 322         k = self.num_unused_trailing_addresses()
 323         for a in self.addresses[0:-k]:
 324             if self.history.get(a):
 325                 n = 0
 326             else:
 327                 n += 1
 328                 if n > nmax: nmax = n
 329         return nmax + 1
 330
 331
 332     def synchronize(self):
 333         if not self.master_public_key:
 334             return []
 335
 336         new_addresses = []
 337         while True:
 338             if self.change_addresses == []:
 339                 new_addresses.append( self.create_new_address(True) )
 340                 continue
 341             a = self.change_addresses[-1]
 342             if self.history.get(a):
 343                 new_addresses.append( self.create_new_address(True) )
 344             else:
 345                 break
 346
 347         n = self.gap_limit
 348         while True:
 349             if len(self.addresses) < n:
 350                 new_addresses.append( self.create_new_address(False) )
 351                 continue
 352             if map( lambda a: self.history.get(a, []), self.addresses[-n:] ) == n*[[]]:
 353                 break
 354             else:
 355                 new_addresses.append( self.create_new_address(False) )
 356
 357         return new_addresses
 358
 359
 360     def is_found(self):
 361         return (len(self.change_addresses) > 1 ) or ( len(self.addresses) > self.gap_limit )
 362
 363     def fill_addressbook(self):
 364         for tx_hash, tx in self.transactions.items():
 365             is_send, _, _ = self.get_tx_value(tx_hash)
 366             if is_send:
 367                 for o in tx['outputs']:
 368                     addr = o.get('address')
 369                     if not self.is_mine(addr) and addr not in self.addressbook:
 370                         self.addressbook.append(addr)
 371         # redo labels
 372         # self.update_tx_labels()
 373
 374
 375     def get_address_flags(self, addr):
 376         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 377         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 378         return flags
 379
 380
 381     def get_tx_value(self, tx_hash, addresses = None):
 382         # return the balance for that tx
 383         if addresses is None: addresses = self.all_addresses()
 384         with self.lock:
 385             is_send = False
 386             is_pruned = False
 387             v_in = v_out = v_out_mine = 0
 388             d = self.transactions.get(tx_hash)
 389             if not d:
 390                 return 0, 0, 0
 391
 392             for item in d.get('inputs'):
 393                 addr = item.get('address')
 394                 if addr in addresses:
 395                     is_send = True
 396                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 397                     value = self.prevout_values.get( key )
 398                     if value is None:
 399                         is_pruned = True
 400                     else:
 401                         v_in += value
 402                 else:
 403                     is_pruned = True
 404
 405             for item in d.get('outputs'):
 406                 addr = item.get('address')
 407                 value = item.get('value')
 408                 v_out += value
 409                 if addr in addresses:
 410                     v_out_mine += value
 411
 412         if not is_pruned:
 413             # all inputs are mine:
 414             fee = v_out - v_in
 415             v = v_out_mine - v_in
 416         else:
 417             # some inputs are mine:
 418             fee = None
 419             if is_send:
 420                 v = v_out_mine - v_out
 421             else:
 422                 # no input is mine
 423                 v = v_out_mine
 424
 425         return is_send, v, fee
 426
 427
 428     def get_tx_details(self, tx_hash):
 429         import datetime
 430         if not tx_hash: return ''
 431         tx = self.transactions.get(tx_hash)
 432         is_mine, v, fee = self.get_tx_value(tx_hash)
 433         conf, timestamp = self.verifier.get_confirmations(tx_hash)
 434
 435         if conf:
 436             time_str = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3]
 437         else:
 438             time_str = 'pending'
 439
 440         inputs = map(lambda x: x.get('address'), tx['inputs'])
 441         outputs = map(lambda x: x.get('address'), tx['outputs'])
 442         tx_details = "Transaction Details" +"\n\n" \
 443             + "Transaction ID:\n" + tx_hash + "\n\n" \
 444             + "Status: %d confirmations\n"%conf
 445         if is_mine:
 446             if fee:
 447                 tx_details += "Amount sent: %s\n"% format_satoshis(v-fee, False) \
 448                               + "Transaction fee: %s\n"% format_satoshis(fee, False)
 449             else:
 450                 tx_details += "Amount sent: %s\n"% format_satoshis(v, False) \
 451                               + "Transaction fee: unknown\n"
 452         else:
 453             tx_details += "Amount received: %s\n"% format_satoshis(v, False) \
 454
 455         tx_details += "Date: %s\n\n"%time_str \
 456             + "Inputs:\n-"+ '\n-'.join(inputs) + "\n\n" \
 457             + "Outputs:\n-"+ '\n-'.join(outputs)
 458
 459         r = self.receipts.get(tx_hash)
 460         if r:
 461             tx_details += "\n_______________________________________" \
 462                 + '\n\nSigned URI: ' + r[2] \
 463                 + "\n\nSigned by: " + r[0] \
 464                 + '\n\nSignature: ' + r[1]
 465
 466         return tx_details
 467
 468
 469     def update_tx_outputs(self, tx_hash):
 470         tx = self.transactions.get(tx_hash)
 471         for item in tx.get('outputs'):
 472             value = item.get('value')
 473             key = tx_hash+ ':%d'%item.get('index')
 474             with self.lock:
 475                 self.prevout_values[key] = value
 476
 477         for item in tx.get('inputs'):
 478             if self.is_mine(item.get('address')):
 479                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 480                 self.spent_outputs.append(key)
 481
 482
 483     def get_addr_balance(self, address):
 484         assert self.is_mine(address)
 485         h = self.history.get(address,[])
 486         if h == ['*']: return 0,0
 487         c = u = 0
 488         received_coins = []   # list of coins received at address
 489
 490         for tx_hash, tx_height in h:
 491             d = self.transactions.get(tx_hash)
 492             if not d: continue
 493             for item in d.get('outputs'):
 494                 addr = item.get('address')
 495                 if addr == address:
 496                     key = tx_hash + ':%d'%item['index']
 497                     received_coins.append(key)
 498
 499         for tx_hash, tx_height in h:
 500             d = self.transactions.get(tx_hash)
 501             if not d: continue
 502             v = 0
 503
 504             for item in d.get('inputs'):
 505                 addr = item.get('address')
 506                 if addr == address:
 507                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 508                     value = self.prevout_values.get( key )
 509                     if key in received_coins:
 510                         v -= value
 511
 512             for item in d.get('outputs'):
 513                 addr = item.get('address')
 514                 key = tx_hash + ':%d'%item['index']
 515                 if addr == address:
 516                     v += item.get('value')
 517
 518             if tx_height:
 519                 c += v
 520             else:
 521                 u += v
 522         return c, u
 523
 524     def get_balance(self):
 525         conf = unconf = 0
 526         for addr in self.all_addresses():
 527             c, u = self.get_addr_balance(addr)
 528             conf += c
 529             unconf += u
 530         return conf, unconf
 531
 532
 533     def choose_tx_inputs( self, amount, fixed_fee, from_addr = None ):
 534         """ todo: minimize tx size """
 535         total = 0
 536         fee = self.fee if fixed_fee is None else fixed_fee
 537
 538         coins = []
 539         prioritized_coins = []
 540         domain = [from_addr] if from_addr else self.all_addresses()
 541         for i in self.frozen_addresses:
 542             if i in domain: domain.remove(i)
 543
 544         for i in self.prioritized_addresses:
 545             if i in domain: domain.remove(i)
 546
 547         for addr in domain:
 548             h = self.history.get(addr, [])
 549             if h == ['*']: continue
 550             for tx_hash, tx_height in h:
 551                 tx = self.transactions.get(tx_hash)
 552                 for output in tx.get('outputs'):
 553                     if output.get('address') != addr: continue
 554                     key = tx_hash + ":%d" % output.get('index')
 555                     if key in self.spent_outputs: continue
 556                     output['tx_hash'] = tx_hash
 557                     coins.append(output)
 558
 559
 560         for addr in self.prioritized_addresses:
 561             h = self.history.get(addr, [])
 562             if h == ['*']: continue
 563             for tx_hash, tx_height in h:
 564                 tx = self.transactions.get(tx_hash)
 565                 for output in tx.get('outputs'):
 566                     if output.get('address') != addr: continue
 567                     key = tx_hash + ":%d" % output.get('index')
 568                     if key in self.spent_outputs: continue
 569                     output['tx_hash'] = tx_hash
 570                     prioritized_coins.append(output)
 571
 572
 573         inputs = []
 574         coins = prioritized_coins + coins
 575
 576         for item in coins:
 577             addr = item.get('address')
 578             v = item.get('value')
 579             total += v
 580             inputs.append((addr, v, item['tx_hash'], item['index'], item['raw_output_script'], None, None) )
 581             fee = self.fee*len(inputs) if fixed_fee is None else fixed_fee
 582             if total >= amount + fee: break
 583         else:
 584             #print "not enough funds: %s %s"%(format_satoshis(total), format_satoshis(fee))
 585             inputs = []
 586         return inputs, total, fee
 587
 588     def add_tx_change( self, outputs, amount, fee, total, change_addr=None ):
 589         change_amount = total - ( amount + fee )
 590         if change_amount != 0:
 591             # normally, the update thread should ensure that the last change address is unused
 592             if not change_addr:
 593                 change_addr = self.change_addresses[-1]
 594             outputs.append( ( change_addr,  change_amount) )
 595         return outputs
 596
 597     def sign_inputs( self, inputs, outputs, password ):
 598         s_inputs = []
 599         for i in range(len(inputs)):
 600             addr, v, p_hash, p_pos, p_scriptPubKey, _, _ = inputs[i]
 601             private_key = ecdsa.SigningKey.from_string( self.get_private_key(addr, password), curve = SECP256k1 )
 602             public_key = private_key.get_verifying_key()
 603             pubkey = public_key.to_string()
 604             tx = filter( raw_tx( inputs, outputs, for_sig = i ) )
 605             sig = private_key.sign_digest( Hash( tx.decode('hex') ), sigencode = ecdsa.util.sigencode_der )
 606             assert public_key.verify_digest( sig, Hash( tx.decode('hex') ), sigdecode = ecdsa.util.sigdecode_der)
 607             s_inputs.append( (addr, v, p_hash, p_pos, p_scriptPubKey, pubkey, sig) )
 608         return s_inputs
 609
 610     def pw_encode(self, s, password):
 611         if password:
 612             secret = Hash(password)
 613             return EncodeAES(secret, s)
 614         else:
 615             return s
 616
 617     def pw_decode(self, s, password):
 618         if password is not None:
 619             secret = Hash(password)
 620             d = DecodeAES(secret, s)
 621             if s == self.seed:
 622                 try:
 623                     d.decode('hex')
 624                 except:
 625                     raise ValueError("Invalid password")
 626             return d
 627         else:
 628             return s
 629
 630
 631     def get_history(self, address):
 632         with self.lock:
 633             return self.history.get(address)
 634
 635     def get_status(self, h):
 636         if not h: return None
 637         if h == ['*']: return '*'
 638         status = ''
 639         for tx_hash, height in h:
 640             status += tx_hash + ':%d:' % height
 641         return hashlib.sha256( status ).digest().encode('hex')
 642
 643
 644
 645     def receive_tx_callback(self, tx_hash, tx):
 646
 647         if not self.check_new_tx(tx_hash, tx):
 648             raise BaseException("error: received transaction is not consistent with history"%tx_hash)
 649
 650         with self.lock:
 651             self.transactions[tx_hash] = tx
 652
 653         tx_height = tx.get('height')
 654         if self.verifier and tx_height>0:
 655             self.verifier.add(tx_hash, tx_height)
 656
 657         self.update_tx_outputs(tx_hash)
 658
 659         self.save()
 660
 661
 662     def receive_history_callback(self, addr, hist):
 663
 664         if not self.check_new_history(addr, hist):
 665             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
 666
 667         with self.lock:
 668             self.history[addr] = hist
 669             self.save()
 670
 671         if hist != ['*']:
 672             for tx_hash, tx_height in hist:
 673                 if tx_height>0:
 674                     # add it in case it was previously unconfirmed
 675                     if self.verifier: self.verifier.add(tx_hash, tx_height)
 676                     # set the height in case it changed
 677                     tx = self.transactions.get(tx_hash)
 678                     if tx:
 679                         if tx.get('height') != tx_height:
 680                             print_error( "changing height for tx", tx_hash )
 681                             tx['height'] = tx_height
 682
 683
 684     def get_tx_history(self):
 685         with self.lock:
 686             history = self.transactions.values()
 687         history.sort(key = lambda x: x.get('height') if x.get('height') else 1e12)
 688         result = []
 689
 690         balance = 0
 691         for tx in history:
 692             is_mine, v, fee = self.get_tx_value(tx['tx_hash'])
 693             if v is not None: balance += v
 694         c, u = self.get_balance()
 695
 696         if balance != c+u:
 697             v_str = format_satoshis( c+u - balance, True, self.num_zeros)
 698             result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
 699
 700         balance = c + u - balance
 701         for tx in history:
 702             tx_hash = tx['tx_hash']
 703             conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else None
 704             is_mine, value, fee = self.get_tx_value(tx_hash)
 705             if value is not None:
 706                 balance += value
 707
 708             result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
 709
 710         return result
 711
 712     def get_transactions_at_height(self, height):
 713         with self.lock:
 714             values = self.transactions.values()[:]
 715
 716         out = []
 717         for tx in values:
 718             if tx['height'] == height:
 719                 out.append(tx['tx_hash'])
 720         return out
 721
 722
 723     def get_label(self, tx_hash):
 724         label = self.labels.get(tx_hash)
 725         is_default = (label == '') or (label is None)
 726         if is_default: label = self.get_default_label(tx_hash)
 727         return label, is_default
 728
 729     def get_default_label(self, tx_hash):
 730         tx = self.transactions.get(tx_hash)
 731         default_label = ''
 732         if tx:
 733             is_mine, _, _ = self.get_tx_value(tx_hash)
 734             if is_mine:
 735                 for o in tx['outputs']:
 736                     o_addr = o.get('address')
 737                     if not self.is_mine(o_addr):
 738                         try:
 739                             default_label = self.labels[o_addr]
 740                         except KeyError:
 741                             default_label = o_addr
 742             else:
 743                 for o in tx['outputs']:
 744                     o_addr = o.get('address')
 745                     if self.is_mine(o_addr) and not self.is_change(o_addr):
 746                         break
 747                 else:
 748                     for o in tx['outputs']:
 749                         o_addr = o.get('address')
 750                         if self.is_mine(o_addr):
 751                             break
 752                     else:
 753                         o_addr = None
 754
 755                 if o_addr:
 756                     dest_label = self.labels.get(o_addr)
 757                     try:
 758                         default_label = self.labels[o_addr]
 759                     except KeyError:
 760                         default_label = o_addr
 761
 762         return default_label
 763
 764
 765     def mktx(self, outputs, label, password, fee=None, change_addr=None, from_addr= None):
 766
 767         for address, x in outputs:
 768             assert self.is_valid(address)
 769
 770         amount = sum( map(lambda x:x[1], outputs) )
 771         inputs, total, fee = self.choose_tx_inputs( amount, fee, from_addr )
 772         if not inputs:
 773             raise ValueError("Not enough funds")
 774
 775         if not self.use_change and not change_addr:
 776             change_addr = inputs[-1][0]
 777             print_error( "Sending change to", change_addr )
 778         outputs = self.add_tx_change(outputs, amount, fee, total, change_addr)
 779
 780         if not self.seed:
 781             return repr({'inputs':inputs, 'outputs':outputs})
 782
 783         tx = self.signed_tx(inputs, outputs, password)
 784
 785         for address, x in outputs:
 786             if address not in self.addressbook and not self.is_mine(address):
 787                 self.addressbook.append(address)
 788
 789         if label:
 790             tx_hash = Hash(tx.decode('hex') )[::-1].encode('hex')
 791             self.labels[tx_hash] = label
 792
 793         return tx
 794
 795     def signed_tx(self, inputs, outputs, password):
 796         s_inputs = self.sign_inputs( inputs, outputs, password )
 797         tx = filter( raw_tx( s_inputs, outputs ) )
 798         return tx
 799
 800     def sendtx(self, tx):
 801         # synchronous
 802         h = self.send_tx(tx)
 803         self.tx_event.wait()
 804         return self.receive_tx(h)
 805
 806     def send_tx(self, tx):
 807         # asynchronous
 808         self.tx_event.clear()
 809         tx_hash = Hash(tx.decode('hex') )[::-1].encode('hex')
 810         self.interface.send([('blockchain.transaction.broadcast', [tx])], 'synchronizer')
 811         return tx_hash
 812
 813     def receive_tx(self,tx_hash):
 814         out = self.tx_result
 815         if out != tx_hash:
 816             return False, "error: " + out
 817         if self.receipt:
 818             self.receipts[tx_hash] = self.receipt
 819             self.receipt = None
 820         return True, out
 821
 822
 823     def read_alias(self, alias):
 824         # this might not be the right place for this function.
 825         import urllib
 826
 827         m1 = re.match('([\w\-\.]+)@((\w[\w\-]+\.)+[\w\-]+)', alias)
 828         m2 = re.match('((\w[\w\-]+\.)+[\w\-]+)', alias)
 829         if m1:
 830             url = 'https://' + m1.group(2) + '/bitcoin.id/' + m1.group(1)
 831         elif m2:
 832             url = 'https://' + alias + '/bitcoin.id'
 833         else:
 834             return ''
 835         try:
 836             lines = urllib.urlopen(url).readlines()
 837         except:
 838             return ''
 839
 840         # line 0
 841         line = lines[0].strip().split(':')
 842         if len(line) == 1:
 843             auth_name = None
 844             target = signing_addr = line[0]
 845         else:
 846             target, auth_name, signing_addr, signature = line
 847             msg = "alias:%s:%s:%s"%(alias,target,auth_name)
 848             print msg, signature
 849             self.verify_message(signing_addr, signature, msg)
 850
 851         # other lines are signed updates
 852         for line in lines[1:]:
 853             line = line.strip()
 854             if not line: continue
 855             line = line.split(':')
 856             previous = target
 857             print repr(line)
 858             target, signature = line
 859             self.verify_message(previous, signature, "alias:%s:%s"%(alias,target))
 860
 861         if not self.is_valid(target):
 862             raise ValueError("Invalid bitcoin address")
 863
 864         return target, signing_addr, auth_name
 865
 866     def update_password(self, seed, old_password, new_password):
 867         if new_password == '': new_password = None
 868         self.use_encryption = (new_password != None)
 869         self.seed = self.pw_encode( seed, new_password)
 870         for k in self.imported_keys.keys():
 871             a = self.imported_keys[k]
 872             b = self.pw_decode(a, old_password)
 873             c = self.pw_encode(b, new_password)
 874             self.imported_keys[k] = c
 875         self.save()
 876
 877     def get_alias(self, alias, interactive = False, show_message=None, question = None):
 878         try:
 879             target, signing_address, auth_name = self.read_alias(alias)
 880         except BaseException, e:
 881             # raise exception if verify fails (verify the chain)
 882             if interactive:
 883                 show_message("Alias error: " + str(e))
 884             return
 885
 886         print target, signing_address, auth_name
 887
 888         if auth_name is None:
 889             a = self.aliases.get(alias)
 890             if not a:
 891                 msg = "Warning: the alias '%s' is self-signed.\nThe signing address is %s.\n\nDo you want to add this alias to your list of contacts?"%(alias,signing_address)
 892                 if interactive and question( msg ):
 893                     self.aliases[alias] = (signing_address, target)
 894                 else:
 895                     target = None
 896             else:
 897                 if signing_address != a[0]:
 898                     msg = "Warning: the key of alias '%s' has changed since your last visit! It is possible that someone is trying to do something nasty!!!\nDo you accept to change your trusted key?"%alias
 899                     if interactive and question( msg ):
 900                         self.aliases[alias] = (signing_address, target)
 901                     else:
 902                         target = None
 903         else:
 904             if signing_address not in self.authorities.keys():
 905                 msg = "The alias: '%s' links to %s\n\nWarning: this alias was signed by an unknown key.\nSigning authority: %s\nSigning address: %s\n\nDo you want to add this key to your list of trusted keys?"%(alias,target,auth_name,signing_address)
 906                 if interactive and question( msg ):
 907                     self.authorities[signing_address] = auth_name
 908                 else:
 909                     target = None
 910
 911         if target:
 912             self.aliases[alias] = (signing_address, target)
 913
 914         return target
 915
 916
 917     def parse_url(self, url, show_message, question):
 918         o = url[8:].split('?')
 919         address = o[0]
 920         if len(o)>1:
 921             params = o[1].split('&')
 922         else:
 923             params = []
 924
 925         amount = label = message = signature = identity = ''
 926         for p in params:
 927             k,v = p.split('=')
 928             uv = urldecode(v)
 929             if k == 'amount': amount = uv
 930             elif k == 'message': message = uv
 931             elif k == 'label': label = uv
 932             elif k == 'signature':
 933                 identity, signature = uv.split(':')
 934                 url = url.replace('&%s=%s'%(k,v),'')
 935             else:
 936                 print k,v
 937
 938         if label and self.labels.get(address) != label:
 939             if question('Give label "%s" to address %s ?'%(label,address)):
 940                 if address not in self.addressbook and address not in self.all_addresses():
 941                     self.addressbook.append(address)
 942                 self.labels[address] = label
 943
 944         if signature:
 945             if re.match('^(|([\w\-\.]+)@)((\w[\w\-]+\.)+[\w\-]+)$', identity):
 946                 signing_address = self.get_alias(identity, True, show_message, question)
 947             elif self.is_valid(identity):
 948                 signing_address = identity
 949             else:
 950                 signing_address = None
 951             if not signing_address:
 952                 return
 953             try:
 954                 self.verify_message(signing_address, signature, url )
 955                 self.receipt = (signing_address, signature, url)
 956             except:
 957                 show_message('Warning: the URI contains a bad signature.\nThe identity of the recipient cannot be verified.')
 958                 address = amount = label = identity = message = ''
 959
 960         if re.match('^(|([\w\-\.]+)@)((\w[\w\-]+\.)+[\w\-]+)$', address):
 961             payto_address = self.get_alias(address, True, show_message, question)
 962             if payto_address:
 963                 address = address + ' <' + payto_address + '>'
 964
 965         return address, amount, label, message, signature, identity, url
 966
 967
 968
 969     def freeze(self,addr):
 970         if addr in self.all_addresses() and addr not in self.frozen_addresses:
 971             self.unprioritize(addr)
 972             self.frozen_addresses.append(addr)
 973             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 974             return True
 975         else:
 976             return False
 977
 978     def unfreeze(self,addr):
 979         if addr in self.all_addresses() and addr in self.frozen_addresses:
 980             self.frozen_addresses.remove(addr)
 981             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 982             return True
 983         else:
 984             return False
 985
 986     def prioritize(self,addr):
 987         if addr in self.all_addresses() and addr not in self.prioritized_addresses:
 988             self.unfreeze(addr)
 989             self.prioritized_addresses.append(addr)
 990             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
 991             return True
 992         else:
 993             return False
 994
 995     def unprioritize(self,addr):
 996         if addr in self.all_addresses() and addr in self.prioritized_addresses:
 997             self.prioritized_addresses.remove(addr)
 998             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
 999             return True
1000         else:
1001             return False
1002
1003     def save(self):
1004         s = {
1005             'seed_version': self.seed_version,
1006             'use_encryption': self.use_encryption,
1007             'use_change': self.use_change,
1008             'master_public_key': self.master_public_key,
1009             'fee': self.fee,
1010             'seed': self.seed,
1011             'addresses': self.addresses,
1012             'change_addresses': self.change_addresses,
1013             'addr_history': self.history,
1014             'labels': self.labels,
1015             'contacts': self.addressbook,
1016             'imported_keys': self.imported_keys,
1017             'aliases': self.aliases,
1018             'authorities': self.authorities,
1019             'receipts': self.receipts,
1020             'num_zeros': self.num_zeros,
1021             'frozen_addresses': self.frozen_addresses,
1022             'prioritized_addresses': self.prioritized_addresses,
1023             'gap_limit': self.gap_limit,
1024             'transactions': self.transactions,
1025             'requested_amounts': self.requested_amounts,
1026         }
1027         for k, v in s.items():
1028             self.config.set_key(k,v)
1029         self.config.save()
1030
1031     def set_verifier(self, verifier):
1032         self.verifier = verifier
1033
1034         # review stored transactions and send them to the verifier
1035         # (they are not necessarily in the history, because history items might have have been pruned)
1036         for tx_hash, tx in self.transactions.items():
1037             tx_height = tx.get('height')
1038             if tx_height <1:
1039                 print_error( "skipping", tx_hash, tx_height )
1040                 continue
1041
1042             if tx_height>0:
1043                 self.verifier.add(tx_hash, tx_height)
1044
1045         # review transactions that are in the history
1046         for addr, hist in self.history.items():
1047             if hist == ['*']: continue
1048             for tx_hash, tx_height in hist:
1049                 if tx_height>0:
1050                     # add it in case it was previously unconfirmed
1051                     self.verifier.add(tx_hash, tx_height)
1052                     # set the height in case it changed
1053                     tx = self.transactions.get(tx_hash)
1054                     if tx:
1055                         if tx.get('height') != tx_height:
1056                             print_error( "changing height for tx", tx_hash )
1057                             tx['height'] = tx_height
1058
1059
1060     def is_addr_in_tx(self, addr, tx):
1061         found = False
1062         for txin in tx.get('inputs'):
1063             if addr == txin.get('address'):
1064                 found = True
1065                 break
1066         for txout in tx.get('outputs'):
1067             if addr == txout.get('address'):
1068                 found = True
1069                 break
1070         return found
1071
1072
1073     def check_new_history(self, addr, hist):
1074
1075         # check that all tx in hist are relevant
1076         if hist != ['*']:
1077             for tx_hash, height in hist:
1078                 tx = self.transactions.get(tx_hash)
1079                 if not tx: continue
1080                 if not self.is_addr_in_tx(addr,tx):
1081                     return False
1082
1083         # check that we are not "orphaning" a transaction
1084         old_hist = self.history.get(addr,[])
1085         if old_hist == ['*']: return True
1086
1087         for tx_hash, height in old_hist:
1088             if tx_hash in map(lambda x:x[0], hist): continue
1089             found = False
1090             for _addr, _hist in self.history.items():
1091                 if _addr == addr: continue
1092                 if _hist == ['*']: continue
1093                 _tx_hist = map(lambda x:x[0], _hist)
1094                 if tx_hash in _tx_hist:
1095                     found = True
1096                     break
1097
1098             if not found:
1099                 tx = self.transactions.get(tx_hash)
1100                 # already verified?
1101                 if tx.get('height'):
1102                     continue
1103                 # unconfirmed tx
1104                 print_error("new history is orphaning transaction:", tx_hash)
1105                 # check that all outputs are not mine, request histories
1106                 ext_requests = []
1107                 for o in tx.get('outputs'):
1108                     _addr = o.get('address')
1109                     # assert not self.is_mine(_addr)
1110                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1111
1112                 ext_h = self.interface.synchronous_get(ext_requests)
1113                 height = None
1114                 for h in ext_h:
1115                     if h == ['*']: continue
1116                     for item in h:
1117                         if item.get('tx_hash') == tx_hash:
1118                             height = item.get('height')
1119                             tx['height'] = height
1120                 if height:
1121                     print_error("found height for", tx_hash, height)
1122                     self.verifier.add(tx_hash, height)
1123                 else:
1124                     print_error("removing orphaned tx from history", tx_hash)
1125                     self.transactions.pop(tx_hash)
1126
1127         return True
1128
1129
1130
1131     def check_new_tx(self, tx_hash, tx):
1132         # 1 check that tx is referenced in addr_history.
1133         addresses = []
1134         for addr, hist in self.history.items():
1135             if hist == ['*']:continue
1136             for txh, height in hist:
1137                 if txh == tx_hash:
1138                     addresses.append(addr)
1139
1140         if not addresses:
1141             return False
1142
1143         # 2 check that referencing addresses are in the tx
1144         for addr in addresses:
1145             if not self.is_addr_in_tx(addr, tx):
1146                 return False
1147
1148         return True
1149
1150
1151
1152
1153 class WalletSynchronizer(threading.Thread):
1154
1155
1156     def __init__(self, wallet, config):
1157         threading.Thread.__init__(self)
1158         self.daemon = True
1159         self.wallet = wallet
1160         self.interface = self.wallet.interface
1161         self.interface.register_channel('synchronizer')
1162         self.wallet.interface.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1163         self.wallet.interface.register_callback('connected', lambda: self.interface.send([('server.banner',[])],'synchronizer') )
1164         self.was_updated = True
1165         self.running = False
1166         self.lock = threading.Lock()
1167
1168     def stop(self):
1169         with self.lock: self.running = False
1170         self.interface.poke('synchronizer')
1171
1172     def is_running(self):
1173         with self.lock: return self.running
1174
1175     def synchronize_wallet(self):
1176         new_addresses = self.wallet.synchronize()
1177         if new_addresses:
1178             self.subscribe_to_addresses(new_addresses)
1179             self.wallet.up_to_date = False
1180             return
1181
1182         if not self.interface.is_up_to_date('synchronizer'):
1183             if self.wallet.is_up_to_date():
1184                 self.wallet.set_up_to_date(False)
1185                 self.was_updated = True
1186             return
1187
1188         self.wallet.set_up_to_date(True)
1189         self.was_updated = True
1190
1191
1192     def subscribe_to_addresses(self, addresses):
1193         messages = []
1194         for addr in addresses:
1195             messages.append(('blockchain.address.subscribe', [addr]))
1196         self.interface.send( messages, 'synchronizer')
1197
1198
1199     def run(self):
1200         with self.lock: self.running = True
1201
1202         requested_tx = []
1203         missing_tx = []
1204         requested_histories = {}
1205
1206         # request any missing transactions
1207         for history in self.wallet.history.values():
1208             if history == ['*']: continue
1209             for tx_hash, tx_height in history:
1210                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1211                     missing_tx.append( (tx_hash, tx_height) )
1212         print_error("missing tx", missing_tx)
1213
1214         # wait until we are connected, in case the user is not connected
1215         while not self.interface.is_connected:
1216             time.sleep(1)
1217
1218         # request banner, because 'connected' event happens before this thread is started
1219         self.interface.send([('server.banner',[])],'synchronizer')
1220
1221         # subscriptions
1222         self.subscribe_to_addresses(self.wallet.all_addresses())
1223
1224         while self.is_running():
1225             # 1. send new requests
1226             self.synchronize_wallet()
1227
1228             for tx_hash, tx_height in missing_tx:
1229                 if (tx_hash, tx_height) not in requested_tx:
1230                     self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
1231                     requested_tx.append( (tx_hash, tx_height) )
1232             missing_tx = []
1233
1234             if self.was_updated:
1235                 self.interface.trigger_callback('updated')
1236                 self.was_updated = False
1237
1238             # 2. get a response
1239             r = self.interface.get_response('synchronizer')
1240
1241             # poke sends None. (needed during stop)
1242             if not r: continue
1243
1244             # 3. handle response
1245             method = r['method']
1246             params = r['params']
1247             result = r.get('result')
1248             error = r.get('error')
1249             if error:
1250                 print "error", r
1251                 continue
1252
1253             if method == 'blockchain.address.subscribe':
1254                 addr = params[0]
1255                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1256                     if requested_histories.get(addr) is None:
1257                         self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
1258                         requested_histories[addr] = result
1259
1260             elif method == 'blockchain.address.get_history':
1261                 addr = params[0]
1262                 print_error("receiving history", addr, result)
1263                 if result == ['*']:
1264                     assert requested_histories.pop(addr) == '*'
1265                     self.wallet.receive_history_callback(addr, result)
1266                 else:
1267                     hist = []
1268                     # check that txids are unique
1269                     txids = []
1270                     for item in result:
1271                         tx_hash = item['tx_hash']
1272                         if tx_hash not in txids:
1273                             txids.append(tx_hash)
1274                             hist.append( (tx_hash, item['height']) )
1275
1276                     if len(hist) != len(result):
1277                         raise BaseException("error: server sent history with non-unique txid", result)
1278
1279                     # check that the status corresponds to what was announced
1280                     rs = requested_histories.pop(addr)
1281                     if self.wallet.get_status(hist) != rs:
1282                         raise BaseException("error: status mismatch: %s"%addr)
1283
1284                     # store received history
1285                     self.wallet.receive_history_callback(addr, hist)
1286
1287                     # request transactions that we don't have
1288                     for tx_hash, tx_height in hist:
1289                         if self.wallet.transactions.get(tx_hash) is None:
1290                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1291                                 missing_tx.append( (tx_hash, tx_height) )
1292
1293             elif method == 'blockchain.transaction.get':
1294                 tx_hash = params[0]
1295                 tx_height = params[1]
1296                 d = self.deserialize_tx(tx_hash, tx_height, result)
1297                 self.wallet.receive_tx_callback(tx_hash, d)
1298                 self.was_updated = True
1299                 requested_tx.remove( (tx_hash, tx_height) )
1300                 print_error("received tx:", d)
1301
1302             elif method == 'blockchain.transaction.broadcast':
1303                 self.wallet.tx_result = result
1304                 self.wallet.tx_event.set()
1305
1306             elif method == 'server.banner':
1307                 self.wallet.banner = result
1308                 self.was_updated = True
1309
1310             else:
1311                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1312
1313             if self.was_updated and not requested_tx:
1314                 self.interface.trigger_callback('updated')
1315                 self.was_updated = False
1316
1317
1318     def deserialize_tx(self, tx_hash, tx_height, raw_tx):
1319
1320         assert tx_hash == hash_encode(Hash(raw_tx.decode('hex')))
1321         import deserialize
1322         vds = deserialize.BCDataStream()
1323         vds.write(raw_tx.decode('hex'))
1324         d = deserialize.parse_Transaction(vds)
1325         d['height'] = tx_height
1326         d['tx_hash'] = tx_hash
1327         return d
1328