lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import ecdsa
  31 import Queue
  32 import time
  33
  34 from ecdsa.util import string_to_number, number_to_string
  35 from util import print_error, user_dir, format_satoshis
  36 from bitcoin import *
  37
  38 # URL decode
  39 _ud = re.compile('%([0-9a-hA-H]{2})', re.MULTILINE)
  40 urldecode = lambda x: _ud.sub(lambda m: chr(int(m.group(1), 16)), x)
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46
  47 from version import ELECTRUM_VERSION, SEED_VERSION
  48
  49
  50 class Wallet:
  51     def __init__(self, config={}):
  52
  53         self.config = config
  54         self.electrum_version = ELECTRUM_VERSION
  55
  56         # saved fields
  57         self.seed_version          = config.get('seed_version', SEED_VERSION)
  58         self.gap_limit             = config.get('gap_limit', 5)
  59         self.use_change            = config.get('use_change',True)
  60         self.fee                   = int(config.get('fee',100000))
  61         self.num_zeros             = int(config.get('num_zeros',0))
  62         self.master_public_key     = config.get('master_public_key','')
  63         self.use_encryption        = config.get('use_encryption', False)
  64         self.addresses             = config.get('addresses', [])          # receiving addresses visible for user
  65         self.change_addresses      = config.get('change_addresses', [])   # addresses used as change
  66         self.seed                  = config.get('seed', '')               # encrypted
  67         self.labels                = config.get('labels',{'1NmduGNyC5XejoysbuioodCN3jR3yf64xM':'Electrum donation address'})
  68         self.aliases               = config.get('aliases', {})            # aliases for addresses
  69         self.authorities           = config.get('authorities', {})        # trusted addresses
  70         self.frozen_addresses      = config.get('frozen_addresses',[])
  71         self.prioritized_addresses = config.get('prioritized_addresses',[])
  72         self.receipts              = config.get('receipts',{})            # signed URIs
  73         self.addressbook           = config.get('contacts', ['1NmduGNyC5XejoysbuioodCN3jR3yf64xM'])
  74         self.imported_keys         = config.get('imported_keys',{})
  75         self.history               = config.get('addr_history',{})        # address -> list(txid, height)
  76         self.transactions          = config.get('transactions',{})        # txid -> deserialised
  77
  78         self.requested_amounts     = config.get('requested_amounts',{})   # txid -> deserialised
  79
  80         # not saved
  81         self.prevout_values = {}     # my own transaction outputs
  82         self.spent_outputs = []
  83         self.receipt = None          # next receipt
  84         self.banner = ''
  85
  86         # spv
  87         self.verifier = None
  88
  89         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
  90         # interface.is_up_to_date() returns true when all requests have been answered and processed
  91         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
  92
  93         self.up_to_date = False
  94         self.lock = threading.Lock()
  95         self.tx_event = threading.Event()
  96
  97         if self.seed_version != SEED_VERSION:
  98             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
  99
 100         for tx_hash in self.transactions.keys():
 101             self.update_tx_outputs(tx_hash)
 102
 103
 104     def set_up_to_date(self,b):
 105         with self.lock: self.up_to_date = b
 106
 107     def is_up_to_date(self):
 108         with self.lock: return self.up_to_date
 109
 110     def update(self):
 111         self.up_to_date = False
 112         self.interface.poke('synchronizer')
 113         while not self.is_up_to_date(): time.sleep(0.1)
 114
 115     def import_key(self, sec, password):
 116
 117         # rebuild public key from private key, compressed or uncompressed
 118         pkey = regenerate_key(sec)
 119         if not pkey:
 120             return False
 121
 122         # figure out if private key is compressed
 123         compressed = is_compressed(sec)
 124
 125         # rebuild private and public key from regenerated secret
 126         private_key = GetPrivKey(pkey, compressed)
 127         public_key = GetPubKey(pkey, compressed)
 128         address = public_key_to_bc_address(public_key)
 129
 130         if address in self.all_addresses():
 131             raise BaseException('Address already in wallet')
 132
 133         # store the originally requested keypair into the imported keys table
 134         self.imported_keys[address] = self.pw_encode(sec, password )
 135         return address
 136
 137
 138     def new_seed(self, password):
 139         seed = "%032x"%ecdsa.util.randrange( pow(2,128) )
 140         #self.init_mpk(seed)
 141         # encrypt
 142         self.seed = self.pw_encode( seed, password )
 143
 144
 145     def init_mpk(self,seed):
 146         # public key
 147         curve = SECP256k1
 148         secexp = self.stretch_key(seed)
 149         master_private_key = ecdsa.SigningKey.from_secret_exponent( secexp, curve = SECP256k1 )
 150         self.master_public_key = master_private_key.get_verifying_key().to_string().encode('hex')
 151
 152     def all_addresses(self):
 153         return self.addresses + self.change_addresses + self.imported_keys.keys()
 154
 155     def is_mine(self, address):
 156         return address in self.all_addresses()
 157
 158     def is_change(self, address):
 159         return address in self.change_addresses
 160
 161     def is_valid(self,addr):
 162         ADDRESS_RE = re.compile('[1-9A-HJ-NP-Za-km-z]{26,}\\Z')
 163         if not ADDRESS_RE.match(addr): return False
 164         try:
 165             h = bc_address_to_hash_160(addr)
 166         except:
 167             return False
 168         return addr == hash_160_to_bc_address(h)
 169
 170     def stretch_key(self,seed):
 171         oldseed = seed
 172         for i in range(100000):
 173             seed = hashlib.sha256(seed + oldseed).digest()
 174         return string_to_number( seed )
 175
 176     def get_sequence(self,n,for_change):
 177         return string_to_number( Hash( "%d:%d:"%(n,for_change) + self.master_public_key.decode('hex') ) )
 178
 179     def get_private_key_base58(self, address, password):
 180         secexp, compressed = self.get_private_key(address, password)
 181         if secexp is None: return None
 182         pk = number_to_string( secexp, generator_secp256k1.order() )
 183         return SecretToASecret( pk, compressed )
 184
 185     def get_private_key(self, address, password):
 186         """  Privatekey(type,n) = Master_private_key + H(n|S|type)  """
 187         order = generator_secp256k1.order()
 188
 189         if address in self.imported_keys.keys():
 190             sec = self.pw_decode( self.imported_keys[address], password )
 191             if not sec: return None, None
 192
 193             pkey = regenerate_key(sec)
 194             compressed = is_compressed(sec)
 195             secexp = pkey.secret
 196
 197         else:
 198             if address in self.addresses:
 199                 n = self.addresses.index(address)
 200                 for_change = False
 201             elif address in self.change_addresses:
 202                 n = self.change_addresses.index(address)
 203                 for_change = True
 204             else:
 205                 raise BaseException("unknown address")
 206             try:
 207                 seed = self.pw_decode( self.seed, password)
 208             except:
 209                 raise BaseException("Invalid password")
 210             if not seed: return None
 211             secexp = self.stretch_key(seed)
 212             secexp = ( secexp + self.get_sequence(n,for_change) ) % order
 213             compressed = False
 214
 215         return secexp, compressed
 216
 217     def msg_magic(self, message):
 218         return "\x18Bitcoin Signed Message:\n" + chr( len(message) ) + message
 219
 220     def sign_message(self, address, message, password):
 221         secexp, compressed = self.get_private_key(address, password)
 222         private_key = ecdsa.SigningKey.from_secret_exponent( secexp, curve = SECP256k1 )
 223         public_key = private_key.get_verifying_key()
 224         signature = private_key.sign_digest( Hash( self.msg_magic( message ) ), sigencode = ecdsa.util.sigencode_string )
 225         assert public_key.verify_digest( signature, Hash( self.msg_magic( message ) ), sigdecode = ecdsa.util.sigdecode_string)
 226         for i in range(4):
 227             sig = base64.b64encode( chr(27 + i + (4 if compressed else 0)) + signature )
 228             try:
 229                 self.verify_message( address, sig, message)
 230                 return sig
 231             except:
 232                 continue
 233         else:
 234             raise BaseException("error: cannot sign message")
 235
 236
 237     def verify_message(self, address, signature, message):
 238         """ See http://www.secg.org/download/aid-780/sec1-v2.pdf for the math """
 239         from ecdsa import numbertheory, ellipticcurve, util
 240         import msqr
 241         curve = curve_secp256k1
 242         G = generator_secp256k1
 243         order = G.order()
 244         # extract r,s from signature
 245         sig = base64.b64decode(signature)
 246         if len(sig) != 65: raise BaseException("Wrong encoding")
 247         r,s = util.sigdecode_string(sig[1:], order)
 248         nV = ord(sig[0])
 249         if nV < 27 or nV >= 35:
 250             raise BaseException("Bad encoding")
 251         if nV >= 31:
 252             compressed = True
 253             nV -= 4
 254         else:
 255             compressed = False
 256
 257         recid = nV - 27
 258         # 1.1
 259         x = r + (recid/2) * order
 260         # 1.3
 261         alpha = ( x * x * x  + curve.a() * x + curve.b() ) % curve.p()
 262         beta = msqr.modular_sqrt(alpha, curve.p())
 263         y = beta if (beta - recid) % 2 == 0 else curve.p() - beta
 264         # 1.4 the constructor checks that nR is at infinity
 265         R = ellipticcurve.Point(curve, x, y, order)
 266         # 1.5 compute e from message:
 267         h = Hash( self.msg_magic( message ) )
 268         e = string_to_number(h)
 269         minus_e = -e % order
 270         # 1.6 compute Q = r^-1 (sR - eG)
 271         inv_r = numbertheory.inverse_mod(r,order)
 272         Q = inv_r * ( s * R + minus_e * G )
 273         public_key = ecdsa.VerifyingKey.from_public_point( Q, curve = SECP256k1 )
 274         # check that Q is the public key
 275         public_key.verify_digest( sig[1:], h, sigdecode = ecdsa.util.sigdecode_string)
 276         # check that we get the original signing address
 277         addr = public_key_to_bc_address( encode_point(public_key, compressed) )
 278         if address != addr:
 279             raise BaseException("Bad signature")
 280
 281
 282     def create_new_address(self, for_change):
 283         n = len(self.change_addresses) if for_change else len(self.addresses)
 284         address = self.get_new_address(n, for_change)
 285         if for_change:
 286             self.change_addresses.append(address)
 287         else:
 288             self.addresses.append(address)
 289         self.history[address] = []
 290         return address
 291
 292     def get_new_address(self, n, for_change):
 293         """   Publickey(type,n) = Master_public_key + H(n|S|type)*point  """
 294         curve = SECP256k1
 295         z = self.get_sequence(n, for_change)
 296         master_public_key = ecdsa.VerifyingKey.from_string( self.master_public_key.decode('hex'), curve = SECP256k1 )
 297         pubkey_point = master_public_key.pubkey.point + z*curve.generator
 298         public_key2 = ecdsa.VerifyingKey.from_public_point( pubkey_point, curve = SECP256k1 )
 299         address = public_key_to_bc_address( '04'.decode('hex') + public_key2.to_string() )
 300         print address
 301         return address
 302
 303
 304     def change_gap_limit(self, value):
 305         if value >= self.gap_limit:
 306             self.gap_limit = value
 307             self.save()
 308             self.interface.poke('synchronizer')
 309             return True
 310
 311         elif value >= self.min_acceptable_gap():
 312             k = self.num_unused_trailing_addresses()
 313             n = len(self.addresses) - k + value
 314             self.addresses = self.addresses[0:n]
 315             self.gap_limit = value
 316             self.save()
 317             return True
 318         else:
 319             return False
 320
 321     def num_unused_trailing_addresses(self):
 322         k = 0
 323         for a in self.addresses[::-1]:
 324             if self.history.get(a):break
 325             k = k + 1
 326         return k
 327
 328     def min_acceptable_gap(self):
 329         # fixme: this assumes wallet is synchronized
 330         n = 0
 331         nmax = 0
 332         k = self.num_unused_trailing_addresses()
 333         for a in self.addresses[0:-k]:
 334             if self.history.get(a):
 335                 n = 0
 336             else:
 337                 n += 1
 338                 if n > nmax: nmax = n
 339         return nmax + 1
 340
 341
 342     def synchronize(self):
 343         if not self.master_public_key:
 344             return []
 345
 346         new_addresses = []
 347         while True:
 348             if self.change_addresses == []:
 349                 new_addresses.append( self.create_new_address(True) )
 350                 continue
 351             a = self.change_addresses[-1]
 352             if self.history.get(a):
 353                 new_addresses.append( self.create_new_address(True) )
 354             else:
 355                 break
 356
 357         n = self.gap_limit
 358         while True:
 359             if len(self.addresses) < n:
 360                 new_addresses.append( self.create_new_address(False) )
 361                 continue
 362             if map( lambda a: self.history.get(a, []), self.addresses[-n:] ) == n*[[]]:
 363                 break
 364             else:
 365                 new_addresses.append( self.create_new_address(False) )
 366
 367         return new_addresses
 368
 369
 370     def is_found(self):
 371         return (len(self.change_addresses) > 1 ) or ( len(self.addresses) > self.gap_limit )
 372
 373     def fill_addressbook(self):
 374         for tx_hash, tx in self.transactions.items():
 375             is_send, _, _ = self.get_tx_value(tx_hash)
 376             if is_send:
 377                 for o in tx['outputs']:
 378                     addr = o.get('address')
 379                     if not self.is_mine(addr) and addr not in self.addressbook:
 380                         self.addressbook.append(addr)
 381         # redo labels
 382         # self.update_tx_labels()
 383
 384
 385     def get_address_flags(self, addr):
 386         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 387         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 388         return flags
 389
 390
 391     def get_tx_value(self, tx_hash, addresses = None):
 392         # return the balance for that tx
 393         if addresses is None: addresses = self.all_addresses()
 394         with self.lock:
 395             is_send = False
 396             is_pruned = False
 397             v_in = v_out = v_out_mine = 0
 398             d = self.transactions.get(tx_hash)
 399             if not d:
 400                 return 0, 0, 0
 401
 402             for item in d.get('inputs'):
 403                 addr = item.get('address')
 404                 if addr in addresses:
 405                     is_send = True
 406                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 407                     value = self.prevout_values.get( key )
 408                     if value is None:
 409                         is_pruned = True
 410                     else:
 411                         v_in += value
 412                 else:
 413                     is_pruned = True
 414
 415             for item in d.get('outputs'):
 416                 addr = item.get('address')
 417                 value = item.get('value')
 418                 v_out += value
 419                 if addr in addresses:
 420                     v_out_mine += value
 421
 422         if not is_pruned:
 423             # all inputs are mine:
 424             fee = v_out - v_in
 425             v = v_out_mine - v_in
 426         else:
 427             # some inputs are mine:
 428             fee = None
 429             if is_send:
 430                 v = v_out_mine - v_out
 431             else:
 432                 # no input is mine
 433                 v = v_out_mine
 434
 435         return is_send, v, fee
 436
 437
 438     def get_tx_details(self, tx_hash):
 439         import datetime
 440         if not tx_hash: return ''
 441         tx = self.transactions.get(tx_hash)
 442         is_mine, v, fee = self.get_tx_value(tx_hash)
 443         conf, timestamp = self.verifier.get_confirmations(tx_hash)
 444
 445         if conf:
 446             time_str = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3]
 447         else:
 448             time_str = 'pending'
 449
 450         inputs = map(lambda x: x.get('address'), tx['inputs'])
 451         outputs = map(lambda x: x.get('address'), tx['outputs'])
 452         tx_details = "Transaction Details" +"\n\n" \
 453             + "Transaction ID:\n" + tx_hash + "\n\n" \
 454             + "Status: %d confirmations\n"%conf
 455         if is_mine:
 456             if fee:
 457                 tx_details += "Amount sent: %s\n"% format_satoshis(v-fee, False) \
 458                               + "Transaction fee: %s\n"% format_satoshis(fee, False)
 459             else:
 460                 tx_details += "Amount sent: %s\n"% format_satoshis(v, False) \
 461                               + "Transaction fee: unknown\n"
 462         else:
 463             tx_details += "Amount received: %s\n"% format_satoshis(v, False) \
 464
 465         tx_details += "Date: %s\n\n"%time_str \
 466             + "Inputs:\n-"+ '\n-'.join(inputs) + "\n\n" \
 467             + "Outputs:\n-"+ '\n-'.join(outputs)
 468
 469         r = self.receipts.get(tx_hash)
 470         if r:
 471             tx_details += "\n_______________________________________" \
 472                 + '\n\nSigned URI: ' + r[2] \
 473                 + "\n\nSigned by: " + r[0] \
 474                 + '\n\nSignature: ' + r[1]
 475
 476         return tx_details
 477
 478
 479     def update_tx_outputs(self, tx_hash):
 480         tx = self.transactions.get(tx_hash)
 481         for item in tx.get('outputs'):
 482             value = item.get('value')
 483             key = tx_hash+ ':%d'%item.get('index')
 484             with self.lock:
 485                 self.prevout_values[key] = value
 486
 487         for item in tx.get('inputs'):
 488             if self.is_mine(item.get('address')):
 489                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 490                 self.spent_outputs.append(key)
 491
 492
 493     def get_addr_balance(self, address):
 494         assert self.is_mine(address)
 495         h = self.history.get(address,[])
 496         if h == ['*']: return 0,0
 497         c = u = 0
 498         received_coins = []   # list of coins received at address
 499
 500         for tx_hash, tx_height in h:
 501             d = self.transactions.get(tx_hash)
 502             if not d: continue
 503             for item in d.get('outputs'):
 504                 addr = item.get('address')
 505                 if addr == address:
 506                     key = tx_hash + ':%d'%item['index']
 507                     received_coins.append(key)
 508
 509         for tx_hash, tx_height in h:
 510             d = self.transactions.get(tx_hash)
 511             if not d: continue
 512             v = 0
 513
 514             for item in d.get('inputs'):
 515                 addr = item.get('address')
 516                 if addr == address:
 517                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 518                     value = self.prevout_values.get( key )
 519                     if key in received_coins:
 520                         v -= value
 521
 522             for item in d.get('outputs'):
 523                 addr = item.get('address')
 524                 key = tx_hash + ':%d'%item['index']
 525                 if addr == address:
 526                     v += item.get('value')
 527
 528             if tx_height:
 529                 c += v
 530             else:
 531                 u += v
 532         return c, u
 533
 534     def get_balance(self):
 535         conf = unconf = 0
 536         for addr in self.all_addresses():
 537             c, u = self.get_addr_balance(addr)
 538             conf += c
 539             unconf += u
 540         return conf, unconf
 541
 542
 543     def choose_tx_inputs( self, amount, fixed_fee, from_addr = None ):
 544         """ todo: minimize tx size """
 545         total = 0
 546         fee = self.fee if fixed_fee is None else fixed_fee
 547
 548         coins = []
 549         prioritized_coins = []
 550         domain = [from_addr] if from_addr else self.all_addresses()
 551         for i in self.frozen_addresses:
 552             if i in domain: domain.remove(i)
 553
 554         for i in self.prioritized_addresses:
 555             if i in domain: domain.remove(i)
 556
 557         for addr in domain:
 558             h = self.history.get(addr, [])
 559             if h == ['*']: continue
 560             for tx_hash, tx_height in h:
 561                 tx = self.transactions.get(tx_hash)
 562                 for output in tx.get('outputs'):
 563                     if output.get('address') != addr: continue
 564                     key = tx_hash + ":%d" % output.get('index')
 565                     if key in self.spent_outputs: continue
 566                     output['tx_hash'] = tx_hash
 567                     coins.append(output)
 568
 569
 570         for addr in self.prioritized_addresses:
 571             h = self.history.get(addr, [])
 572             if h == ['*']: continue
 573             for tx_hash, tx_height in h:
 574                 tx = self.transactions.get(tx_hash)
 575                 for output in tx.get('outputs'):
 576                     if output.get('address') != addr: continue
 577                     key = tx_hash + ":%d" % output.get('index')
 578                     if key in self.spent_outputs: continue
 579                     output['tx_hash'] = tx_hash
 580                     prioritized_coins.append(output)
 581
 582
 583         inputs = []
 584         coins = prioritized_coins + coins
 585
 586         for item in coins:
 587             addr = item.get('address')
 588             v = item.get('value')
 589             total += v
 590             inputs.append((addr, v, item['tx_hash'], item['index'], item['raw_output_script'], None, None) )
 591             fee = self.fee*len(inputs) if fixed_fee is None else fixed_fee
 592             if total >= amount + fee: break
 593         else:
 594             #print "not enough funds: %s %s"%(format_satoshis(total), format_satoshis(fee))
 595             inputs = []
 596         return inputs, total, fee
 597
 598     def add_tx_change( self, outputs, amount, fee, total, change_addr=None ):
 599         change_amount = total - ( amount + fee )
 600         if change_amount != 0:
 601             # normally, the update thread should ensure that the last change address is unused
 602             if not change_addr:
 603                 change_addr = self.change_addresses[-1]
 604             outputs.append( ( change_addr,  change_amount) )
 605         return outputs
 606
 607     def sign_inputs( self, inputs, outputs, password ):
 608         s_inputs = []
 609         for i in range(len(inputs)):
 610             addr, v, p_hash, p_pos, p_scriptPubKey, _, _ = inputs[i]
 611             secexp, compressed = self.get_private_key(addr, password)
 612             private_key = ecdsa.SigningKey.from_secret_exponent( secexp, curve = SECP256k1 )
 613             public_key = private_key.get_verifying_key()
 614
 615             pkey = EC_KEY(secexp)
 616             pubkey = GetPubKey(pkey, compressed)
 617
 618             tx = filter( raw_tx( inputs, outputs, for_sig = i ) )
 619             sig = private_key.sign_digest( Hash( tx.decode('hex') ), sigencode = ecdsa.util.sigencode_der )
 620             assert public_key.verify_digest( sig, Hash( tx.decode('hex') ), sigdecode = ecdsa.util.sigdecode_der)
 621             s_inputs.append( (addr, v, p_hash, p_pos, p_scriptPubKey, pubkey, sig) )
 622         return s_inputs
 623
 624     def pw_encode(self, s, password):
 625         if password:
 626             secret = Hash(password)
 627             return EncodeAES(secret, s)
 628         else:
 629             return s
 630
 631     def pw_decode(self, s, password):
 632         if password is not None:
 633             secret = Hash(password)
 634             d = DecodeAES(secret, s)
 635             if s == self.seed:
 636                 try:
 637                     d.decode('hex')
 638                 except:
 639                     raise ValueError("Invalid password")
 640             return d
 641         else:
 642             return s
 643
 644
 645     def get_history(self, address):
 646         with self.lock:
 647             return self.history.get(address)
 648
 649     def get_status(self, h):
 650         if not h: return None
 651         if h == ['*']: return '*'
 652         status = ''
 653         for tx_hash, height in h:
 654             status += tx_hash + ':%d:' % height
 655         return hashlib.sha256( status ).digest().encode('hex')
 656
 657
 658
 659     def receive_tx_callback(self, tx_hash, tx):
 660
 661         if not self.check_new_tx(tx_hash, tx):
 662             raise BaseException("error: received transaction is not consistent with history"%tx_hash)
 663
 664         with self.lock:
 665             self.transactions[tx_hash] = tx
 666
 667         tx_height = tx.get('height')
 668         if self.verifier and tx_height>0:
 669             self.verifier.add(tx_hash, tx_height)
 670
 671         self.update_tx_outputs(tx_hash)
 672
 673         self.save()
 674
 675
 676     def receive_history_callback(self, addr, hist):
 677
 678         if not self.check_new_history(addr, hist):
 679             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
 680
 681         with self.lock:
 682             self.history[addr] = hist
 683             self.save()
 684
 685         if hist != ['*']:
 686             for tx_hash, tx_height in hist:
 687                 if tx_height>0:
 688                     # add it in case it was previously unconfirmed
 689                     if self.verifier: self.verifier.add(tx_hash, tx_height)
 690                     # set the height in case it changed
 691                     tx = self.transactions.get(tx_hash)
 692                     if tx:
 693                         if tx.get('height') != tx_height:
 694                             print_error( "changing height for tx", tx_hash )
 695                             tx['height'] = tx_height
 696
 697
 698     def get_tx_history(self):
 699         with self.lock:
 700             history = self.transactions.values()
 701         history.sort(key = lambda x: x.get('height') if x.get('height') else 1e12)
 702         result = []
 703
 704         balance = 0
 705         for tx in history:
 706             is_mine, v, fee = self.get_tx_value(tx['tx_hash'])
 707             if v is not None: balance += v
 708         c, u = self.get_balance()
 709
 710         if balance != c+u:
 711             v_str = format_satoshis( c+u - balance, True, self.num_zeros)
 712             result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
 713
 714         balance = c + u - balance
 715         for tx in history:
 716             tx_hash = tx['tx_hash']
 717             conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
 718             is_mine, value, fee = self.get_tx_value(tx_hash)
 719             if value is not None:
 720                 balance += value
 721
 722             result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
 723
 724         return result
 725
 726     def get_transactions_at_height(self, height):
 727         with self.lock:
 728             values = self.transactions.values()[:]
 729
 730         out = []
 731         for tx in values:
 732             if tx['height'] == height:
 733                 out.append(tx['tx_hash'])
 734         return out
 735
 736
 737     def get_label(self, tx_hash):
 738         label = self.labels.get(tx_hash)
 739         is_default = (label == '') or (label is None)
 740         if is_default: label = self.get_default_label(tx_hash)
 741         return label, is_default
 742
 743     def get_default_label(self, tx_hash):
 744         tx = self.transactions.get(tx_hash)
 745         default_label = ''
 746         if tx:
 747             is_mine, _, _ = self.get_tx_value(tx_hash)
 748             if is_mine:
 749                 for o in tx['outputs']:
 750                     o_addr = o.get('address')
 751                     if not self.is_mine(o_addr):
 752                         try:
 753                             default_label = self.labels[o_addr]
 754                         except KeyError:
 755                             default_label = o_addr
 756             else:
 757                 for o in tx['outputs']:
 758                     o_addr = o.get('address')
 759                     if self.is_mine(o_addr) and not self.is_change(o_addr):
 760                         break
 761                 else:
 762                     for o in tx['outputs']:
 763                         o_addr = o.get('address')
 764                         if self.is_mine(o_addr):
 765                             break
 766                     else:
 767                         o_addr = None
 768
 769                 if o_addr:
 770                     dest_label = self.labels.get(o_addr)
 771                     try:
 772                         default_label = self.labels[o_addr]
 773                     except KeyError:
 774                         default_label = o_addr
 775
 776         return default_label
 777
 778
 779     def mktx(self, outputs, label, password, fee=None, change_addr=None, from_addr= None):
 780
 781         for address, x in outputs:
 782             assert self.is_valid(address)
 783
 784         amount = sum( map(lambda x:x[1], outputs) )
 785         inputs, total, fee = self.choose_tx_inputs( amount, fee, from_addr )
 786         if not inputs:
 787             raise ValueError("Not enough funds")
 788
 789         if not self.use_change and not change_addr:
 790             change_addr = inputs[-1][0]
 791             print_error( "Sending change to", change_addr )
 792         outputs = self.add_tx_change(outputs, amount, fee, total, change_addr)
 793
 794         if not self.seed:
 795             return repr({'inputs':inputs, 'outputs':outputs})
 796
 797         tx = self.signed_tx(inputs, outputs, password)
 798
 799         for address, x in outputs:
 800             if address not in self.addressbook and not self.is_mine(address):
 801                 self.addressbook.append(address)
 802
 803         if label:
 804             tx_hash = Hash(tx.decode('hex') )[::-1].encode('hex')
 805             self.labels[tx_hash] = label
 806
 807         return tx
 808
 809     def signed_tx(self, inputs, outputs, password):
 810         s_inputs = self.sign_inputs( inputs, outputs, password )
 811         tx = filter( raw_tx( s_inputs, outputs ) )
 812         return tx
 813
 814     def sendtx(self, tx):
 815         # synchronous
 816         h = self.send_tx(tx)
 817         self.tx_event.wait()
 818         return self.receive_tx(h)
 819
 820     def send_tx(self, tx):
 821         # asynchronous
 822         self.tx_event.clear()
 823         tx_hash = Hash(tx.decode('hex') )[::-1].encode('hex')
 824         self.interface.send([('blockchain.transaction.broadcast', [tx])], 'synchronizer')
 825         return tx_hash
 826
 827     def receive_tx(self,tx_hash):
 828         out = self.tx_result
 829         if out != tx_hash:
 830             return False, "error: " + out
 831         if self.receipt:
 832             self.receipts[tx_hash] = self.receipt
 833             self.receipt = None
 834         return True, out
 835
 836
 837     def read_alias(self, alias):
 838         # this might not be the right place for this function.
 839         import urllib
 840
 841         m1 = re.match('([\w\-\.]+)@((\w[\w\-]+\.)+[\w\-]+)', alias)
 842         m2 = re.match('((\w[\w\-]+\.)+[\w\-]+)', alias)
 843         if m1:
 844             url = 'https://' + m1.group(2) + '/bitcoin.id/' + m1.group(1)
 845         elif m2:
 846             url = 'https://' + alias + '/bitcoin.id'
 847         else:
 848             return ''
 849         try:
 850             lines = urllib.urlopen(url).readlines()
 851         except:
 852             return ''
 853
 854         # line 0
 855         line = lines[0].strip().split(':')
 856         if len(line) == 1:
 857             auth_name = None
 858             target = signing_addr = line[0]
 859         else:
 860             target, auth_name, signing_addr, signature = line
 861             msg = "alias:%s:%s:%s"%(alias,target,auth_name)
 862             print msg, signature
 863             self.verify_message(signing_addr, signature, msg)
 864
 865         # other lines are signed updates
 866         for line in lines[1:]:
 867             line = line.strip()
 868             if not line: continue
 869             line = line.split(':')
 870             previous = target
 871             print repr(line)
 872             target, signature = line
 873             self.verify_message(previous, signature, "alias:%s:%s"%(alias,target))
 874
 875         if not self.is_valid(target):
 876             raise ValueError("Invalid bitcoin address")
 877
 878         return target, signing_addr, auth_name
 879
 880     def update_password(self, seed, old_password, new_password):
 881         if new_password == '': new_password = None
 882         self.use_encryption = (new_password != None)
 883         self.seed = self.pw_encode( seed, new_password)
 884         for k in self.imported_keys.keys():
 885             a = self.imported_keys[k]
 886             b = self.pw_decode(a, old_password)
 887             c = self.pw_encode(b, new_password)
 888             self.imported_keys[k] = c
 889         self.save()
 890
 891     def get_alias(self, alias, interactive = False, show_message=None, question = None):
 892         try:
 893             target, signing_address, auth_name = self.read_alias(alias)
 894         except BaseException, e:
 895             # raise exception if verify fails (verify the chain)
 896             if interactive:
 897                 show_message("Alias error: " + str(e))
 898             return
 899
 900         print target, signing_address, auth_name
 901
 902         if auth_name is None:
 903             a = self.aliases.get(alias)
 904             if not a:
 905                 msg = "Warning: the alias '%s' is self-signed.\nThe signing address is %s.\n\nDo you want to add this alias to your list of contacts?"%(alias,signing_address)
 906                 if interactive and question( msg ):
 907                     self.aliases[alias] = (signing_address, target)
 908                 else:
 909                     target = None
 910             else:
 911                 if signing_address != a[0]:
 912                     msg = "Warning: the key of alias '%s' has changed since your last visit! It is possible that someone is trying to do something nasty!!!\nDo you accept to change your trusted key?"%alias
 913                     if interactive and question( msg ):
 914                         self.aliases[alias] = (signing_address, target)
 915                     else:
 916                         target = None
 917         else:
 918             if signing_address not in self.authorities.keys():
 919                 msg = "The alias: '%s' links to %s\n\nWarning: this alias was signed by an unknown key.\nSigning authority: %s\nSigning address: %s\n\nDo you want to add this key to your list of trusted keys?"%(alias,target,auth_name,signing_address)
 920                 if interactive and question( msg ):
 921                     self.authorities[signing_address] = auth_name
 922                 else:
 923                     target = None
 924
 925         if target:
 926             self.aliases[alias] = (signing_address, target)
 927
 928         return target
 929
 930
 931     def parse_url(self, url, show_message, question):
 932         o = url[8:].split('?')
 933         address = o[0]
 934         if len(o)>1:
 935             params = o[1].split('&')
 936         else:
 937             params = []
 938
 939         amount = label = message = signature = identity = ''
 940         for p in params:
 941             k,v = p.split('=')
 942             uv = urldecode(v)
 943             if k == 'amount': amount = uv
 944             elif k == 'message': message = uv
 945             elif k == 'label': label = uv
 946             elif k == 'signature':
 947                 identity, signature = uv.split(':')
 948                 url = url.replace('&%s=%s'%(k,v),'')
 949             else:
 950                 print k,v
 951
 952         if label and self.labels.get(address) != label:
 953             if question('Give label "%s" to address %s ?'%(label,address)):
 954                 if address not in self.addressbook and address not in self.all_addresses():
 955                     self.addressbook.append(address)
 956                 self.labels[address] = label
 957
 958         if signature:
 959             if re.match('^(|([\w\-\.]+)@)((\w[\w\-]+\.)+[\w\-]+)$', identity):
 960                 signing_address = self.get_alias(identity, True, show_message, question)
 961             elif self.is_valid(identity):
 962                 signing_address = identity
 963             else:
 964                 signing_address = None
 965             if not signing_address:
 966                 return
 967             try:
 968                 self.verify_message(signing_address, signature, url )
 969                 self.receipt = (signing_address, signature, url)
 970             except:
 971                 show_message('Warning: the URI contains a bad signature.\nThe identity of the recipient cannot be verified.')
 972                 address = amount = label = identity = message = ''
 973
 974         if re.match('^(|([\w\-\.]+)@)((\w[\w\-]+\.)+[\w\-]+)$', address):
 975             payto_address = self.get_alias(address, True, show_message, question)
 976             if payto_address:
 977                 address = address + ' <' + payto_address + '>'
 978
 979         return address, amount, label, message, signature, identity, url
 980
 981
 982
 983     def freeze(self,addr):
 984         if addr in self.all_addresses() and addr not in self.frozen_addresses:
 985             self.unprioritize(addr)
 986             self.frozen_addresses.append(addr)
 987             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 988             return True
 989         else:
 990             return False
 991
 992     def unfreeze(self,addr):
 993         if addr in self.all_addresses() and addr in self.frozen_addresses:
 994             self.frozen_addresses.remove(addr)
 995             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 996             return True
 997         else:
 998             return False
 999
1000     def prioritize(self,addr):
1001         if addr in self.all_addresses() and addr not in self.prioritized_addresses:
1002             self.unfreeze(addr)
1003             self.prioritized_addresses.append(addr)
1004             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1005             return True
1006         else:
1007             return False
1008
1009     def unprioritize(self,addr):
1010         if addr in self.all_addresses() and addr in self.prioritized_addresses:
1011             self.prioritized_addresses.remove(addr)
1012             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1013             return True
1014         else:
1015             return False
1016
1017     def save(self):
1018         s = {
1019             'seed_version': self.seed_version,
1020             'use_encryption': self.use_encryption,
1021             'use_change': self.use_change,
1022             'master_public_key': self.master_public_key,
1023             'fee': self.fee,
1024             'seed': self.seed,
1025             'addresses': self.addresses,
1026             'change_addresses': self.change_addresses,
1027             'addr_history': self.history,
1028             'labels': self.labels,
1029             'contacts': self.addressbook,
1030             'imported_keys': self.imported_keys,
1031             'aliases': self.aliases,
1032             'authorities': self.authorities,
1033             'receipts': self.receipts,
1034             'num_zeros': self.num_zeros,
1035             'frozen_addresses': self.frozen_addresses,
1036             'prioritized_addresses': self.prioritized_addresses,
1037             'gap_limit': self.gap_limit,
1038             'transactions': self.transactions,
1039             'requested_amounts': self.requested_amounts,
1040         }
1041         for k, v in s.items():
1042             self.config.set_key(k,v)
1043         self.config.save()
1044
1045     def set_verifier(self, verifier):
1046         self.verifier = verifier
1047
1048         # review stored transactions and send them to the verifier
1049         # (they are not necessarily in the history, because history items might have have been pruned)
1050         for tx_hash, tx in self.transactions.items():
1051             tx_height = tx.get('height')
1052             if tx_height <1:
1053                 print_error( "skipping", tx_hash, tx_height )
1054                 continue
1055
1056             if tx_height>0:
1057                 self.verifier.add(tx_hash, tx_height)
1058
1059         # review transactions that are in the history
1060         for addr, hist in self.history.items():
1061             if hist == ['*']: continue
1062             for tx_hash, tx_height in hist:
1063                 if tx_height>0:
1064                     # add it in case it was previously unconfirmed
1065                     self.verifier.add(tx_hash, tx_height)
1066                     # set the height in case it changed
1067                     tx = self.transactions.get(tx_hash)
1068                     if tx:
1069                         if tx.get('height') != tx_height:
1070                             print_error( "changing height for tx", tx_hash )
1071                             tx['height'] = tx_height
1072
1073
1074     def is_addr_in_tx(self, addr, tx):
1075         found = False
1076         for txin in tx.get('inputs'):
1077             if addr == txin.get('address'):
1078                 found = True
1079                 break
1080         for txout in tx.get('outputs'):
1081             if addr == txout.get('address'):
1082                 found = True
1083                 break
1084         return found
1085
1086
1087     def check_new_history(self, addr, hist):
1088
1089         # check that all tx in hist are relevant
1090         if hist != ['*']:
1091             for tx_hash, height in hist:
1092                 tx = self.transactions.get(tx_hash)
1093                 if not tx: continue
1094                 if not self.is_addr_in_tx(addr,tx):
1095                     return False
1096
1097         # check that we are not "orphaning" a transaction
1098         old_hist = self.history.get(addr,[])
1099         if old_hist == ['*']: return True
1100
1101         for tx_hash, height in old_hist:
1102             if tx_hash in map(lambda x:x[0], hist): continue
1103             found = False
1104             for _addr, _hist in self.history.items():
1105                 if _addr == addr: continue
1106                 if _hist == ['*']: continue
1107                 _tx_hist = map(lambda x:x[0], _hist)
1108                 if tx_hash in _tx_hist:
1109                     found = True
1110                     break
1111
1112             if not found:
1113                 tx = self.transactions.get(tx_hash)
1114                 # already verified?
1115                 if tx.get('height'):
1116                     continue
1117                 # unconfirmed tx
1118                 print_error("new history is orphaning transaction:", tx_hash)
1119                 # check that all outputs are not mine, request histories
1120                 ext_requests = []
1121                 for o in tx.get('outputs'):
1122                     _addr = o.get('address')
1123                     # assert not self.is_mine(_addr)
1124                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1125
1126                 ext_h = self.interface.synchronous_get(ext_requests)
1127                 height = None
1128                 for h in ext_h:
1129                     if h == ['*']: continue
1130                     for item in h:
1131                         if item.get('tx_hash') == tx_hash:
1132                             height = item.get('height')
1133                             tx['height'] = height
1134                 if height:
1135                     print_error("found height for", tx_hash, height)
1136                     self.verifier.add(tx_hash, height)
1137                 else:
1138                     print_error("removing orphaned tx from history", tx_hash)
1139                     self.transactions.pop(tx_hash)
1140
1141         return True
1142
1143
1144
1145     def check_new_tx(self, tx_hash, tx):
1146         # 1 check that tx is referenced in addr_history.
1147         addresses = []
1148         for addr, hist in self.history.items():
1149             if hist == ['*']:continue
1150             for txh, height in hist:
1151                 if txh == tx_hash:
1152                     addresses.append(addr)
1153
1154         if not addresses:
1155             return False
1156
1157         # 2 check that referencing addresses are in the tx
1158         for addr in addresses:
1159             if not self.is_addr_in_tx(addr, tx):
1160                 return False
1161
1162         return True
1163
1164
1165
1166
1167 class WalletSynchronizer(threading.Thread):
1168
1169
1170     def __init__(self, wallet, config):
1171         threading.Thread.__init__(self)
1172         self.daemon = True
1173         self.wallet = wallet
1174         self.interface = self.wallet.interface
1175         self.interface.register_channel('synchronizer')
1176         self.wallet.interface.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1177         self.wallet.interface.register_callback('connected', lambda: self.interface.send([('server.banner',[])],'synchronizer') )
1178         self.was_updated = True
1179         self.running = False
1180         self.lock = threading.Lock()
1181
1182     def stop(self):
1183         with self.lock: self.running = False
1184         self.interface.poke('synchronizer')
1185
1186     def is_running(self):
1187         with self.lock: return self.running
1188
1189     def synchronize_wallet(self):
1190         new_addresses = self.wallet.synchronize()
1191         if new_addresses:
1192             self.subscribe_to_addresses(new_addresses)
1193             self.wallet.up_to_date = False
1194             return
1195
1196         if not self.interface.is_up_to_date('synchronizer'):
1197             if self.wallet.is_up_to_date():
1198                 self.wallet.set_up_to_date(False)
1199                 self.was_updated = True
1200             return
1201
1202         self.wallet.set_up_to_date(True)
1203         self.was_updated = True
1204
1205
1206     def subscribe_to_addresses(self, addresses):
1207         messages = []
1208         for addr in addresses:
1209             messages.append(('blockchain.address.subscribe', [addr]))
1210         self.interface.send( messages, 'synchronizer')
1211
1212
1213     def run(self):
1214         with self.lock: self.running = True
1215
1216         requested_tx = []
1217         missing_tx = []
1218         requested_histories = {}
1219
1220         # request any missing transactions
1221         for history in self.wallet.history.values():
1222             if history == ['*']: continue
1223             for tx_hash, tx_height in history:
1224                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1225                     missing_tx.append( (tx_hash, tx_height) )
1226         print_error("missing tx", missing_tx)
1227
1228         # wait until we are connected, in case the user is not connected
1229         while not self.interface.is_connected:
1230             time.sleep(1)
1231
1232         # request banner, because 'connected' event happens before this thread is started
1233         self.interface.send([('server.banner',[])],'synchronizer')
1234
1235         # subscriptions
1236         self.subscribe_to_addresses(self.wallet.all_addresses())
1237
1238         while self.is_running():
1239             # 1. send new requests
1240             self.synchronize_wallet()
1241
1242             for tx_hash, tx_height in missing_tx:
1243                 if (tx_hash, tx_height) not in requested_tx:
1244                     self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
1245                     requested_tx.append( (tx_hash, tx_height) )
1246             missing_tx = []
1247
1248             if self.was_updated:
1249                 self.interface.trigger_callback('updated')
1250                 self.was_updated = False
1251
1252             # 2. get a response
1253             r = self.interface.get_response('synchronizer')
1254
1255             # poke sends None. (needed during stop)
1256             if not r: continue
1257
1258             # 3. handle response
1259             method = r['method']
1260             params = r['params']
1261             result = r.get('result')
1262             error = r.get('error')
1263             if error:
1264                 print "error", r
1265                 continue
1266
1267             if method == 'blockchain.address.subscribe':
1268                 addr = params[0]
1269                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1270                     if requested_histories.get(addr) is None:
1271                         self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
1272                         requested_histories[addr] = result
1273
1274             elif method == 'blockchain.address.get_history':
1275                 addr = params[0]
1276                 print_error("receiving history", addr, result)
1277                 if result == ['*']:
1278                     assert requested_histories.pop(addr) == '*'
1279                     self.wallet.receive_history_callback(addr, result)
1280                 else:
1281                     hist = []
1282                     # check that txids are unique
1283                     txids = []
1284                     for item in result:
1285                         tx_hash = item['tx_hash']
1286                         if tx_hash not in txids:
1287                             txids.append(tx_hash)
1288                             hist.append( (tx_hash, item['height']) )
1289
1290                     if len(hist) != len(result):
1291                         raise BaseException("error: server sent history with non-unique txid", result)
1292
1293                     # check that the status corresponds to what was announced
1294                     rs = requested_histories.pop(addr)
1295                     if self.wallet.get_status(hist) != rs:
1296                         raise BaseException("error: status mismatch: %s"%addr)
1297
1298                     # store received history
1299                     self.wallet.receive_history_callback(addr, hist)
1300
1301                     # request transactions that we don't have
1302                     for tx_hash, tx_height in hist:
1303                         if self.wallet.transactions.get(tx_hash) is None:
1304                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1305                                 missing_tx.append( (tx_hash, tx_height) )
1306
1307             elif method == 'blockchain.transaction.get':
1308                 tx_hash = params[0]
1309                 tx_height = params[1]
1310                 d = self.deserialize_tx(tx_hash, tx_height, result)
1311                 self.wallet.receive_tx_callback(tx_hash, d)
1312                 self.was_updated = True
1313                 requested_tx.remove( (tx_hash, tx_height) )
1314                 print_error("received tx:", d)
1315
1316             elif method == 'blockchain.transaction.broadcast':
1317                 self.wallet.tx_result = result
1318                 self.wallet.tx_event.set()
1319
1320             elif method == 'server.banner':
1321                 self.wallet.banner = result
1322                 self.was_updated = True
1323
1324             else:
1325                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1326
1327             if self.was_updated and not requested_tx:
1328                 self.interface.trigger_callback('updated')
1329                 self.was_updated = False
1330
1331
1332     def deserialize_tx(self, tx_hash, tx_height, raw_tx):
1333
1334         assert tx_hash == hash_encode(Hash(raw_tx.decode('hex')))
1335         import deserialize
1336         vds = deserialize.BCDataStream()
1337         vds.write(raw_tx.decode('hex'))
1338         d = deserialize.parse_Transaction(vds)
1339         d['height'] = tx_height
1340         d['tx_hash'] = tx_hash
1341         return d
1342