lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, user_dir, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36
  37 # AES encryption
  38 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  39 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  40
  41 def pw_encode(s, password):
  42     if password:
  43         secret = Hash(password)
  44         return EncodeAES(secret, s)
  45     else:
  46         return s
  47
  48 def pw_decode(s, password):
  49     if password is not None:
  50         secret = Hash(password)
  51         try:
  52             d = DecodeAES(secret, s)
  53         except:
  54             raise BaseException('Invalid password')
  55         return d
  56     else:
  57         return s
  58
  59
  60
  61
  62
  63 from version import ELECTRUM_VERSION, SEED_VERSION
  64
  65
  66 class Wallet:
  67     def __init__(self, config={}):
  68
  69         self.config = config
  70         self.electrum_version = ELECTRUM_VERSION
  71         self.gap_limit_for_change = 3 # constant
  72
  73         # saved fields
  74         self.seed_version          = config.get('seed_version', SEED_VERSION)
  75         self.gap_limit             = config.get('gap_limit', 5)
  76         self.use_change            = config.get('use_change',True)
  77         self.fee                   = int(config.get('fee_per_kb',50000))
  78         self.num_zeros             = int(config.get('num_zeros',0))
  79         self.use_encryption        = config.get('use_encryption', False)
  80         self.seed                  = config.get('seed', '')               # encrypted
  81         self.labels                = config.get('labels', {})
  82         self.frozen_addresses      = config.get('frozen_addresses',[])
  83         self.prioritized_addresses = config.get('prioritized_addresses',[])
  84         self.addressbook           = config.get('contacts', [])
  85
  86         self.imported_keys         = config.get('imported_keys',{})
  87         self.history               = config.get('addr_history',{})        # address -> list(txid, height)
  88
  89
  90         self.master_public_keys = config.get('master_public_keys',{})
  91         self.master_private_keys = config.get('master_private_keys', {})
  92
  93         self.load_accounts(config)
  94
  95         self.transactions = {}
  96         tx = config.get('transactions',{})
  97         try:
  98             for k,v in tx.items(): self.transactions[k] = Transaction(v)
  99         except:
 100             print_msg("Warning: Cannot deserialize transactions. skipping")
 101
 102         # not saved
 103         self.prevout_values = {}     # my own transaction outputs
 104         self.spent_outputs = []
 105
 106         # spv
 107         self.verifier = None
 108
 109         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 110         # interface.is_up_to_date() returns true when all requests have been answered and processed
 111         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 112
 113         self.up_to_date = False
 114         self.lock = threading.Lock()
 115         self.transaction_lock = threading.Lock()
 116         self.tx_event = threading.Event()
 117
 118         if self.seed_version != SEED_VERSION:
 119             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
 120
 121         for tx_hash, tx in self.transactions.items():
 122             if self.check_new_tx(tx_hash, tx):
 123                 self.update_tx_outputs(tx_hash)
 124             else:
 125                 print_error("unreferenced tx", tx_hash)
 126                 self.transactions.pop(tx_hash)
 127
 128
 129     def set_up_to_date(self,b):
 130         with self.lock: self.up_to_date = b
 131
 132     def is_up_to_date(self):
 133         with self.lock: return self.up_to_date
 134
 135     def update(self):
 136         self.up_to_date = False
 137         self.interface.poke('synchronizer')
 138         while not self.is_up_to_date(): time.sleep(0.1)
 139
 140     def import_key(self, sec, password):
 141         # check password
 142         seed = self.decode_seed(password)
 143         try:
 144             address = address_from_private_key(sec)
 145         except:
 146             raise BaseException('Invalid private key')
 147
 148         if self.is_mine(address):
 149             raise BaseException('Address already in wallet')
 150
 151         # store the originally requested keypair into the imported keys table
 152         self.imported_keys[address] = pw_encode(sec, password )
 153         self.config.set_key('imported_keys', self.imported_keys, True)
 154         return address
 155
 156     def delete_imported_key(self, addr):
 157         if addr in self.imported_keys:
 158             self.imported_keys.pop(addr)
 159             self.config.set_key('imported_keys', self.imported_keys, True)
 160
 161
 162     def init_seed(self, seed):
 163         if self.seed: raise BaseException("a seed exists")
 164         if not seed:
 165             seed = random_seed(128)
 166         self.seed = seed
 167
 168
 169     def save_seed(self):
 170         self.config.set_key('seed', self.seed, True)
 171         self.config.set_key('seed_version', self.seed_version, True)
 172
 173         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 174
 175         # normal accounts
 176         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 177         # p2sh 2of2
 178         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 179         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 180         # p2sh 2of3
 181         k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 182         k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 183         k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 184
 185         self.master_public_keys = {
 186             "m/0'/": (c0, K0, cK0),
 187             "m/1'/": (c1, K1, cK1),
 188             "m/2'/": (c2, K2, cK2),
 189             "m/3'/": (c3, K3, cK3),
 190             "m/4'/": (c4, K4, cK4),
 191             "m/5'/": (c5, K5, cK5)
 192             }
 193
 194         self.master_private_keys = {
 195             "m/0'/": k0,
 196             "m/1'/": k1,
 197             "m/2'/": k2,
 198             "m/3'/": k3,
 199             "m/4'/": k4,
 200             "m/5'/": k5
 201             }
 202
 203         self.config.set_key('master_public_keys', self.master_public_keys, True)
 204         self.config.set_key('master_private_keys', self.master_private_keys, True)
 205
 206         # create default account
 207         self.create_account('Main account')
 208
 209
 210     def account_id(self, account_type, i):
 211         if account_type is None:
 212             return "m/0'/%d'"%i
 213         elif account_type == '2of2':
 214             return "m/1'/%d & m/2'/%d"%(i,i)
 215         elif account_type == '2of3':
 216             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 217         else:
 218             raise BaseException('unknown account type')
 219
 220
 221     def num_accounts(self, account_type):
 222         keys = self.accounts.keys()
 223         i = 0
 224         while True:
 225             account_id = self.account_id(account_type, i)
 226             if account_id not in keys: break
 227             i += 1
 228         return i
 229
 230
 231     def create_account(self, name, account_type = None):
 232         i = self.num_accounts(account_type)
 233         acount_id = self.account_id(account_type,i)
 234
 235         if account_type is None:
 236             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 237             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 238             account = BIP32_Account({ 'name':name, 'c':c0, 'K':K0, 'cK':cK0 })
 239
 240         elif account_type == '2of2':
 241             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 242             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 243             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 244             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 245             account = BIP32_Account_2of2({ 'name':name, 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 246
 247         elif account_type == '2of3':
 248             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 249             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 250             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 251             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 252             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 253             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 254             account = BIP32_Account_2of3({ 'name':name, 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 255
 256         self.accounts[account_id] = account
 257         self.save_accounts()
 258
 259
 260     def save_accounts(self):
 261         d = {}
 262         for k, v in self.accounts.items():
 263             d[k] = v.dump()
 264         self.config.set_key('accounts', d, True)
 265
 266
 267     def load_accounts(self, config):
 268         d = config.get('accounts', {})
 269         self.accounts = {}
 270         for k, v in d.items():
 271             if '&' in k:
 272                 self.accounts[k] = BIP32_Account_2of2(v)
 273             else:
 274                 self.accounts[k] = BIP32_Account(v)
 275
 276
 277
 278
 279     def addresses(self, include_change = True):
 280         o = self.get_account_addresses(-1, include_change)
 281         for a in self.accounts.keys():
 282             o += self.get_account_addresses(a, include_change)
 283         return o
 284
 285
 286     def is_mine(self, address):
 287         return address in self.addresses(True)
 288
 289     def is_change(self, address):
 290         if not self.is_mine(address): return False
 291         if address in self.imported_keys.keys(): return False
 292         acct, s = self.get_address_index(address)
 293         return s[0] == 1
 294
 295     def get_master_public_key(self):
 296         raise
 297         return self.config.get("master_public_key")
 298
 299     def get_master_private_key(self, account, password):
 300         master_k = pw_decode( self.master_private_keys[account], password)
 301         master_c, master_K, master_Kc = self.master_public_keys[account]
 302         try:
 303             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 304             assert K.encode('hex') == master_K
 305         except:
 306             raise BaseException("Invalid password")
 307         return master_k
 308
 309
 310     def get_address_index(self, address):
 311         if address in self.imported_keys.keys():
 312             return -1, None
 313         for account in self.accounts.keys():
 314             for for_change in [0,1]:
 315                 addresses = self.accounts[account].get_addresses(for_change)
 316                 for addr in addresses:
 317                     if address == addr:
 318                         return account, (for_change, addresses.index(addr))
 319         raise BaseException("not found")
 320
 321
 322     def get_public_key(self, address):
 323         account, sequence = self.get_address_index(address)
 324         return self.accounts[account].get_pubkey( *sequence )
 325
 326
 327     def decode_seed(self, password):
 328         seed = pw_decode(self.seed, password)
 329         #todo:  #self.sequences[0].check_seed(seed)
 330         return seed
 331
 332
 333     def get_private_key(self, address, password):
 334         out = []
 335         if address in self.imported_keys.keys():
 336             out.append( pw_decode( self.imported_keys[address], password ) )
 337         else:
 338             account, sequence = self.get_address_index(address)
 339             # assert address == self.accounts[account].get_address(*sequence)
 340             l = account.split("&")
 341             for s in l:
 342                 s = s.strip()
 343                 m = re.match("(m/\d+'/)(\d+)", s)
 344                 if m:
 345                     root = m.group(1)
 346                     if root not in self.master_private_keys.keys(): continue
 347                     num = int(m.group(2))
 348                     master_k = self.get_master_private_key(root, password)
 349                     master_c, _, _ = self.master_public_keys[root]
 350                     pk = bip32_private_key( (num,) + sequence, master_k.decode('hex'), master_c.decode('hex'))
 351                     out.append(pk)
 352
 353         return out
 354
 355
 356
 357
 358     def signrawtransaction(self, tx, input_info, private_keys, password):
 359         import deserialize
 360         unspent_coins = self.get_unspent_coins()
 361         seed = self.decode_seed(password)
 362
 363         # build a list of public/private keys
 364         keypairs = {}
 365         for sec in private_keys:
 366             compressed = is_compressed(sec)
 367             pkey = regenerate_key(sec)
 368             pubkey = GetPubKey(pkey.pubkey, compressed)
 369             keypairs[ pubkey.encode('hex') ] = sec
 370
 371
 372         for txin in tx.inputs:
 373             # convert to own format
 374             txin['tx_hash'] = txin['prevout_hash']
 375             txin['index'] = txin['prevout_n']
 376
 377             for item in input_info:
 378                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 379                     txin['raw_output_script'] = item['scriptPubKey']
 380                     txin['redeemScript'] = item.get('redeemScript')
 381                     txin['KeyID'] = item.get('KeyID')
 382                     break
 383             else:
 384                 for item in unspent_coins:
 385                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 386                         txin['raw_output_script'] = item['raw_output_script']
 387                         break
 388                 else:
 389                     # if neither, we might want to get it from the server..
 390                     raise
 391
 392             # find the address and fill private_keys
 393             if txin.get('KeyID'):
 394                 account, name, sequence = txin.get('KeyID')
 395                 if name != 'BIP32': continue
 396                 sec = self.accounts[account].get_private_key(sequence, seed)
 397                 pubkey = self.accounts[account].get_pubkey(sequence)
 398                 txin['address'] = addr
 399                 keypairs[pubkey] = [sec]
 400
 401             redeem_script = txin.get("redeemScript")
 402             if redeem_script:
 403                 num, redeem_pubkeys = deserialize.parse_redeemScript(redeem_script)
 404                 addr = hash_160_to_bc_address(hash_160(redeem_script.decode('hex')), 5)
 405                 txin['address'] = addr
 406
 407
 408             elif txin.get("raw_output_script"):
 409                 addr = deserialize.get_address_from_output_script(txin.get("raw_output_script").decode('hex'))
 410                 sec = self.get_private_key(addr, password)
 411                 if sec:
 412                     keypairs[pubkey] = [sec]
 413                     txin['address'] = addr
 414
 415         tx.sign( keypairs )
 416
 417     def sign_message(self, address, message, password):
 418         sec = self.get_private_key(address, password)
 419         key = regenerate_key(sec)
 420         compressed = is_compressed(sec)
 421         return key.sign_message(message, compressed, address)
 422
 423     def verify_message(self, address, signature, message):
 424         try:
 425             EC_KEY.verify_message(address, signature, message)
 426             return True
 427         except BaseException as e:
 428             print_error("Verification error: {0}".format(e))
 429             return False
 430
 431
 432     def change_gap_limit(self, value):
 433         if value >= self.gap_limit:
 434             self.gap_limit = value
 435             self.config.set_key('gap_limit', self.gap_limit, True)
 436             self.interface.poke('synchronizer')
 437             return True
 438
 439         elif value >= self.min_acceptable_gap():
 440             for key, account in self.accounts.items():
 441                 addresses = account[0]
 442                 k = self.num_unused_trailing_addresses(addresses)
 443                 n = len(addresses) - k + value
 444                 addresses = addresses[0:n]
 445                 self.accounts[key][0] = addresses
 446
 447             self.gap_limit = value
 448             self.config.set_key('gap_limit', self.gap_limit, True)
 449             self.save_accounts()
 450             return True
 451         else:
 452             return False
 453
 454     def num_unused_trailing_addresses(self, addresses):
 455         k = 0
 456         for a in addresses[::-1]:
 457             if self.history.get(a):break
 458             k = k + 1
 459         return k
 460
 461     def min_acceptable_gap(self):
 462         # fixme: this assumes wallet is synchronized
 463         n = 0
 464         nmax = 0
 465
 466         for account in self.accounts.values():
 467             addresses = account.get_addresses(0)
 468             k = self.num_unused_trailing_addresses(addresses)
 469             for a in addresses[0:-k]:
 470                 if self.history.get(a):
 471                     n = 0
 472                 else:
 473                     n += 1
 474                     if n > nmax: nmax = n
 475         return nmax + 1
 476
 477
 478     def address_is_old(self, address):
 479         age = -1
 480         h = self.history.get(address, [])
 481         if h == ['*']:
 482             return True
 483         for tx_hash, tx_height in h:
 484             if tx_height == 0:
 485                 tx_age = 0
 486             else:
 487                 tx_age = self.verifier.height - tx_height + 1
 488             if tx_age > age:
 489                 age = tx_age
 490         return age > 2
 491
 492
 493     def synchronize_sequence(self, account, for_change):
 494         limit = self.gap_limit_for_change if for_change else self.gap_limit
 495         new_addresses = []
 496         while True:
 497             addresses = account.get_addresses(for_change)
 498             if len(addresses) < limit:
 499                 address = account.create_new_address(for_change)
 500                 self.history[address] = []
 501                 new_addresses.append( address )
 502                 continue
 503
 504             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 505                 break
 506             else:
 507                 address = account.create_new_address(for_change)
 508                 self.history[address] = []
 509                 new_addresses.append( address )
 510
 511         return new_addresses
 512
 513
 514     def synchronize_account(self, account):
 515         new = []
 516         new += self.synchronize_sequence(account, 0)
 517         new += self.synchronize_sequence(account, 1)
 518         return new
 519
 520     def synchronize(self):
 521         new = []
 522         for account in self.accounts.values():
 523             new += self.synchronize_account(account)
 524         if new:
 525             self.save_accounts()
 526             self.config.set_key('addr_history', self.history, True)
 527         return new
 528
 529
 530     def is_found(self):
 531         return self.history.values() != [[]] * len(self.history)
 532
 533
 534     def add_contact(self, address, label=None):
 535         self.addressbook.append(address)
 536         self.config.set_key('contacts', self.addressbook, True)
 537         if label:
 538             self.labels[address] = label
 539             self.config.set_key('labels', self.labels)
 540
 541     def delete_contact(self, addr):
 542         if addr in self.addressbook:
 543             self.addressbook.remove(addr)
 544             self.config.set_key('addressbook', self.addressbook, True)
 545
 546
 547     def fill_addressbook(self):
 548         for tx_hash, tx in self.transactions.items():
 549             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 550             if is_send:
 551                 for addr, v in tx.outputs:
 552                     if not self.is_mine(addr) and addr not in self.addressbook:
 553                         self.addressbook.append(addr)
 554         # redo labels
 555         # self.update_tx_labels()
 556
 557     def get_num_tx(self, address):
 558         n = 0
 559         for tx in self.transactions.values():
 560             if address in map(lambda x:x[0], tx.outputs): n += 1
 561         return n
 562
 563
 564     def get_address_flags(self, addr):
 565         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 566         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 567         return flags
 568
 569
 570     def get_tx_value(self, tx, account=None):
 571         domain = self.get_account_addresses(account)
 572         return tx.get_value(domain, self.prevout_values)
 573
 574
 575     def update_tx_outputs(self, tx_hash):
 576         tx = self.transactions.get(tx_hash)
 577
 578         for i, (addr, value) in enumerate(tx.outputs):
 579             key = tx_hash+ ':%d'%i
 580             self.prevout_values[key] = value
 581
 582         for item in tx.inputs:
 583             if self.is_mine(item.get('address')):
 584                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 585                 self.spent_outputs.append(key)
 586
 587
 588     def get_addr_balance(self, address):
 589         assert self.is_mine(address)
 590         h = self.history.get(address,[])
 591         if h == ['*']: return 0,0
 592         c = u = 0
 593         received_coins = []   # list of coins received at address
 594
 595         for tx_hash, tx_height in h:
 596             tx = self.transactions.get(tx_hash)
 597             if not tx: continue
 598
 599             for i, (addr, value) in enumerate(tx.outputs):
 600                 if addr == address:
 601                     key = tx_hash + ':%d'%i
 602                     received_coins.append(key)
 603
 604         for tx_hash, tx_height in h:
 605             tx = self.transactions.get(tx_hash)
 606             if not tx: continue
 607             v = 0
 608
 609             for item in tx.inputs:
 610                 addr = item.get('address')
 611                 if addr == address:
 612                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 613                     value = self.prevout_values.get( key )
 614                     if key in received_coins:
 615                         v -= value
 616
 617             for i, (addr, value) in enumerate(tx.outputs):
 618                 key = tx_hash + ':%d'%i
 619                 if addr == address:
 620                     v += value
 621
 622             if tx_height:
 623                 c += v
 624             else:
 625                 u += v
 626         return c, u
 627
 628
 629     def get_accounts(self):
 630         accounts = {}
 631         for k, account in self.accounts.items():
 632             accounts[k] = account.name
 633         if self.imported_keys:
 634             accounts[-1] = 'Imported keys'
 635         return accounts
 636
 637     def get_account_addresses(self, a, include_change=True):
 638         if a is None:
 639             o = self.addresses(True)
 640         elif a == -1:
 641             o = self.imported_keys.keys()
 642         else:
 643             ac = self.accounts[a]
 644             o = ac.get_addresses(0)
 645             if include_change: o += ac.get_addresses(1)
 646         return o
 647
 648     def get_imported_balance(self):
 649         cc = uu = 0
 650         for addr in self.imported_keys.keys():
 651             c, u = self.get_addr_balance(addr)
 652             cc += c
 653             uu += u
 654         return cc, uu
 655
 656     def get_account_balance(self, account):
 657         if account is None:
 658             return self.get_balance()
 659         elif account == -1:
 660             return self.get_imported_balance()
 661
 662         conf = unconf = 0
 663         for addr in self.get_account_addresses(account):
 664             c, u = self.get_addr_balance(addr)
 665             conf += c
 666             unconf += u
 667         return conf, unconf
 668
 669     def get_frozen_balance(self):
 670         conf = unconf = 0
 671         for addr in self.frozen_addresses:
 672             c, u = self.get_addr_balance(addr)
 673             conf += c
 674             unconf += u
 675         return conf, unconf
 676
 677
 678     def get_balance(self):
 679         cc = uu = 0
 680         for a in self.accounts.keys():
 681             c, u = self.get_account_balance(a)
 682             cc += c
 683             uu += u
 684         c, u = self.get_imported_balance()
 685         cc += c
 686         uu += u
 687         return cc, uu
 688
 689
 690     def get_unspent_coins(self, domain=None):
 691         coins = []
 692         if domain is None: domain = self.addresses(True)
 693         for addr in domain:
 694             h = self.history.get(addr, [])
 695             if h == ['*']: continue
 696             for tx_hash, tx_height in h:
 697                 tx = self.transactions.get(tx_hash)
 698                 if tx is None: raise BaseException("Wallet not synchronized")
 699                 for output in tx.d.get('outputs'):
 700                     if output.get('address') != addr: continue
 701                     key = tx_hash + ":%d" % output.get('index')
 702                     if key in self.spent_outputs: continue
 703                     output['tx_hash'] = tx_hash
 704                     coins.append(output)
 705         return coins
 706
 707
 708
 709     def choose_tx_inputs( self, amount, fixed_fee, account = None ):
 710         """ todo: minimize tx size """
 711         total = 0
 712         fee = self.fee if fixed_fee is None else fixed_fee
 713         domain = self.get_account_addresses(account)
 714         coins = []
 715         prioritized_coins = []
 716         for i in self.frozen_addresses:
 717             if i in domain: domain.remove(i)
 718
 719         for i in self.prioritized_addresses:
 720             if i in domain: domain.remove(i)
 721
 722         coins = self.get_unspent_coins(domain)
 723         prioritized_coins = self.get_unspent_coins(self.prioritized_addresses)
 724
 725         inputs = []
 726         coins = prioritized_coins + coins
 727
 728         for item in coins:
 729             addr = item.get('address')
 730             v = item.get('value')
 731             total += v
 732             inputs.append( item )
 733             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
 734             if total >= amount + fee: break
 735         else:
 736             inputs = []
 737
 738         return inputs, total, fee
 739
 740
 741     def estimated_fee(self, inputs):
 742         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
 743         fee = self.fee * int(round(estimated_size/1024.))
 744         if fee == 0: fee = self.fee
 745         return fee
 746
 747
 748     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None, account=0 ):
 749         "add change to a transaction"
 750         change_amount = total - ( amount + fee )
 751         if change_amount != 0:
 752             if not change_addr:
 753                 if account is None:
 754                     # send change to one of the accounts involved in the tx
 755                     address = inputs[0].get('address')
 756                     account, _ = self.get_address_index(address)
 757
 758                 if not self.use_change or account == -1:
 759                     change_addr = inputs[-1]['address']
 760                 else:
 761                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
 762
 763             # Insert the change output at a random position in the outputs
 764             posn = random.randint(0, len(outputs))
 765             outputs[posn:posn] = [( change_addr,  change_amount)]
 766         return outputs
 767
 768
 769     def get_history(self, address):
 770         with self.lock:
 771             return self.history.get(address)
 772
 773
 774     def get_status(self, h):
 775         if not h: return None
 776         if h == ['*']: return '*'
 777         status = ''
 778         for tx_hash, height in h:
 779             status += tx_hash + ':%d:' % height
 780         return hashlib.sha256( status ).digest().encode('hex')
 781
 782
 783     def receive_tx_callback(self, tx_hash, tx, tx_height):
 784         if not self.check_new_tx(tx_hash, tx):
 785             # may happen due to pruning
 786             print_error("received transaction that is no longer referenced in history", tx_hash)
 787             return
 788
 789         with self.transaction_lock:
 790             self.transactions[tx_hash] = tx
 791
 792             self.interface.pending_transactions_for_notifications.append(tx)
 793
 794             self.save_transactions()
 795             if self.verifier and tx_height>0:
 796                 self.verifier.add(tx_hash, tx_height)
 797             self.update_tx_outputs(tx_hash)
 798
 799
 800     def save_transactions(self):
 801         tx = {}
 802         for k,v in self.transactions.items():
 803             tx[k] = str(v)
 804         self.config.set_key('transactions', tx, True)
 805
 806     def receive_history_callback(self, addr, hist):
 807
 808         if not self.check_new_history(addr, hist):
 809             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
 810
 811         with self.lock:
 812             self.history[addr] = hist
 813             self.config.set_key('addr_history', self.history, True)
 814
 815         if hist != ['*']:
 816             for tx_hash, tx_height in hist:
 817                 if tx_height>0:
 818                     # add it in case it was previously unconfirmed
 819                     if self.verifier: self.verifier.add(tx_hash, tx_height)
 820
 821
 822     def get_tx_history(self, account=None):
 823         with self.transaction_lock:
 824             history = self.transactions.items()
 825             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
 826             result = []
 827
 828             balance = 0
 829             for tx_hash, tx in history:
 830                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
 831                 if v is not None: balance += v
 832
 833             c, u = self.get_account_balance(account)
 834
 835             if balance != c+u:
 836                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
 837
 838             balance = c + u - balance
 839             for tx_hash, tx in history:
 840                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
 841                 if not is_relevant:
 842                     continue
 843                 if value is not None:
 844                     balance += value
 845
 846                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
 847                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
 848
 849         return result
 850
 851
 852     def get_label(self, tx_hash):
 853         label = self.labels.get(tx_hash)
 854         is_default = (label == '') or (label is None)
 855         if is_default: label = self.get_default_label(tx_hash)
 856         return label, is_default
 857
 858
 859     def get_default_label(self, tx_hash):
 860         tx = self.transactions.get(tx_hash)
 861         default_label = ''
 862         if tx:
 863             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
 864             if is_mine:
 865                 for o in tx.outputs:
 866                     o_addr, _ = o
 867                     if not self.is_mine(o_addr):
 868                         try:
 869                             default_label = self.labels[o_addr]
 870                         except KeyError:
 871                             default_label = o_addr
 872                         break
 873                 else:
 874                     default_label = '(internal)'
 875             else:
 876                 for o in tx.outputs:
 877                     o_addr, _ = o
 878                     if self.is_mine(o_addr) and not self.is_change(o_addr):
 879                         break
 880                 else:
 881                     for o in tx.outputs:
 882                         o_addr, _ = o
 883                         if self.is_mine(o_addr):
 884                             break
 885                     else:
 886                         o_addr = None
 887
 888                 if o_addr:
 889                     dest_label = self.labels.get(o_addr)
 890                     try:
 891                         default_label = self.labels[o_addr]
 892                     except KeyError:
 893                         default_label = o_addr
 894
 895         return default_label
 896
 897
 898     def mktx(self, outputs, password, fee=None, change_addr=None, account=None ):
 899         """
 900         create a transaction
 901         account parameter:
 902            None means use all accounts
 903            -1 means imported keys
 904            0, 1, etc are seed accounts
 905         """
 906
 907         for address, x in outputs:
 908             assert is_valid(address)
 909
 910         amount = sum( map(lambda x:x[1], outputs) )
 911
 912         inputs, total, fee = self.choose_tx_inputs( amount, fee, account )
 913         if not inputs:
 914             raise ValueError("Not enough funds")
 915
 916         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr, account)
 917
 918         tx = Transaction.from_io(inputs, outputs)
 919
 920
 921         keypairs = {}
 922         for i, txin in enumerate(tx.inputs):
 923             address = txin['address']
 924             if address in self.imported_keys.keys():
 925                 pk_addresses.append(address)
 926                 continue
 927             account, sequence = self.get_address_index(address)
 928             txin['KeyID'] = (account, 'BIP32', sequence) # used by the server to find the key
 929
 930             redeemScript = self.accounts[account].redeem_script(sequence)
 931             if redeemScript:
 932                 txin['redeemScript'] = redeemScript
 933                 assert address == self.accounts[account].get_address(*sequence)
 934             else:
 935                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
 936
 937             private_keys = self.get_private_key(address, password)
 938             for sec in private_keys:
 939                 compressed = is_compressed(sec)
 940                 pkey = regenerate_key(sec)
 941                 pubkey = GetPubKey(pkey.pubkey, compressed)
 942                 keypairs[ pubkey.encode('hex') ] = sec
 943
 944         tx.sign(keypairs)
 945
 946         for address, x in outputs:
 947             if address not in self.addressbook and not self.is_mine(address):
 948                 self.addressbook.append(address)
 949
 950         return tx
 951
 952
 953
 954     def sendtx(self, tx):
 955         # synchronous
 956         h = self.send_tx(tx)
 957         self.tx_event.wait()
 958         return self.receive_tx(h)
 959
 960     def send_tx(self, tx):
 961         # asynchronous
 962         self.tx_event.clear()
 963         self.interface.send([('blockchain.transaction.broadcast', [str(tx)])], 'synchronizer')
 964         return tx.hash()
 965
 966     def receive_tx(self,tx_hash):
 967         out = self.tx_result
 968         if out != tx_hash:
 969             return False, "error: " + out
 970         return True, out
 971
 972
 973
 974     def update_password(self, seed, old_password, new_password):
 975         if new_password == '': new_password = None
 976         # this will throw an exception if unicode cannot be converted
 977         self.seed = pw_encode( seed, new_password)
 978         self.config.set_key('seed', self.seed, True)
 979         self.use_encryption = (new_password != None)
 980         self.config.set_key('use_encryption', self.use_encryption,True)
 981         for k in self.imported_keys.keys():
 982             a = self.imported_keys[k]
 983             b = pw_decode(a, old_password)
 984             c = pw_encode(b, new_password)
 985             self.imported_keys[k] = c
 986         self.config.set_key('imported_keys', self.imported_keys, True)
 987
 988         for k, v in self.master_private_keys.items():
 989             b = pw_decode(v, old_password)
 990             c = pw_encode(b, new_password)
 991             self.master_private_keys[k] = c
 992         self.config.set_key('master_private_keys', self.master_private_keys, True)
 993
 994
 995     def freeze(self,addr):
 996         if self.is_mine(addr) and addr not in self.frozen_addresses:
 997             self.unprioritize(addr)
 998             self.frozen_addresses.append(addr)
 999             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
1000             return True
1001         else:
1002             return False
1003
1004     def unfreeze(self,addr):
1005         if self.is_mine(addr) and addr in self.frozen_addresses:
1006             self.frozen_addresses.remove(addr)
1007             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
1008             return True
1009         else:
1010             return False
1011
1012     def prioritize(self,addr):
1013         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1014             self.unfreeze(addr)
1015             self.prioritized_addresses.append(addr)
1016             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1017             return True
1018         else:
1019             return False
1020
1021     def unprioritize(self,addr):
1022         if self.is_mine(addr) and addr in self.prioritized_addresses:
1023             self.prioritized_addresses.remove(addr)
1024             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1025             return True
1026         else:
1027             return False
1028
1029     def set_fee(self, fee):
1030         if self.fee != fee:
1031             self.fee = fee
1032             self.config.set_key('fee_per_kb', self.fee, True)
1033
1034
1035     def save(self):
1036         print_error("Warning: wallet.save() is deprecated")
1037         tx = {}
1038         for k,v in self.transactions.items():
1039             tx[k] = str(v)
1040
1041         s = {
1042             'use_change': self.use_change,
1043             'fee_per_kb': self.fee,
1044             'addr_history': self.history,
1045             'labels': self.labels,
1046             'contacts': self.addressbook,
1047             'num_zeros': self.num_zeros,
1048             'frozen_addresses': self.frozen_addresses,
1049             'prioritized_addresses': self.prioritized_addresses,
1050             'gap_limit': self.gap_limit,
1051             'transactions': tx,
1052         }
1053         for k, v in s.items():
1054             self.config.set_key(k,v)
1055         self.config.save()
1056
1057     def set_verifier(self, verifier):
1058         self.verifier = verifier
1059
1060         # review transactions that are in the history
1061         for addr, hist in self.history.items():
1062             if hist == ['*']: continue
1063             for tx_hash, tx_height in hist:
1064                 if tx_height>0:
1065                     # add it in case it was previously unconfirmed
1066                     self.verifier.add(tx_hash, tx_height)
1067
1068
1069         # if we are on a pruning server, remove unverified transactions
1070         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1071         for tx_hash in self.transactions.keys():
1072             if tx_hash not in vr:
1073                 self.transactions.pop(tx_hash)
1074
1075
1076
1077     def check_new_history(self, addr, hist):
1078
1079         # check that all tx in hist are relevant
1080         if hist != ['*']:
1081             for tx_hash, height in hist:
1082                 tx = self.transactions.get(tx_hash)
1083                 if not tx: continue
1084                 if not tx.has_address(addr):
1085                     return False
1086
1087         # check that we are not "orphaning" a transaction
1088         old_hist = self.history.get(addr,[])
1089         if old_hist == ['*']: return True
1090
1091         for tx_hash, height in old_hist:
1092             if tx_hash in map(lambda x:x[0], hist): continue
1093             found = False
1094             for _addr, _hist in self.history.items():
1095                 if _addr == addr: continue
1096                 if _hist == ['*']: continue
1097                 _tx_hist = map(lambda x:x[0], _hist)
1098                 if tx_hash in _tx_hist:
1099                     found = True
1100                     break
1101
1102             if not found:
1103                 tx = self.transactions.get(tx_hash)
1104                 # tx might not be there
1105                 if not tx: continue
1106
1107                 # already verified?
1108                 if self.verifier.get_height(tx_hash):
1109                     continue
1110                 # unconfirmed tx
1111                 print_error("new history is orphaning transaction:", tx_hash)
1112                 # check that all outputs are not mine, request histories
1113                 ext_requests = []
1114                 for _addr, _v in tx.outputs:
1115                     # assert not self.is_mine(_addr)
1116                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1117
1118                 ext_h = self.interface.synchronous_get(ext_requests)
1119                 print_error("sync:", ext_requests, ext_h)
1120                 height = None
1121                 for h in ext_h:
1122                     if h == ['*']: continue
1123                     for item in h:
1124                         if item.get('tx_hash') == tx_hash:
1125                             height = item.get('height')
1126                 if height:
1127                     print_error("found height for", tx_hash, height)
1128                     self.verifier.add(tx_hash, height)
1129                 else:
1130                     print_error("removing orphaned tx from history", tx_hash)
1131                     self.transactions.pop(tx_hash)
1132
1133         return True
1134
1135
1136
1137     def check_new_tx(self, tx_hash, tx):
1138         # 1 check that tx is referenced in addr_history.
1139         addresses = []
1140         for addr, hist in self.history.items():
1141             if hist == ['*']:continue
1142             for txh, height in hist:
1143                 if txh == tx_hash:
1144                     addresses.append(addr)
1145
1146         if not addresses:
1147             return False
1148
1149         # 2 check that referencing addresses are in the tx
1150         for addr in addresses:
1151             if not tx.has_address(addr):
1152                 return False
1153
1154         return True
1155
1156
1157
1158
1159 class WalletSynchronizer(threading.Thread):
1160
1161
1162     def __init__(self, wallet, config):
1163         threading.Thread.__init__(self)
1164         self.daemon = True
1165         self.wallet = wallet
1166         wallet.synchronizer = self
1167         self.interface = self.wallet.interface
1168         self.interface.register_channel('synchronizer')
1169         self.wallet.interface.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1170         self.was_updated = True
1171         self.running = False
1172         self.lock = threading.Lock()
1173
1174     def stop(self):
1175         with self.lock: self.running = False
1176         self.interface.poke('synchronizer')
1177
1178     def is_running(self):
1179         with self.lock: return self.running
1180
1181
1182     def subscribe_to_addresses(self, addresses):
1183         messages = []
1184         for addr in addresses:
1185             messages.append(('blockchain.address.subscribe', [addr]))
1186         self.interface.send( messages, 'synchronizer')
1187
1188
1189     def run(self):
1190         with self.lock: self.running = True
1191
1192         requested_tx = []
1193         missing_tx = []
1194         requested_histories = {}
1195
1196         # request any missing transactions
1197         for history in self.wallet.history.values():
1198             if history == ['*']: continue
1199             for tx_hash, tx_height in history:
1200                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1201                     missing_tx.append( (tx_hash, tx_height) )
1202         print_error("missing tx", missing_tx)
1203
1204         # wait until we are connected, in case the user is not connected
1205         while not self.interface.is_connected:
1206             time.sleep(1)
1207
1208         # subscriptions
1209         self.subscribe_to_addresses(self.wallet.addresses(True))
1210
1211         while self.is_running():
1212             # 1. create new addresses
1213             new_addresses = self.wallet.synchronize()
1214
1215             # request missing addresses
1216             if new_addresses:
1217                 self.subscribe_to_addresses(new_addresses)
1218
1219             # request missing transactions
1220             for tx_hash, tx_height in missing_tx:
1221                 if (tx_hash, tx_height) not in requested_tx:
1222                     self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
1223                     requested_tx.append( (tx_hash, tx_height) )
1224             missing_tx = []
1225
1226             # detect if situation has changed
1227             if not self.interface.is_up_to_date('synchronizer'):
1228                 if self.wallet.is_up_to_date():
1229                     self.wallet.set_up_to_date(False)
1230                     self.was_updated = True
1231             else:
1232                 if not self.wallet.is_up_to_date():
1233                     self.wallet.set_up_to_date(True)
1234                     self.was_updated = True
1235
1236             if self.was_updated:
1237                 self.interface.trigger_callback('updated')
1238                 self.was_updated = False
1239
1240             # 2. get a response
1241             r = self.interface.get_response('synchronizer')
1242
1243             # poke sends None. (needed during stop)
1244             if not r: continue
1245
1246             # 3. handle response
1247             method = r['method']
1248             params = r['params']
1249             result = r.get('result')
1250             error = r.get('error')
1251             if error:
1252                 print "error", r
1253                 continue
1254
1255             if method == 'blockchain.address.subscribe':
1256                 addr = params[0]
1257                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1258                     if requested_histories.get(addr) is None:
1259                         self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
1260                         requested_histories[addr] = result
1261
1262             elif method == 'blockchain.address.get_history':
1263                 addr = params[0]
1264                 print_error("receiving history", addr, result)
1265                 if result == ['*']:
1266                     assert requested_histories.pop(addr) == '*'
1267                     self.wallet.receive_history_callback(addr, result)
1268                 else:
1269                     hist = []
1270                     # check that txids are unique
1271                     txids = []
1272                     for item in result:
1273                         tx_hash = item['tx_hash']
1274                         if tx_hash not in txids:
1275                             txids.append(tx_hash)
1276                             hist.append( (tx_hash, item['height']) )
1277
1278                     if len(hist) != len(result):
1279                         raise BaseException("error: server sent history with non-unique txid", result)
1280
1281                     # check that the status corresponds to what was announced
1282                     rs = requested_histories.pop(addr)
1283                     if self.wallet.get_status(hist) != rs:
1284                         raise BaseException("error: status mismatch: %s"%addr)
1285
1286                     # store received history
1287                     self.wallet.receive_history_callback(addr, hist)
1288
1289                     # request transactions that we don't have
1290                     for tx_hash, tx_height in hist:
1291                         if self.wallet.transactions.get(tx_hash) is None:
1292                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1293                                 missing_tx.append( (tx_hash, tx_height) )
1294
1295             elif method == 'blockchain.transaction.get':
1296                 tx_hash = params[0]
1297                 tx_height = params[1]
1298                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1299                 tx = Transaction(result)
1300                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1301                 self.was_updated = True
1302                 requested_tx.remove( (tx_hash, tx_height) )
1303                 print_error("received tx:", tx_hash, len(tx.raw))
1304
1305             elif method == 'blockchain.transaction.broadcast':
1306                 self.wallet.tx_result = result
1307                 self.wallet.tx_event.set()
1308
1309             else:
1310                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1311
1312             if self.was_updated and not requested_tx:
1313                 self.interface.trigger_callback('updated')
1314                 self.interface.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1315
1316
1317                 self.was_updated = False