lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, user_dir, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36
  37 # AES encryption
  38 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  39 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  40
  41 def pw_encode(s, password):
  42     if password:
  43         secret = Hash(password)
  44         return EncodeAES(secret, s)
  45     else:
  46         return s
  47
  48 def pw_decode(s, password):
  49     if password is not None:
  50         secret = Hash(password)
  51         try:
  52             d = DecodeAES(secret, s)
  53         except:
  54             raise BaseException('Invalid password')
  55         return d
  56     else:
  57         return s
  58
  59
  60
  61
  62
  63 from version import ELECTRUM_VERSION, SEED_VERSION
  64
  65
  66 class Wallet:
  67     def __init__(self, config={}):
  68
  69         self.config = config
  70         self.electrum_version = ELECTRUM_VERSION
  71         self.gap_limit_for_change = 3 # constant
  72
  73         # saved fields
  74         self.seed_version          = config.get('seed_version', SEED_VERSION)
  75         self.gap_limit             = config.get('gap_limit', 5)
  76         self.use_change            = config.get('use_change',True)
  77         self.fee                   = int(config.get('fee_per_kb',50000))
  78         self.num_zeros             = int(config.get('num_zeros',0))
  79         self.use_encryption        = config.get('use_encryption', False)
  80         self.seed                  = config.get('seed', '')               # encrypted
  81         self.labels                = config.get('labels', {})
  82         self.frozen_addresses      = config.get('frozen_addresses',[])
  83         self.prioritized_addresses = config.get('prioritized_addresses',[])
  84         self.addressbook           = config.get('contacts', [])
  85
  86         self.imported_keys         = config.get('imported_keys',{})
  87         self.history               = config.get('addr_history',{})        # address -> list(txid, height)
  88
  89
  90         self.master_public_keys = config.get('master_public_keys',{})
  91         self.master_private_keys = config.get('master_private_keys', {})
  92
  93         self.load_accounts(config)
  94
  95         self.transactions = {}
  96         tx = config.get('transactions',{})
  97         try:
  98             for k,v in tx.items(): self.transactions[k] = Transaction(v)
  99         except:
 100             print_msg("Warning: Cannot deserialize transactions. skipping")
 101
 102         # not saved
 103         self.prevout_values = {}     # my own transaction outputs
 104         self.spent_outputs = []
 105
 106         # spv
 107         self.verifier = None
 108
 109         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 110         # interface.is_up_to_date() returns true when all requests have been answered and processed
 111         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 112
 113         self.up_to_date = False
 114         self.lock = threading.Lock()
 115         self.transaction_lock = threading.Lock()
 116         self.tx_event = threading.Event()
 117
 118         if self.seed_version != SEED_VERSION:
 119             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
 120
 121         for tx_hash, tx in self.transactions.items():
 122             if self.check_new_tx(tx_hash, tx):
 123                 self.update_tx_outputs(tx_hash)
 124             else:
 125                 print_error("unreferenced tx", tx_hash)
 126                 self.transactions.pop(tx_hash)
 127
 128
 129     def set_up_to_date(self,b):
 130         with self.lock: self.up_to_date = b
 131
 132     def is_up_to_date(self):
 133         with self.lock: return self.up_to_date
 134
 135     def update(self):
 136         self.up_to_date = False
 137         self.interface.poke('synchronizer')
 138         while not self.is_up_to_date(): time.sleep(0.1)
 139
 140     def import_key(self, sec, password):
 141         # check password
 142         seed = self.decode_seed(password)
 143         try:
 144             address = address_from_private_key(sec)
 145         except:
 146             raise BaseException('Invalid private key')
 147
 148         if self.is_mine(address):
 149             raise BaseException('Address already in wallet')
 150
 151         # store the originally requested keypair into the imported keys table
 152         self.imported_keys[address] = pw_encode(sec, password )
 153         self.config.set_key('imported_keys', self.imported_keys, True)
 154         return address
 155
 156     def delete_imported_key(self, addr):
 157         if addr in self.imported_keys:
 158             self.imported_keys.pop(addr)
 159             self.config.set_key('imported_keys', self.imported_keys, True)
 160
 161
 162     def init_seed(self, seed):
 163         if self.seed: raise BaseException("a seed exists")
 164         if not seed:
 165             seed = random_seed(128)
 166         self.seed = seed
 167
 168
 169     def save_seed(self):
 170         self.config.set_key('seed', self.seed, True)
 171         self.config.set_key('seed_version', self.seed_version, True)
 172
 173         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 174
 175         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 176         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 177         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 178
 179         self.master_public_keys = {
 180             "m/0'/": (c0, K0, cK0),
 181             "m/1'/": (c1, K1, cK1),
 182             "m/2'/": (c2, K2, cK2)
 183             }
 184
 185         self.master_private_keys = {
 186             "m/0'/": k0,
 187             "m/1'/": k1
 188             }
 189         # send k2 to service
 190
 191         self.config.set_key('master_public_keys', self.master_public_keys, True)
 192         self.config.set_key('master_private_keys', self.master_private_keys, True)
 193
 194         # create default account
 195         self.create_new_account('Main account')
 196
 197
 198     def create_new_account(self, name):
 199         keys = self.accounts.keys()
 200         i = 0
 201
 202         while True:
 203             derivation = "m/0'/%d'"%i
 204             if derivation not in keys: break
 205             i += 1
 206
 207         start = "m/0'/"
 208         master_c, master_K, master_cK = self.master_public_keys[start]
 209         master_k = self.master_private_keys[start] # needs decryption
 210         k, c, K, cK = bip32_private_derivation(master_k, master_c, start, derivation) # this is a type 1 derivation
 211
 212         self.accounts[derivation] = BIP32_Account({ 'name':name, 'c':c, 'K':K, 'cK':cK })
 213         self.save_accounts()
 214
 215     def create_p2sh_account(self, name):
 216         keys = self.accounts.keys()
 217         i = 0
 218         while True:
 219             account_id = "m/1'/%d & m/2'/%d"%(i,i)
 220             if account_id not in keys: break
 221             i += 1
 222
 223         master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 224         c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 225
 226         master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 227         c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 228
 229         self.accounts[account_id] = BIP32_Account_2of2({ 'name':name, 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 230         self.save_accounts()
 231
 232
 233     def save_accounts(self):
 234         d = {}
 235         for k, v in self.accounts.items():
 236             d[k] = v.dump()
 237         self.config.set_key('accounts', d, True)
 238
 239
 240     def load_accounts(self, config):
 241         d = config.get('accounts', {})
 242         self.accounts = {}
 243         for k, v in d.items():
 244             if '&' in k:
 245                 self.accounts[k] = BIP32_Account_2of2(v)
 246             else:
 247                 self.accounts[k] = BIP32_Account(v)
 248
 249
 250
 251
 252     def addresses(self, include_change = True):
 253         o = self.get_account_addresses(-1, include_change)
 254         for a in self.accounts.keys():
 255             o += self.get_account_addresses(a, include_change)
 256         return o
 257
 258
 259     def is_mine(self, address):
 260         return address in self.addresses(True)
 261
 262     def is_change(self, address):
 263         if not self.is_mine(address): return False
 264         if address in self.imported_keys.keys(): return False
 265         acct, s = self.get_address_index(address)
 266         return s[0] == 1
 267
 268     def get_master_public_key(self):
 269         raise
 270         return self.config.get("master_public_key")
 271
 272     def get_address_index(self, address):
 273         if address in self.imported_keys.keys():
 274             return -1, None
 275         for account in self.accounts.keys():
 276             for for_change in [0,1]:
 277                 addresses = self.accounts[account].get_addresses(for_change)
 278                 for addr in addresses:
 279                     if address == addr:
 280                         return account, (for_change, addresses.index(addr))
 281         raise BaseException("not found")
 282
 283
 284     def get_public_key(self, address):
 285         account, sequence = self.get_address_index(address)
 286         return self.accounts[account].get_pubkey( sequence )
 287
 288
 289     def decode_seed(self, password):
 290         seed = pw_decode(self.seed, password)
 291         #todo:  #self.sequences[0].check_seed(seed)
 292         return seed
 293
 294     def get_private_key(self, address, password):
 295         return self.get_private_keys([address], password).get(address)
 296
 297     def get_private_keys(self, addresses, password):
 298         if not self.seed: return {}
 299         # decode seed in any case, in order to test the password
 300         seed = self.decode_seed(password)
 301         out = {}
 302         for address in addresses:
 303             if address in self.imported_keys.keys():
 304                 out[address] = pw_decode( self.imported_keys[address], password )
 305             else:
 306                 account, sequence = self.get_address_index(address)
 307                 print_error( "found index", address, account, sequence)
 308
 309                 m = re.match("m/0'/(\d+)'", account)
 310                 if m:
 311                     num = int(m.group(1))
 312                     master_k = self.master_private_keys["m/0'/"]
 313                     master_c, _, _ = self.master_public_keys["m/0'/"]
 314                     master_k, master_c = CKD(master_k, master_c, num + BIP32_PRIME)
 315                     pk = self.accounts[account].get_private_key(sequence, master_k)
 316                     out[address] = pk
 317
 318                 m2 = re.match("m/1'/(\d+) & m/2'/(\d+)", account)
 319                 if m2:
 320                     num = int(m2.group(1))
 321                     master_k = self.master_private_keys["m/1'/"]
 322                     master_c, master_K, _ = self.master_public_keys["m/1'/"]
 323                     master_k, master_c = CKD(master_k.decode('hex'), master_c.decode('hex'), num)
 324                     pk = self.accounts[account].get_private_key(sequence, master_k)
 325                     out[address] = pk
 326
 327         return out
 328
 329
 330     def signrawtransaction(self, tx, input_info, private_keys, password):
 331         unspent_coins = self.get_unspent_coins()
 332         seed = self.decode_seed(password)
 333
 334         # convert private_keys to dict
 335         pk = {}
 336         for sec in private_keys:
 337             address = address_from_private_key(sec)
 338             pk[address] = sec
 339         private_keys = pk
 340
 341         for txin in tx.inputs:
 342             # convert to own format
 343             txin['tx_hash'] = txin['prevout_hash']
 344             txin['index'] = txin['prevout_n']
 345
 346             for item in input_info:
 347                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 348                     txin['raw_output_script'] = item['scriptPubKey']
 349                     txin['redeemScript'] = item.get('redeemScript')
 350                     txin['KeyID'] = item.get('KeyID')
 351                     break
 352             else:
 353                 for item in unspent_coins:
 354                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 355                         txin['raw_output_script'] = item['raw_output_script']
 356                         break
 357                 else:
 358                     # if neither, we might want to get it from the server..
 359                     raise
 360
 361             # find the address:
 362             if txin.get('KeyID'):
 363                 account, name, sequence = txin.get('KeyID')
 364                 if name != 'Electrum': continue
 365                 sec = self.accounts[account].get_private_key(sequence, seed)
 366                 addr = self.accounts[account].get_address(sequence)
 367                 txin['address'] = addr
 368                 private_keys[addr] = sec
 369
 370             elif txin.get("redeemScript"):
 371                 txin['address'] = hash_160_to_bc_address(hash_160(txin.get("redeemScript").decode('hex')), 5)
 372
 373             elif txin.get("raw_output_script"):
 374                 import deserialize
 375                 addr = deserialize.get_address_from_output_script(txin.get("raw_output_script").decode('hex'))
 376                 sec = self.get_private_key(addr, password)
 377                 if sec:
 378                     private_keys[addr] = sec
 379                     txin['address'] = addr
 380
 381         tx.sign( private_keys )
 382
 383     def sign_message(self, address, message, password):
 384         sec = self.get_private_key(address, password)
 385         key = regenerate_key(sec)
 386         compressed = is_compressed(sec)
 387         return key.sign_message(message, compressed, address)
 388
 389     def verify_message(self, address, signature, message):
 390         try:
 391             EC_KEY.verify_message(address, signature, message)
 392             return True
 393         except BaseException as e:
 394             print_error("Verification error: {0}".format(e))
 395             return False
 396
 397
 398     def change_gap_limit(self, value):
 399         if value >= self.gap_limit:
 400             self.gap_limit = value
 401             self.config.set_key('gap_limit', self.gap_limit, True)
 402             self.interface.poke('synchronizer')
 403             return True
 404
 405         elif value >= self.min_acceptable_gap():
 406             for key, account in self.accounts.items():
 407                 addresses = account[0]
 408                 k = self.num_unused_trailing_addresses(addresses)
 409                 n = len(addresses) - k + value
 410                 addresses = addresses[0:n]
 411                 self.accounts[key][0] = addresses
 412
 413             self.gap_limit = value
 414             self.config.set_key('gap_limit', self.gap_limit, True)
 415             self.save_accounts()
 416             return True
 417         else:
 418             return False
 419
 420     def num_unused_trailing_addresses(self, addresses):
 421         k = 0
 422         for a in addresses[::-1]:
 423             if self.history.get(a):break
 424             k = k + 1
 425         return k
 426
 427     def min_acceptable_gap(self):
 428         # fixme: this assumes wallet is synchronized
 429         n = 0
 430         nmax = 0
 431
 432         for account in self.accounts.values():
 433             addresses = account.get_addresses(0)
 434             k = self.num_unused_trailing_addresses(addresses)
 435             for a in addresses[0:-k]:
 436                 if self.history.get(a):
 437                     n = 0
 438                 else:
 439                     n += 1
 440                     if n > nmax: nmax = n
 441         return nmax + 1
 442
 443
 444     def address_is_old(self, address):
 445         age = -1
 446         h = self.history.get(address, [])
 447         if h == ['*']:
 448             return True
 449         for tx_hash, tx_height in h:
 450             if tx_height == 0:
 451                 tx_age = 0
 452             else:
 453                 tx_age = self.verifier.height - tx_height + 1
 454             if tx_age > age:
 455                 age = tx_age
 456         return age > 2
 457
 458
 459     def synchronize_sequence(self, account, for_change):
 460         limit = self.gap_limit_for_change if for_change else self.gap_limit
 461         new_addresses = []
 462         while True:
 463             addresses = account.get_addresses(for_change)
 464             if len(addresses) < limit:
 465                 address = account.create_new_address(for_change)
 466                 self.history[address] = []
 467                 new_addresses.append( address )
 468                 continue
 469
 470             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 471                 break
 472             else:
 473                 address = account.create_new_address(for_change)
 474                 self.history[address] = []
 475                 new_addresses.append( address )
 476
 477         return new_addresses
 478
 479
 480     def synchronize_account(self, account):
 481         new = []
 482         new += self.synchronize_sequence(account, 0)
 483         new += self.synchronize_sequence(account, 1)
 484         return new
 485
 486     def synchronize(self):
 487         new = []
 488         for account in self.accounts.values():
 489             new += self.synchronize_account(account)
 490         if new:
 491             self.save_accounts()
 492             self.config.set_key('addr_history', self.history, True)
 493         return new
 494
 495
 496     def is_found(self):
 497         return self.history.values() != [[]] * len(self.history)
 498
 499
 500     def add_contact(self, address, label=None):
 501         self.addressbook.append(address)
 502         self.config.set_key('contacts', self.addressbook, True)
 503         if label:
 504             self.labels[address] = label
 505             self.config.set_key('labels', self.labels)
 506
 507     def delete_contact(self, addr):
 508         if addr in self.addressbook:
 509             self.addressbook.remove(addr)
 510             self.config.set_key('addressbook', self.addressbook, True)
 511
 512
 513     def fill_addressbook(self):
 514         for tx_hash, tx in self.transactions.items():
 515             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 516             if is_send:
 517                 for addr, v in tx.outputs:
 518                     if not self.is_mine(addr) and addr not in self.addressbook:
 519                         self.addressbook.append(addr)
 520         # redo labels
 521         # self.update_tx_labels()
 522
 523     def get_num_tx(self, address):
 524         n = 0
 525         for tx in self.transactions.values():
 526             if address in map(lambda x:x[0], tx.outputs): n += 1
 527         return n
 528
 529
 530     def get_address_flags(self, addr):
 531         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 532         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 533         return flags
 534
 535
 536     def get_tx_value(self, tx, account=None):
 537         domain = self.get_account_addresses(account)
 538         return tx.get_value(domain, self.prevout_values)
 539
 540
 541     def update_tx_outputs(self, tx_hash):
 542         tx = self.transactions.get(tx_hash)
 543
 544         for i, (addr, value) in enumerate(tx.outputs):
 545             key = tx_hash+ ':%d'%i
 546             self.prevout_values[key] = value
 547
 548         for item in tx.inputs:
 549             if self.is_mine(item.get('address')):
 550                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 551                 self.spent_outputs.append(key)
 552
 553
 554     def get_addr_balance(self, address):
 555         assert self.is_mine(address)
 556         h = self.history.get(address,[])
 557         if h == ['*']: return 0,0
 558         c = u = 0
 559         received_coins = []   # list of coins received at address
 560
 561         for tx_hash, tx_height in h:
 562             tx = self.transactions.get(tx_hash)
 563             if not tx: continue
 564
 565             for i, (addr, value) in enumerate(tx.outputs):
 566                 if addr == address:
 567                     key = tx_hash + ':%d'%i
 568                     received_coins.append(key)
 569
 570         for tx_hash, tx_height in h:
 571             tx = self.transactions.get(tx_hash)
 572             if not tx: continue
 573             v = 0
 574
 575             for item in tx.inputs:
 576                 addr = item.get('address')
 577                 if addr == address:
 578                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 579                     value = self.prevout_values.get( key )
 580                     if key in received_coins:
 581                         v -= value
 582
 583             for i, (addr, value) in enumerate(tx.outputs):
 584                 key = tx_hash + ':%d'%i
 585                 if addr == address:
 586                     v += value
 587
 588             if tx_height:
 589                 c += v
 590             else:
 591                 u += v
 592         return c, u
 593
 594
 595     def get_accounts(self):
 596         accounts = {}
 597         for k, account in self.accounts.items():
 598             accounts[k] = account.name
 599         if self.imported_keys:
 600             accounts[-1] = 'Imported keys'
 601         return accounts
 602
 603     def get_account_addresses(self, a, include_change=True):
 604         if a is None:
 605             o = self.addresses(True)
 606         elif a == -1:
 607             o = self.imported_keys.keys()
 608         else:
 609             ac = self.accounts[a]
 610             o = ac.get_addresses(0)
 611             if include_change: o += ac.get_addresses(1)
 612         return o
 613
 614     def get_imported_balance(self):
 615         cc = uu = 0
 616         for addr in self.imported_keys.keys():
 617             c, u = self.get_addr_balance(addr)
 618             cc += c
 619             uu += u
 620         return cc, uu
 621
 622     def get_account_balance(self, account):
 623         if account is None:
 624             return self.get_balance()
 625         elif account == -1:
 626             return self.get_imported_balance()
 627
 628         conf = unconf = 0
 629         for addr in self.get_account_addresses(account):
 630             c, u = self.get_addr_balance(addr)
 631             conf += c
 632             unconf += u
 633         return conf, unconf
 634
 635     def get_frozen_balance(self):
 636         conf = unconf = 0
 637         for addr in self.frozen_addresses:
 638             c, u = self.get_addr_balance(addr)
 639             conf += c
 640             unconf += u
 641         return conf, unconf
 642
 643
 644     def get_balance(self):
 645         cc = uu = 0
 646         for a in self.accounts.keys():
 647             c, u = self.get_account_balance(a)
 648             cc += c
 649             uu += u
 650         c, u = self.get_imported_balance()
 651         cc += c
 652         uu += u
 653         return cc, uu
 654
 655
 656     def get_unspent_coins(self, domain=None):
 657         coins = []
 658         if domain is None: domain = self.addresses(True)
 659         for addr in domain:
 660             h = self.history.get(addr, [])
 661             if h == ['*']: continue
 662             for tx_hash, tx_height in h:
 663                 tx = self.transactions.get(tx_hash)
 664                 if tx is None: raise BaseException("Wallet not synchronized")
 665                 for output in tx.d.get('outputs'):
 666                     if output.get('address') != addr: continue
 667                     key = tx_hash + ":%d" % output.get('index')
 668                     if key in self.spent_outputs: continue
 669                     output['tx_hash'] = tx_hash
 670                     coins.append(output)
 671         return coins
 672
 673
 674
 675     def choose_tx_inputs( self, amount, fixed_fee, account = None ):
 676         """ todo: minimize tx size """
 677         total = 0
 678         fee = self.fee if fixed_fee is None else fixed_fee
 679         domain = self.get_account_addresses(account)
 680         coins = []
 681         prioritized_coins = []
 682         for i in self.frozen_addresses:
 683             if i in domain: domain.remove(i)
 684
 685         for i in self.prioritized_addresses:
 686             if i in domain: domain.remove(i)
 687
 688         coins = self.get_unspent_coins(domain)
 689         prioritized_coins = self.get_unspent_coins(self.prioritized_addresses)
 690
 691         inputs = []
 692         coins = prioritized_coins + coins
 693
 694         for item in coins:
 695             addr = item.get('address')
 696             v = item.get('value')
 697             total += v
 698             inputs.append( item )
 699             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
 700             if total >= amount + fee: break
 701         else:
 702             inputs = []
 703
 704         return inputs, total, fee
 705
 706
 707     def estimated_fee(self, inputs):
 708         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
 709         fee = self.fee * int(round(estimated_size/1024.))
 710         if fee == 0: fee = self.fee
 711         return fee
 712
 713
 714     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None, account=0 ):
 715         "add change to a transaction"
 716         change_amount = total - ( amount + fee )
 717         if change_amount != 0:
 718             if not change_addr:
 719                 if account is None:
 720                     # send change to one of the accounts involved in the tx
 721                     address = inputs[0].get('address')
 722                     account, _ = self.get_address_index(address)
 723
 724                 if not self.use_change or account == -1:
 725                     change_addr = inputs[-1]['address']
 726                 else:
 727                     change_addr = self.accounts[account][1][-self.gap_limit_for_change]
 728
 729             # Insert the change output at a random position in the outputs
 730             posn = random.randint(0, len(outputs))
 731             outputs[posn:posn] = [( change_addr,  change_amount)]
 732         return outputs
 733
 734
 735     def get_history(self, address):
 736         with self.lock:
 737             return self.history.get(address)
 738
 739
 740     def get_status(self, h):
 741         if not h: return None
 742         if h == ['*']: return '*'
 743         status = ''
 744         for tx_hash, height in h:
 745             status += tx_hash + ':%d:' % height
 746         return hashlib.sha256( status ).digest().encode('hex')
 747
 748
 749     def receive_tx_callback(self, tx_hash, tx, tx_height):
 750         if not self.check_new_tx(tx_hash, tx):
 751             # may happen due to pruning
 752             print_error("received transaction that is no longer referenced in history", tx_hash)
 753             return
 754
 755         with self.transaction_lock:
 756             self.transactions[tx_hash] = tx
 757
 758             self.interface.pending_transactions_for_notifications.append(tx)
 759
 760             self.save_transactions()
 761             if self.verifier and tx_height>0:
 762                 self.verifier.add(tx_hash, tx_height)
 763             self.update_tx_outputs(tx_hash)
 764
 765
 766     def save_transactions(self):
 767         tx = {}
 768         for k,v in self.transactions.items():
 769             tx[k] = str(v)
 770         self.config.set_key('transactions', tx, True)
 771
 772     def receive_history_callback(self, addr, hist):
 773
 774         if not self.check_new_history(addr, hist):
 775             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
 776
 777         with self.lock:
 778             self.history[addr] = hist
 779             self.config.set_key('addr_history', self.history, True)
 780
 781         if hist != ['*']:
 782             for tx_hash, tx_height in hist:
 783                 if tx_height>0:
 784                     # add it in case it was previously unconfirmed
 785                     if self.verifier: self.verifier.add(tx_hash, tx_height)
 786
 787
 788     def get_tx_history(self, account=None):
 789         with self.transaction_lock:
 790             history = self.transactions.items()
 791             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
 792             result = []
 793
 794             balance = 0
 795             for tx_hash, tx in history:
 796                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
 797                 if v is not None: balance += v
 798
 799             c, u = self.get_account_balance(account)
 800
 801             if balance != c+u:
 802                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
 803
 804             balance = c + u - balance
 805             for tx_hash, tx in history:
 806                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
 807                 if not is_relevant:
 808                     continue
 809                 if value is not None:
 810                     balance += value
 811
 812                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
 813                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
 814
 815         return result
 816
 817
 818     def get_label(self, tx_hash):
 819         label = self.labels.get(tx_hash)
 820         is_default = (label == '') or (label is None)
 821         if is_default: label = self.get_default_label(tx_hash)
 822         return label, is_default
 823
 824
 825     def get_default_label(self, tx_hash):
 826         tx = self.transactions.get(tx_hash)
 827         default_label = ''
 828         if tx:
 829             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
 830             if is_mine:
 831                 for o in tx.outputs:
 832                     o_addr, _ = o
 833                     if not self.is_mine(o_addr):
 834                         try:
 835                             default_label = self.labels[o_addr]
 836                         except KeyError:
 837                             default_label = o_addr
 838                         break
 839                 else:
 840                     default_label = '(internal)'
 841             else:
 842                 for o in tx.outputs:
 843                     o_addr, _ = o
 844                     if self.is_mine(o_addr) and not self.is_change(o_addr):
 845                         break
 846                 else:
 847                     for o in tx.outputs:
 848                         o_addr, _ = o
 849                         if self.is_mine(o_addr):
 850                             break
 851                     else:
 852                         o_addr = None
 853
 854                 if o_addr:
 855                     dest_label = self.labels.get(o_addr)
 856                     try:
 857                         default_label = self.labels[o_addr]
 858                     except KeyError:
 859                         default_label = o_addr
 860
 861         return default_label
 862
 863
 864     def mktx(self, outputs, password, fee=None, change_addr=None, account=None ):
 865         """
 866         create a transaction
 867         account parameter:
 868            None means use all accounts
 869            -1 means imported keys
 870            0, 1, etc are seed accounts
 871         """
 872
 873         for address, x in outputs:
 874             assert is_valid(address)
 875
 876         amount = sum( map(lambda x:x[1], outputs) )
 877
 878         inputs, total, fee = self.choose_tx_inputs( amount, fee, account )
 879         if not inputs:
 880             raise ValueError("Not enough funds")
 881
 882         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr, account)
 883
 884         tx = Transaction.from_io(inputs, outputs)
 885
 886         pk_addresses = []
 887         for i in range(len(tx.inputs)):
 888             txin = tx.inputs[i]
 889             address = txin['address']
 890             if address in self.imported_keys.keys():
 891                 pk_addresses.append(address)
 892                 continue
 893             account, sequence = self.get_address_index(address)
 894
 895             txin['KeyID'] = (account, 'BIP32', sequence) # used by the server to find the key
 896
 897             _, redeemScript = self.accounts[account].get_input_info(sequence)
 898
 899             if redeemScript: txin['redeemScript'] = redeemScript
 900             pk_addresses.append(address)
 901
 902         print "pk_addresses", pk_addresses
 903
 904         # get all private keys at once.
 905         if self.seed:
 906             private_keys = self.get_private_keys(pk_addresses, password)
 907             print "private keys", private_keys
 908             tx.sign(private_keys)
 909
 910         for address, x in outputs:
 911             if address not in self.addressbook and not self.is_mine(address):
 912                 self.addressbook.append(address)
 913
 914         return tx
 915
 916
 917
 918     def sendtx(self, tx):
 919         # synchronous
 920         h = self.send_tx(tx)
 921         self.tx_event.wait()
 922         return self.receive_tx(h)
 923
 924     def send_tx(self, tx):
 925         # asynchronous
 926         self.tx_event.clear()
 927         self.interface.send([('blockchain.transaction.broadcast', [str(tx)])], 'synchronizer')
 928         return tx.hash()
 929
 930     def receive_tx(self,tx_hash):
 931         out = self.tx_result
 932         if out != tx_hash:
 933             return False, "error: " + out
 934         return True, out
 935
 936
 937
 938     def update_password(self, seed, old_password, new_password):
 939         if new_password == '': new_password = None
 940         # this will throw an exception if unicode cannot be converted
 941         self.seed = pw_encode( seed, new_password)
 942         self.config.set_key('seed', self.seed, True)
 943         self.use_encryption = (new_password != None)
 944         self.config.set_key('use_encryption', self.use_encryption,True)
 945         for k in self.imported_keys.keys():
 946             a = self.imported_keys[k]
 947             b = pw_decode(a, old_password)
 948             c = pw_encode(b, new_password)
 949             self.imported_keys[k] = c
 950         self.config.set_key('imported_keys', self.imported_keys, True)
 951
 952
 953     def freeze(self,addr):
 954         if self.is_mine(addr) and addr not in self.frozen_addresses:
 955             self.unprioritize(addr)
 956             self.frozen_addresses.append(addr)
 957             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 958             return True
 959         else:
 960             return False
 961
 962     def unfreeze(self,addr):
 963         if self.is_mine(addr) and addr in self.frozen_addresses:
 964             self.frozen_addresses.remove(addr)
 965             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 966             return True
 967         else:
 968             return False
 969
 970     def prioritize(self,addr):
 971         if self.is_mine(addr) and addr not in self.prioritized_addresses:
 972             self.unfreeze(addr)
 973             self.prioritized_addresses.append(addr)
 974             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
 975             return True
 976         else:
 977             return False
 978
 979     def unprioritize(self,addr):
 980         if self.is_mine(addr) and addr in self.prioritized_addresses:
 981             self.prioritized_addresses.remove(addr)
 982             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
 983             return True
 984         else:
 985             return False
 986
 987     def set_fee(self, fee):
 988         if self.fee != fee:
 989             self.fee = fee
 990             self.config.set_key('fee_per_kb', self.fee, True)
 991
 992
 993     def save(self):
 994         print_error("Warning: wallet.save() is deprecated")
 995         tx = {}
 996         for k,v in self.transactions.items():
 997             tx[k] = str(v)
 998
 999         s = {
1000             'use_change': self.use_change,
1001             'fee_per_kb': self.fee,
1002             'addr_history': self.history,
1003             'labels': self.labels,
1004             'contacts': self.addressbook,
1005             'num_zeros': self.num_zeros,
1006             'frozen_addresses': self.frozen_addresses,
1007             'prioritized_addresses': self.prioritized_addresses,
1008             'gap_limit': self.gap_limit,
1009             'transactions': tx,
1010         }
1011         for k, v in s.items():
1012             self.config.set_key(k,v)
1013         self.config.save()
1014
1015     def set_verifier(self, verifier):
1016         self.verifier = verifier
1017
1018         # review transactions that are in the history
1019         for addr, hist in self.history.items():
1020             if hist == ['*']: continue
1021             for tx_hash, tx_height in hist:
1022                 if tx_height>0:
1023                     # add it in case it was previously unconfirmed
1024                     self.verifier.add(tx_hash, tx_height)
1025
1026
1027         # if we are on a pruning server, remove unverified transactions
1028         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1029         for tx_hash in self.transactions.keys():
1030             if tx_hash not in vr:
1031                 self.transactions.pop(tx_hash)
1032
1033
1034
1035     def check_new_history(self, addr, hist):
1036
1037         # check that all tx in hist are relevant
1038         if hist != ['*']:
1039             for tx_hash, height in hist:
1040                 tx = self.transactions.get(tx_hash)
1041                 if not tx: continue
1042                 if not tx.has_address(addr):
1043                     return False
1044
1045         # check that we are not "orphaning" a transaction
1046         old_hist = self.history.get(addr,[])
1047         if old_hist == ['*']: return True
1048
1049         for tx_hash, height in old_hist:
1050             if tx_hash in map(lambda x:x[0], hist): continue
1051             found = False
1052             for _addr, _hist in self.history.items():
1053                 if _addr == addr: continue
1054                 if _hist == ['*']: continue
1055                 _tx_hist = map(lambda x:x[0], _hist)
1056                 if tx_hash in _tx_hist:
1057                     found = True
1058                     break
1059
1060             if not found:
1061                 tx = self.transactions.get(tx_hash)
1062                 # tx might not be there
1063                 if not tx: continue
1064
1065                 # already verified?
1066                 if self.verifier.get_height(tx_hash):
1067                     continue
1068                 # unconfirmed tx
1069                 print_error("new history is orphaning transaction:", tx_hash)
1070                 # check that all outputs are not mine, request histories
1071                 ext_requests = []
1072                 for _addr, _v in tx.outputs:
1073                     # assert not self.is_mine(_addr)
1074                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1075
1076                 ext_h = self.interface.synchronous_get(ext_requests)
1077                 print_error("sync:", ext_requests, ext_h)
1078                 height = None
1079                 for h in ext_h:
1080                     if h == ['*']: continue
1081                     for item in h:
1082                         if item.get('tx_hash') == tx_hash:
1083                             height = item.get('height')
1084                 if height:
1085                     print_error("found height for", tx_hash, height)
1086                     self.verifier.add(tx_hash, height)
1087                 else:
1088                     print_error("removing orphaned tx from history", tx_hash)
1089                     self.transactions.pop(tx_hash)
1090
1091         return True
1092
1093
1094
1095     def check_new_tx(self, tx_hash, tx):
1096         # 1 check that tx is referenced in addr_history.
1097         addresses = []
1098         for addr, hist in self.history.items():
1099             if hist == ['*']:continue
1100             for txh, height in hist:
1101                 if txh == tx_hash:
1102                     addresses.append(addr)
1103
1104         if not addresses:
1105             return False
1106
1107         # 2 check that referencing addresses are in the tx
1108         for addr in addresses:
1109             if not tx.has_address(addr):
1110                 return False
1111
1112         return True
1113
1114
1115
1116
1117 class WalletSynchronizer(threading.Thread):
1118
1119
1120     def __init__(self, wallet, config):
1121         threading.Thread.__init__(self)
1122         self.daemon = True
1123         self.wallet = wallet
1124         wallet.synchronizer = self
1125         self.interface = self.wallet.interface
1126         self.interface.register_channel('synchronizer')
1127         self.wallet.interface.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1128         self.was_updated = True
1129         self.running = False
1130         self.lock = threading.Lock()
1131
1132     def stop(self):
1133         with self.lock: self.running = False
1134         self.interface.poke('synchronizer')
1135
1136     def is_running(self):
1137         with self.lock: return self.running
1138
1139
1140     def subscribe_to_addresses(self, addresses):
1141         messages = []
1142         for addr in addresses:
1143             messages.append(('blockchain.address.subscribe', [addr]))
1144         self.interface.send( messages, 'synchronizer')
1145
1146
1147     def run(self):
1148         with self.lock: self.running = True
1149
1150         requested_tx = []
1151         missing_tx = []
1152         requested_histories = {}
1153
1154         # request any missing transactions
1155         for history in self.wallet.history.values():
1156             if history == ['*']: continue
1157             for tx_hash, tx_height in history:
1158                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1159                     missing_tx.append( (tx_hash, tx_height) )
1160         print_error("missing tx", missing_tx)
1161
1162         # wait until we are connected, in case the user is not connected
1163         while not self.interface.is_connected:
1164             time.sleep(1)
1165
1166         # subscriptions
1167         self.subscribe_to_addresses(self.wallet.addresses(True))
1168
1169         while self.is_running():
1170             # 1. create new addresses
1171             new_addresses = self.wallet.synchronize()
1172
1173             # request missing addresses
1174             if new_addresses:
1175                 self.subscribe_to_addresses(new_addresses)
1176
1177             # request missing transactions
1178             for tx_hash, tx_height in missing_tx:
1179                 if (tx_hash, tx_height) not in requested_tx:
1180                     self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
1181                     requested_tx.append( (tx_hash, tx_height) )
1182             missing_tx = []
1183
1184             # detect if situation has changed
1185             if not self.interface.is_up_to_date('synchronizer'):
1186                 if self.wallet.is_up_to_date():
1187                     self.wallet.set_up_to_date(False)
1188                     self.was_updated = True
1189             else:
1190                 if not self.wallet.is_up_to_date():
1191                     self.wallet.set_up_to_date(True)
1192                     self.was_updated = True
1193
1194             if self.was_updated:
1195                 self.interface.trigger_callback('updated')
1196                 self.was_updated = False
1197
1198             # 2. get a response
1199             r = self.interface.get_response('synchronizer')
1200
1201             # poke sends None. (needed during stop)
1202             if not r: continue
1203
1204             # 3. handle response
1205             method = r['method']
1206             params = r['params']
1207             result = r.get('result')
1208             error = r.get('error')
1209             if error:
1210                 print "error", r
1211                 continue
1212
1213             if method == 'blockchain.address.subscribe':
1214                 addr = params[0]
1215                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1216                     if requested_histories.get(addr) is None:
1217                         self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
1218                         requested_histories[addr] = result
1219
1220             elif method == 'blockchain.address.get_history':
1221                 addr = params[0]
1222                 print_error("receiving history", addr, result)
1223                 if result == ['*']:
1224                     assert requested_histories.pop(addr) == '*'
1225                     self.wallet.receive_history_callback(addr, result)
1226                 else:
1227                     hist = []
1228                     # check that txids are unique
1229                     txids = []
1230                     for item in result:
1231                         tx_hash = item['tx_hash']
1232                         if tx_hash not in txids:
1233                             txids.append(tx_hash)
1234                             hist.append( (tx_hash, item['height']) )
1235
1236                     if len(hist) != len(result):
1237                         raise BaseException("error: server sent history with non-unique txid", result)
1238
1239                     # check that the status corresponds to what was announced
1240                     rs = requested_histories.pop(addr)
1241                     if self.wallet.get_status(hist) != rs:
1242                         raise BaseException("error: status mismatch: %s"%addr)
1243
1244                     # store received history
1245                     self.wallet.receive_history_callback(addr, hist)
1246
1247                     # request transactions that we don't have
1248                     for tx_hash, tx_height in hist:
1249                         if self.wallet.transactions.get(tx_hash) is None:
1250                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1251                                 missing_tx.append( (tx_hash, tx_height) )
1252
1253             elif method == 'blockchain.transaction.get':
1254                 tx_hash = params[0]
1255                 tx_height = params[1]
1256                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1257                 tx = Transaction(result)
1258                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1259                 self.was_updated = True
1260                 requested_tx.remove( (tx_hash, tx_height) )
1261                 print_error("received tx:", tx_hash, len(tx.raw))
1262
1263             elif method == 'blockchain.transaction.broadcast':
1264                 self.wallet.tx_result = result
1265                 self.wallet.tx_event.set()
1266
1267             else:
1268                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1269
1270             if self.was_updated and not requested_tx:
1271                 self.interface.trigger_callback('updated')
1272                 self.interface.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1273
1274
1275                 self.was_updated = False