lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, user_dir, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36
  37 # AES encryption
  38 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  39 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  40
  41 def pw_encode(s, password):
  42     if password:
  43         secret = Hash(password)
  44         return EncodeAES(secret, s)
  45     else:
  46         return s
  47
  48 def pw_decode(s, password):
  49     if password is not None:
  50         secret = Hash(password)
  51         try:
  52             d = DecodeAES(secret, s)
  53         except:
  54             raise BaseException('Invalid password')
  55         return d
  56     else:
  57         return s
  58
  59
  60
  61
  62
  63 from version import ELECTRUM_VERSION, SEED_VERSION
  64
  65
  66 class Wallet:
  67     def __init__(self, config={}):
  68
  69         self.config = config
  70         self.electrum_version = ELECTRUM_VERSION
  71         self.gap_limit_for_change = 3 # constant
  72
  73         # saved fields
  74         self.seed_version          = config.get('seed_version', SEED_VERSION)
  75         self.gap_limit             = config.get('gap_limit', 5)
  76         self.use_change            = config.get('use_change',True)
  77         self.fee                   = int(config.get('fee_per_kb',50000))
  78         self.num_zeros             = int(config.get('num_zeros',0))
  79         self.use_encryption        = config.get('use_encryption', False)
  80         self.seed                  = config.get('seed', '')               # encrypted
  81         self.labels                = config.get('labels', {})
  82         self.frozen_addresses      = config.get('frozen_addresses',[])
  83         self.prioritized_addresses = config.get('prioritized_addresses',[])
  84         self.addressbook           = config.get('contacts', [])
  85
  86         self.imported_keys         = config.get('imported_keys',{})
  87         self.history               = config.get('addr_history',{})        # address -> list(txid, height)
  88
  89
  90         self.master_public_keys = config.get('master_public_keys',{})
  91         self.master_private_keys = config.get('master_private_keys', {})
  92
  93         self.load_accounts(config)
  94
  95         self.transactions = {}
  96         tx = config.get('transactions',{})
  97         try:
  98             for k,v in tx.items(): self.transactions[k] = Transaction(v)
  99         except:
 100             print_msg("Warning: Cannot deserialize transactions. skipping")
 101
 102         # not saved
 103         self.prevout_values = {}     # my own transaction outputs
 104         self.spent_outputs = []
 105
 106         # spv
 107         self.verifier = None
 108
 109         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 110         # interface.is_up_to_date() returns true when all requests have been answered and processed
 111         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 112
 113         self.up_to_date = False
 114         self.lock = threading.Lock()
 115         self.transaction_lock = threading.Lock()
 116         self.tx_event = threading.Event()
 117
 118         if self.seed_version != SEED_VERSION:
 119             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
 120
 121         for tx_hash, tx in self.transactions.items():
 122             if self.check_new_tx(tx_hash, tx):
 123                 self.update_tx_outputs(tx_hash)
 124             else:
 125                 print_error("unreferenced tx", tx_hash)
 126                 self.transactions.pop(tx_hash)
 127
 128
 129     def set_up_to_date(self,b):
 130         with self.lock: self.up_to_date = b
 131
 132     def is_up_to_date(self):
 133         with self.lock: return self.up_to_date
 134
 135     def update(self):
 136         self.up_to_date = False
 137         self.interface.poke('synchronizer')
 138         while not self.is_up_to_date(): time.sleep(0.1)
 139
 140     def import_key(self, sec, password):
 141         # check password
 142         seed = self.decode_seed(password)
 143         try:
 144             address = address_from_private_key(sec)
 145         except:
 146             raise BaseException('Invalid private key')
 147
 148         if self.is_mine(address):
 149             raise BaseException('Address already in wallet')
 150
 151         # store the originally requested keypair into the imported keys table
 152         self.imported_keys[address] = pw_encode(sec, password )
 153         self.config.set_key('imported_keys', self.imported_keys, True)
 154         return address
 155
 156     def delete_imported_key(self, addr):
 157         if addr in self.imported_keys:
 158             self.imported_keys.pop(addr)
 159             self.config.set_key('imported_keys', self.imported_keys, True)
 160
 161
 162     def init_seed(self, seed):
 163         if self.seed: raise BaseException("a seed exists")
 164         if not seed:
 165             seed = random_seed(128)
 166         self.seed = seed
 167
 168
 169     def save_seed(self):
 170         self.config.set_key('seed', self.seed, True)
 171         self.config.set_key('seed_version', self.seed_version, True)
 172
 173         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 174
 175         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 176         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 177         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 178
 179         self.master_public_keys = {
 180             "m/0'/": (c0, K0, cK0),
 181             "m/1'/": (c1, K1, cK1),
 182             "m/2'/": (c2, K2, cK2)
 183             }
 184
 185         self.master_private_keys = {
 186             "m/0'/": k0,
 187             "m/1'/": k1
 188             }
 189         # send k2 to service
 190
 191         self.config.set_key('master_public_keys', self.master_public_keys, True)
 192         self.config.set_key('master_private_keys', self.master_private_keys, True)
 193
 194         # create default account
 195         self.create_new_account('Main account')
 196
 197
 198     def create_new_account(self, name):
 199         keys = self.accounts.keys()
 200         i = 0
 201
 202         while True:
 203             derivation = "m/0'/%d'"%i
 204             if derivation not in keys: break
 205             i += 1
 206
 207         start = "m/0'/"
 208         master_c, master_K, master_cK = self.master_public_keys[start]
 209         master_k = self.master_private_keys[start] # needs decryption
 210         k, c, K, cK = bip32_private_derivation(master_k, master_c, start, derivation) # this is a type 1 derivation
 211
 212         self.accounts[derivation] = BIP32_Account({ 'name':name, 'c':c, 'K':K, 'cK':cK })
 213         self.save_accounts()
 214
 215     def create_p2sh_account(self, name):
 216         keys = self.accounts.keys()
 217         i = 0
 218         while True:
 219             account_id = "m/1'/%d & m/2'/%d"%(i,i)
 220             if account_id not in keys: break
 221             i += 1
 222
 223         master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 224         c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 225
 226         master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 227         c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 228
 229         self.accounts[account_id] = BIP32_Account_2of2({ 'name':name, 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 230         self.save_accounts()
 231
 232
 233     def save_accounts(self):
 234         d = {}
 235         for k, v in self.accounts.items():
 236             d[k] = v.dump()
 237         self.config.set_key('accounts', d, True)
 238
 239
 240     def load_accounts(self, config):
 241         d = config.get('accounts', {})
 242         self.accounts = {}
 243         for k, v in d.items():
 244             if '&' in k:
 245                 self.accounts[k] = BIP32_Account_2of2(v)
 246             else:
 247                 self.accounts[k] = BIP32_Account(v)
 248
 249
 250
 251
 252     def addresses(self, include_change = True):
 253         o = self.get_account_addresses(-1, include_change)
 254         for a in self.accounts.keys():
 255             o += self.get_account_addresses(a, include_change)
 256         return o
 257
 258
 259     def is_mine(self, address):
 260         return address in self.addresses(True)
 261
 262     def is_change(self, address):
 263         if not self.is_mine(address): return False
 264         if address in self.imported_keys.keys(): return False
 265         acct, s = self.get_address_index(address)
 266         return s[0] == 1
 267
 268     def get_master_public_key(self):
 269         raise
 270         return self.config.get("master_public_key")
 271
 272     def get_address_index(self, address):
 273         if address in self.imported_keys.keys():
 274             return -1, None
 275         for account in self.accounts.keys():
 276             for for_change in [0,1]:
 277                 addresses = self.accounts[account].get_addresses(for_change)
 278                 for addr in addresses:
 279                     if address == addr:
 280                         return account, (for_change, addresses.index(addr))
 281         raise BaseException("not found")
 282
 283
 284     def get_public_key(self, address):
 285         account, sequence = self.get_address_index(address)
 286         return self.accounts[account].get_pubkey( sequence )
 287
 288
 289     def decode_seed(self, password):
 290         seed = pw_decode(self.seed, password)
 291         #todo:  #self.sequences[0].check_seed(seed)
 292         return seed
 293
 294     def get_private_key(self, address, password):
 295         return self.get_private_keys([address], password).get(address)
 296
 297     def get_private_keys(self, addresses, password):
 298         if not self.seed: return {}
 299         # decode seed in any case, in order to test the password
 300         seed = self.decode_seed(password)
 301         out = {}
 302         for address in addresses:
 303             if address in self.imported_keys.keys():
 304                 out[address] = pw_decode( self.imported_keys[address], password )
 305             else:
 306                 account, sequence = self.get_address_index(address)
 307                 print "found index", address, account, sequence
 308                 if account == "m/0'/0'":
 309                     # FIXME: this is ugly
 310                     master_k = self.master_private_keys["m/0'/"]
 311                     master_c, _, _ = self.master_public_keys["m/0'/"]
 312                     master_k, master_c = CKD(master_k, master_c, 0 + BIP32_PRIME)
 313                     pk = self.accounts["m/0'/0'"].get_private_key(sequence, master_k)
 314                     out[address] = pk
 315
 316                 elif account == "m/1'/0 & m/2'/0":
 317                     master_k = self.master_private_keys["m/1'/"]
 318                     master_c, master_K, _ = self.master_public_keys["m/1'/"]
 319                     master_k, master_c = CKD(master_k.decode('hex'), master_c.decode('hex'), 0)
 320                     pk = self.accounts[account].get_private_key(sequence, master_k)
 321                     out[address] = pk
 322
 323         return out
 324
 325
 326     def signrawtransaction(self, tx, input_info, private_keys, password):
 327         unspent_coins = self.get_unspent_coins()
 328         seed = self.decode_seed(password)
 329
 330         # convert private_keys to dict
 331         pk = {}
 332         for sec in private_keys:
 333             address = address_from_private_key(sec)
 334             pk[address] = sec
 335         private_keys = pk
 336
 337         for txin in tx.inputs:
 338             # convert to own format
 339             txin['tx_hash'] = txin['prevout_hash']
 340             txin['index'] = txin['prevout_n']
 341
 342             for item in input_info:
 343                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 344                     txin['raw_output_script'] = item['scriptPubKey']
 345                     txin['redeemScript'] = item.get('redeemScript')
 346                     txin['KeyID'] = item.get('KeyID')
 347                     break
 348             else:
 349                 for item in unspent_coins:
 350                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 351                         txin['raw_output_script'] = item['raw_output_script']
 352                         break
 353                 else:
 354                     # if neither, we might want to get it from the server..
 355                     raise
 356
 357             # find the address:
 358             if txin.get('KeyID'):
 359                 account, name, sequence = txin.get('KeyID')
 360                 if name != 'Electrum': continue
 361                 sec = self.accounts[account].get_private_key(sequence, seed)
 362                 addr = self.accounts[account].get_address(sequence)
 363                 txin['address'] = addr
 364                 private_keys[addr] = sec
 365
 366             elif txin.get("redeemScript"):
 367                 txin['address'] = hash_160_to_bc_address(hash_160(txin.get("redeemScript").decode('hex')), 5)
 368
 369             elif txin.get("raw_output_script"):
 370                 import deserialize
 371                 addr = deserialize.get_address_from_output_script(txin.get("raw_output_script").decode('hex'))
 372                 sec = self.get_private_key(addr, password)
 373                 if sec:
 374                     private_keys[addr] = sec
 375                     txin['address'] = addr
 376
 377         tx.sign( private_keys )
 378
 379     def sign_message(self, address, message, password):
 380         sec = self.get_private_key(address, password)
 381         key = regenerate_key(sec)
 382         compressed = is_compressed(sec)
 383         return key.sign_message(message, compressed, address)
 384
 385     def verify_message(self, address, signature, message):
 386         try:
 387             EC_KEY.verify_message(address, signature, message)
 388             return True
 389         except BaseException as e:
 390             print_error("Verification error: {0}".format(e))
 391             return False
 392
 393
 394     def change_gap_limit(self, value):
 395         if value >= self.gap_limit:
 396             self.gap_limit = value
 397             self.config.set_key('gap_limit', self.gap_limit, True)
 398             self.interface.poke('synchronizer')
 399             return True
 400
 401         elif value >= self.min_acceptable_gap():
 402             for key, account in self.accounts.items():
 403                 addresses = account[0]
 404                 k = self.num_unused_trailing_addresses(addresses)
 405                 n = len(addresses) - k + value
 406                 addresses = addresses[0:n]
 407                 self.accounts[key][0] = addresses
 408
 409             self.gap_limit = value
 410             self.config.set_key('gap_limit', self.gap_limit, True)
 411             self.save_accounts()
 412             return True
 413         else:
 414             return False
 415
 416     def num_unused_trailing_addresses(self, addresses):
 417         k = 0
 418         for a in addresses[::-1]:
 419             if self.history.get(a):break
 420             k = k + 1
 421         return k
 422
 423     def min_acceptable_gap(self):
 424         # fixme: this assumes wallet is synchronized
 425         n = 0
 426         nmax = 0
 427
 428         for account in self.accounts.values():
 429             addresses = account.get_addresses(0)
 430             k = self.num_unused_trailing_addresses(addresses)
 431             for a in addresses[0:-k]:
 432                 if self.history.get(a):
 433                     n = 0
 434                 else:
 435                     n += 1
 436                     if n > nmax: nmax = n
 437         return nmax + 1
 438
 439
 440     def address_is_old(self, address):
 441         age = -1
 442         h = self.history.get(address, [])
 443         if h == ['*']:
 444             return True
 445         for tx_hash, tx_height in h:
 446             if tx_height == 0:
 447                 tx_age = 0
 448             else:
 449                 tx_age = self.verifier.height - tx_height + 1
 450             if tx_age > age:
 451                 age = tx_age
 452         return age > 2
 453
 454
 455     def synchronize_sequence(self, account, for_change):
 456         limit = self.gap_limit_for_change if for_change else self.gap_limit
 457         new_addresses = []
 458         while True:
 459             addresses = account.get_addresses(for_change)
 460             if len(addresses) < limit:
 461                 address = account.create_new_address(for_change)
 462                 self.history[address] = []
 463                 new_addresses.append( address )
 464                 continue
 465
 466             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 467                 break
 468             else:
 469                 address = account.create_new_address(for_change)
 470                 self.history[address] = []
 471                 new_addresses.append( address )
 472
 473         return new_addresses
 474
 475
 476     def synchronize_account(self, account):
 477         new = []
 478         new += self.synchronize_sequence(account, 0)
 479         new += self.synchronize_sequence(account, 1)
 480         return new
 481
 482     def synchronize(self):
 483         new = []
 484         for account in self.accounts.values():
 485             new += self.synchronize_account(account)
 486         if new:
 487             self.save_accounts()
 488             self.config.set_key('addr_history', self.history, True)
 489         return new
 490
 491
 492     def is_found(self):
 493         return self.history.values() != [[]] * len(self.history)
 494
 495
 496     def add_contact(self, address, label=None):
 497         self.addressbook.append(address)
 498         self.config.set_key('contacts', self.addressbook, True)
 499         if label:
 500             self.labels[address] = label
 501             self.config.set_key('labels', self.labels)
 502
 503     def delete_contact(self, addr):
 504         if addr in self.addressbook:
 505             self.addressbook.remove(addr)
 506             self.config.set_key('addressbook', self.addressbook, True)
 507
 508
 509     def fill_addressbook(self):
 510         for tx_hash, tx in self.transactions.items():
 511             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 512             if is_send:
 513                 for addr, v in tx.outputs:
 514                     if not self.is_mine(addr) and addr not in self.addressbook:
 515                         self.addressbook.append(addr)
 516         # redo labels
 517         # self.update_tx_labels()
 518
 519     def get_num_tx(self, address):
 520         n = 0
 521         for tx in self.transactions.values():
 522             if address in map(lambda x:x[0], tx.outputs): n += 1
 523         return n
 524
 525
 526     def get_address_flags(self, addr):
 527         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 528         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 529         return flags
 530
 531
 532     def get_tx_value(self, tx, account=None):
 533         domain = self.get_account_addresses(account)
 534         return tx.get_value(domain, self.prevout_values)
 535
 536
 537     def update_tx_outputs(self, tx_hash):
 538         tx = self.transactions.get(tx_hash)
 539
 540         for i, (addr, value) in enumerate(tx.outputs):
 541             key = tx_hash+ ':%d'%i
 542             self.prevout_values[key] = value
 543
 544         for item in tx.inputs:
 545             if self.is_mine(item.get('address')):
 546                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 547                 self.spent_outputs.append(key)
 548
 549
 550     def get_addr_balance(self, address):
 551         assert self.is_mine(address)
 552         h = self.history.get(address,[])
 553         if h == ['*']: return 0,0
 554         c = u = 0
 555         received_coins = []   # list of coins received at address
 556
 557         for tx_hash, tx_height in h:
 558             tx = self.transactions.get(tx_hash)
 559             if not tx: continue
 560
 561             for i, (addr, value) in enumerate(tx.outputs):
 562                 if addr == address:
 563                     key = tx_hash + ':%d'%i
 564                     received_coins.append(key)
 565
 566         for tx_hash, tx_height in h:
 567             tx = self.transactions.get(tx_hash)
 568             if not tx: continue
 569             v = 0
 570
 571             for item in tx.inputs:
 572                 addr = item.get('address')
 573                 if addr == address:
 574                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 575                     value = self.prevout_values.get( key )
 576                     if key in received_coins:
 577                         v -= value
 578
 579             for i, (addr, value) in enumerate(tx.outputs):
 580                 key = tx_hash + ':%d'%i
 581                 if addr == address:
 582                     v += value
 583
 584             if tx_height:
 585                 c += v
 586             else:
 587                 u += v
 588         return c, u
 589
 590
 591     def get_accounts(self):
 592         accounts = {}
 593         for k, account in self.accounts.items():
 594             accounts[k] = account.name
 595         if self.imported_keys:
 596             accounts[-1] = 'Imported keys'
 597         return accounts
 598
 599     def get_account_addresses(self, a, include_change=True):
 600         if a is None:
 601             o = self.addresses(True)
 602         elif a == -1:
 603             o = self.imported_keys.keys()
 604         else:
 605             ac = self.accounts[a]
 606             o = ac.get_addresses(0)
 607             if include_change: o += ac.get_addresses(1)
 608         return o
 609
 610     def get_imported_balance(self):
 611         cc = uu = 0
 612         for addr in self.imported_keys.keys():
 613             c, u = self.get_addr_balance(addr)
 614             cc += c
 615             uu += u
 616         return cc, uu
 617
 618     def get_account_balance(self, account):
 619         if account is None:
 620             return self.get_balance()
 621         elif account == -1:
 622             return self.get_imported_balance()
 623
 624         conf = unconf = 0
 625         for addr in self.get_account_addresses(account):
 626             c, u = self.get_addr_balance(addr)
 627             conf += c
 628             unconf += u
 629         return conf, unconf
 630
 631     def get_frozen_balance(self):
 632         conf = unconf = 0
 633         for addr in self.frozen_addresses:
 634             c, u = self.get_addr_balance(addr)
 635             conf += c
 636             unconf += u
 637         return conf, unconf
 638
 639
 640     def get_balance(self):
 641         cc = uu = 0
 642         for a in self.accounts.keys():
 643             c, u = self.get_account_balance(a)
 644             cc += c
 645             uu += u
 646         c, u = self.get_imported_balance()
 647         cc += c
 648         uu += u
 649         return cc, uu
 650
 651
 652     def get_unspent_coins(self, domain=None):
 653         coins = []
 654         if domain is None: domain = self.addresses(True)
 655         for addr in domain:
 656             h = self.history.get(addr, [])
 657             if h == ['*']: continue
 658             for tx_hash, tx_height in h:
 659                 tx = self.transactions.get(tx_hash)
 660                 if tx is None: raise BaseException("Wallet not synchronized")
 661                 for output in tx.d.get('outputs'):
 662                     if output.get('address') != addr: continue
 663                     key = tx_hash + ":%d" % output.get('index')
 664                     if key in self.spent_outputs: continue
 665                     output['tx_hash'] = tx_hash
 666                     coins.append(output)
 667         return coins
 668
 669
 670
 671     def choose_tx_inputs( self, amount, fixed_fee, account = None ):
 672         """ todo: minimize tx size """
 673         total = 0
 674         fee = self.fee if fixed_fee is None else fixed_fee
 675         domain = self.get_account_addresses(account)
 676         coins = []
 677         prioritized_coins = []
 678         for i in self.frozen_addresses:
 679             if i in domain: domain.remove(i)
 680
 681         for i in self.prioritized_addresses:
 682             if i in domain: domain.remove(i)
 683
 684         coins = self.get_unspent_coins(domain)
 685         prioritized_coins = self.get_unspent_coins(self.prioritized_addresses)
 686
 687         inputs = []
 688         coins = prioritized_coins + coins
 689
 690         for item in coins:
 691             addr = item.get('address')
 692             v = item.get('value')
 693             total += v
 694             inputs.append( item )
 695             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
 696             if total >= amount + fee: break
 697         else:
 698             inputs = []
 699
 700         return inputs, total, fee
 701
 702
 703     def estimated_fee(self, inputs):
 704         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
 705         fee = self.fee * int(round(estimated_size/1024.))
 706         if fee == 0: fee = self.fee
 707         return fee
 708
 709
 710     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None, account=0 ):
 711         "add change to a transaction"
 712         change_amount = total - ( amount + fee )
 713         if change_amount != 0:
 714             if not change_addr:
 715                 if account is None:
 716                     # send change to one of the accounts involved in the tx
 717                     address = inputs[0].get('address')
 718                     account, _ = self.get_address_index(address)
 719
 720                 if not self.use_change or account == -1:
 721                     change_addr = inputs[-1]['address']
 722                 else:
 723                     change_addr = self.accounts[account][1][-self.gap_limit_for_change]
 724
 725             # Insert the change output at a random position in the outputs
 726             posn = random.randint(0, len(outputs))
 727             outputs[posn:posn] = [( change_addr,  change_amount)]
 728         return outputs
 729
 730
 731     def get_history(self, address):
 732         with self.lock:
 733             return self.history.get(address)
 734
 735
 736     def get_status(self, h):
 737         if not h: return None
 738         if h == ['*']: return '*'
 739         status = ''
 740         for tx_hash, height in h:
 741             status += tx_hash + ':%d:' % height
 742         return hashlib.sha256( status ).digest().encode('hex')
 743
 744
 745     def receive_tx_callback(self, tx_hash, tx, tx_height):
 746         if not self.check_new_tx(tx_hash, tx):
 747             # may happen due to pruning
 748             print_error("received transaction that is no longer referenced in history", tx_hash)
 749             return
 750
 751         with self.transaction_lock:
 752             self.transactions[tx_hash] = tx
 753
 754             self.interface.pending_transactions_for_notifications.append(tx)
 755
 756             self.save_transactions()
 757             if self.verifier and tx_height>0:
 758                 self.verifier.add(tx_hash, tx_height)
 759             self.update_tx_outputs(tx_hash)
 760
 761
 762     def save_transactions(self):
 763         tx = {}
 764         for k,v in self.transactions.items():
 765             tx[k] = str(v)
 766         self.config.set_key('transactions', tx, True)
 767
 768     def receive_history_callback(self, addr, hist):
 769
 770         if not self.check_new_history(addr, hist):
 771             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
 772
 773         with self.lock:
 774             self.history[addr] = hist
 775             self.config.set_key('addr_history', self.history, True)
 776
 777         if hist != ['*']:
 778             for tx_hash, tx_height in hist:
 779                 if tx_height>0:
 780                     # add it in case it was previously unconfirmed
 781                     if self.verifier: self.verifier.add(tx_hash, tx_height)
 782
 783
 784     def get_tx_history(self, account=None):
 785         with self.transaction_lock:
 786             history = self.transactions.items()
 787             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
 788             result = []
 789
 790             balance = 0
 791             for tx_hash, tx in history:
 792                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
 793                 if v is not None: balance += v
 794
 795             c, u = self.get_account_balance(account)
 796
 797             if balance != c+u:
 798                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
 799
 800             balance = c + u - balance
 801             for tx_hash, tx in history:
 802                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
 803                 if not is_relevant:
 804                     continue
 805                 if value is not None:
 806                     balance += value
 807
 808                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
 809                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
 810
 811         return result
 812
 813
 814     def get_label(self, tx_hash):
 815         label = self.labels.get(tx_hash)
 816         is_default = (label == '') or (label is None)
 817         if is_default: label = self.get_default_label(tx_hash)
 818         return label, is_default
 819
 820
 821     def get_default_label(self, tx_hash):
 822         tx = self.transactions.get(tx_hash)
 823         default_label = ''
 824         if tx:
 825             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
 826             if is_mine:
 827                 for o in tx.outputs:
 828                     o_addr, _ = o
 829                     if not self.is_mine(o_addr):
 830                         try:
 831                             default_label = self.labels[o_addr]
 832                         except KeyError:
 833                             default_label = o_addr
 834                         break
 835                 else:
 836                     default_label = '(internal)'
 837             else:
 838                 for o in tx.outputs:
 839                     o_addr, _ = o
 840                     if self.is_mine(o_addr) and not self.is_change(o_addr):
 841                         break
 842                 else:
 843                     for o in tx.outputs:
 844                         o_addr, _ = o
 845                         if self.is_mine(o_addr):
 846                             break
 847                     else:
 848                         o_addr = None
 849
 850                 if o_addr:
 851                     dest_label = self.labels.get(o_addr)
 852                     try:
 853                         default_label = self.labels[o_addr]
 854                     except KeyError:
 855                         default_label = o_addr
 856
 857         return default_label
 858
 859
 860     def mktx(self, outputs, password, fee=None, change_addr=None, account=None ):
 861         """
 862         create a transaction
 863         account parameter:
 864            None means use all accounts
 865            -1 means imported keys
 866            0, 1, etc are seed accounts
 867         """
 868
 869         for address, x in outputs:
 870             assert is_valid(address)
 871
 872         amount = sum( map(lambda x:x[1], outputs) )
 873
 874         inputs, total, fee = self.choose_tx_inputs( amount, fee, account )
 875         if not inputs:
 876             raise ValueError("Not enough funds")
 877
 878         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr, account)
 879
 880         tx = Transaction.from_io(inputs, outputs)
 881
 882         pk_addresses = []
 883         for i in range(len(tx.inputs)):
 884             txin = tx.inputs[i]
 885             address = txin['address']
 886             if address in self.imported_keys.keys():
 887                 pk_addresses.append(address)
 888                 continue
 889             account, sequence = self.get_address_index(address)
 890
 891             txin['KeyID'] = (account, 'BIP32', sequence) # used by the server to find the key
 892
 893             _, redeemScript = self.accounts[account].get_input_info(sequence)
 894
 895             if redeemScript: txin['redeemScript'] = redeemScript
 896             pk_addresses.append(address)
 897
 898         print "pk_addresses", pk_addresses
 899
 900         # get all private keys at once.
 901         if self.seed:
 902             private_keys = self.get_private_keys(pk_addresses, password)
 903             print "private keys", private_keys
 904             tx.sign(private_keys)
 905
 906         for address, x in outputs:
 907             if address not in self.addressbook and not self.is_mine(address):
 908                 self.addressbook.append(address)
 909
 910         return tx
 911
 912
 913
 914     def sendtx(self, tx):
 915         # synchronous
 916         h = self.send_tx(tx)
 917         self.tx_event.wait()
 918         return self.receive_tx(h)
 919
 920     def send_tx(self, tx):
 921         # asynchronous
 922         self.tx_event.clear()
 923         self.interface.send([('blockchain.transaction.broadcast', [str(tx)])], 'synchronizer')
 924         return tx.hash()
 925
 926     def receive_tx(self,tx_hash):
 927         out = self.tx_result
 928         if out != tx_hash:
 929             return False, "error: " + out
 930         return True, out
 931
 932
 933
 934     def update_password(self, seed, old_password, new_password):
 935         if new_password == '': new_password = None
 936         # this will throw an exception if unicode cannot be converted
 937         self.seed = pw_encode( seed, new_password)
 938         self.config.set_key('seed', self.seed, True)
 939         self.use_encryption = (new_password != None)
 940         self.config.set_key('use_encryption', self.use_encryption,True)
 941         for k in self.imported_keys.keys():
 942             a = self.imported_keys[k]
 943             b = pw_decode(a, old_password)
 944             c = pw_encode(b, new_password)
 945             self.imported_keys[k] = c
 946         self.config.set_key('imported_keys', self.imported_keys, True)
 947
 948
 949     def freeze(self,addr):
 950         if self.is_mine(addr) and addr not in self.frozen_addresses:
 951             self.unprioritize(addr)
 952             self.frozen_addresses.append(addr)
 953             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 954             return True
 955         else:
 956             return False
 957
 958     def unfreeze(self,addr):
 959         if self.is_mine(addr) and addr in self.frozen_addresses:
 960             self.frozen_addresses.remove(addr)
 961             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 962             return True
 963         else:
 964             return False
 965
 966     def prioritize(self,addr):
 967         if self.is_mine(addr) and addr not in self.prioritized_addresses:
 968             self.unfreeze(addr)
 969             self.prioritized_addresses.append(addr)
 970             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
 971             return True
 972         else:
 973             return False
 974
 975     def unprioritize(self,addr):
 976         if self.is_mine(addr) and addr in self.prioritized_addresses:
 977             self.prioritized_addresses.remove(addr)
 978             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
 979             return True
 980         else:
 981             return False
 982
 983     def set_fee(self, fee):
 984         if self.fee != fee:
 985             self.fee = fee
 986             self.config.set_key('fee_per_kb', self.fee, True)
 987
 988
 989     def save(self):
 990         print_error("Warning: wallet.save() is deprecated")
 991         tx = {}
 992         for k,v in self.transactions.items():
 993             tx[k] = str(v)
 994
 995         s = {
 996             'use_change': self.use_change,
 997             'fee_per_kb': self.fee,
 998             'addr_history': self.history,
 999             'labels': self.labels,
1000             'contacts': self.addressbook,
1001             'num_zeros': self.num_zeros,
1002             'frozen_addresses': self.frozen_addresses,
1003             'prioritized_addresses': self.prioritized_addresses,
1004             'gap_limit': self.gap_limit,
1005             'transactions': tx,
1006         }
1007         for k, v in s.items():
1008             self.config.set_key(k,v)
1009         self.config.save()
1010
1011     def set_verifier(self, verifier):
1012         self.verifier = verifier
1013
1014         # review transactions that are in the history
1015         for addr, hist in self.history.items():
1016             if hist == ['*']: continue
1017             for tx_hash, tx_height in hist:
1018                 if tx_height>0:
1019                     # add it in case it was previously unconfirmed
1020                     self.verifier.add(tx_hash, tx_height)
1021
1022
1023         # if we are on a pruning server, remove unverified transactions
1024         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1025         for tx_hash in self.transactions.keys():
1026             if tx_hash not in vr:
1027                 self.transactions.pop(tx_hash)
1028
1029
1030
1031     def check_new_history(self, addr, hist):
1032
1033         # check that all tx in hist are relevant
1034         if hist != ['*']:
1035             for tx_hash, height in hist:
1036                 tx = self.transactions.get(tx_hash)
1037                 if not tx: continue
1038                 if not tx.has_address(addr):
1039                     return False
1040
1041         # check that we are not "orphaning" a transaction
1042         old_hist = self.history.get(addr,[])
1043         if old_hist == ['*']: return True
1044
1045         for tx_hash, height in old_hist:
1046             if tx_hash in map(lambda x:x[0], hist): continue
1047             found = False
1048             for _addr, _hist in self.history.items():
1049                 if _addr == addr: continue
1050                 if _hist == ['*']: continue
1051                 _tx_hist = map(lambda x:x[0], _hist)
1052                 if tx_hash in _tx_hist:
1053                     found = True
1054                     break
1055
1056             if not found:
1057                 tx = self.transactions.get(tx_hash)
1058                 # tx might not be there
1059                 if not tx: continue
1060
1061                 # already verified?
1062                 if self.verifier.get_height(tx_hash):
1063                     continue
1064                 # unconfirmed tx
1065                 print_error("new history is orphaning transaction:", tx_hash)
1066                 # check that all outputs are not mine, request histories
1067                 ext_requests = []
1068                 for _addr, _v in tx.outputs:
1069                     # assert not self.is_mine(_addr)
1070                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1071
1072                 ext_h = self.interface.synchronous_get(ext_requests)
1073                 print_error("sync:", ext_requests, ext_h)
1074                 height = None
1075                 for h in ext_h:
1076                     if h == ['*']: continue
1077                     for item in h:
1078                         if item.get('tx_hash') == tx_hash:
1079                             height = item.get('height')
1080                 if height:
1081                     print_error("found height for", tx_hash, height)
1082                     self.verifier.add(tx_hash, height)
1083                 else:
1084                     print_error("removing orphaned tx from history", tx_hash)
1085                     self.transactions.pop(tx_hash)
1086
1087         return True
1088
1089
1090
1091     def check_new_tx(self, tx_hash, tx):
1092         # 1 check that tx is referenced in addr_history.
1093         addresses = []
1094         for addr, hist in self.history.items():
1095             if hist == ['*']:continue
1096             for txh, height in hist:
1097                 if txh == tx_hash:
1098                     addresses.append(addr)
1099
1100         if not addresses:
1101             return False
1102
1103         # 2 check that referencing addresses are in the tx
1104         for addr in addresses:
1105             if not tx.has_address(addr):
1106                 return False
1107
1108         return True
1109
1110
1111
1112
1113 class WalletSynchronizer(threading.Thread):
1114
1115
1116     def __init__(self, wallet, config):
1117         threading.Thread.__init__(self)
1118         self.daemon = True
1119         self.wallet = wallet
1120         wallet.synchronizer = self
1121         self.interface = self.wallet.interface
1122         self.interface.register_channel('synchronizer')
1123         self.wallet.interface.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1124         self.was_updated = True
1125         self.running = False
1126         self.lock = threading.Lock()
1127
1128     def stop(self):
1129         with self.lock: self.running = False
1130         self.interface.poke('synchronizer')
1131
1132     def is_running(self):
1133         with self.lock: return self.running
1134
1135
1136     def subscribe_to_addresses(self, addresses):
1137         messages = []
1138         for addr in addresses:
1139             messages.append(('blockchain.address.subscribe', [addr]))
1140         self.interface.send( messages, 'synchronizer')
1141
1142
1143     def run(self):
1144         with self.lock: self.running = True
1145
1146         requested_tx = []
1147         missing_tx = []
1148         requested_histories = {}
1149
1150         # request any missing transactions
1151         for history in self.wallet.history.values():
1152             if history == ['*']: continue
1153             for tx_hash, tx_height in history:
1154                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1155                     missing_tx.append( (tx_hash, tx_height) )
1156         print_error("missing tx", missing_tx)
1157
1158         # wait until we are connected, in case the user is not connected
1159         while not self.interface.is_connected:
1160             time.sleep(1)
1161
1162         # subscriptions
1163         self.subscribe_to_addresses(self.wallet.addresses(True))
1164
1165         while self.is_running():
1166             # 1. create new addresses
1167             new_addresses = self.wallet.synchronize()
1168
1169             # request missing addresses
1170             if new_addresses:
1171                 self.subscribe_to_addresses(new_addresses)
1172
1173             # request missing transactions
1174             for tx_hash, tx_height in missing_tx:
1175                 if (tx_hash, tx_height) not in requested_tx:
1176                     self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
1177                     requested_tx.append( (tx_hash, tx_height) )
1178             missing_tx = []
1179
1180             # detect if situation has changed
1181             if not self.interface.is_up_to_date('synchronizer'):
1182                 if self.wallet.is_up_to_date():
1183                     self.wallet.set_up_to_date(False)
1184                     self.was_updated = True
1185             else:
1186                 if not self.wallet.is_up_to_date():
1187                     self.wallet.set_up_to_date(True)
1188                     self.was_updated = True
1189
1190             if self.was_updated:
1191                 self.interface.trigger_callback('updated')
1192                 self.was_updated = False
1193
1194             # 2. get a response
1195             r = self.interface.get_response('synchronizer')
1196
1197             # poke sends None. (needed during stop)
1198             if not r: continue
1199
1200             # 3. handle response
1201             method = r['method']
1202             params = r['params']
1203             result = r.get('result')
1204             error = r.get('error')
1205             if error:
1206                 print "error", r
1207                 continue
1208
1209             if method == 'blockchain.address.subscribe':
1210                 addr = params[0]
1211                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1212                     if requested_histories.get(addr) is None:
1213                         self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
1214                         requested_histories[addr] = result
1215
1216             elif method == 'blockchain.address.get_history':
1217                 addr = params[0]
1218                 print_error("receiving history", addr, result)
1219                 if result == ['*']:
1220                     assert requested_histories.pop(addr) == '*'
1221                     self.wallet.receive_history_callback(addr, result)
1222                 else:
1223                     hist = []
1224                     # check that txids are unique
1225                     txids = []
1226                     for item in result:
1227                         tx_hash = item['tx_hash']
1228                         if tx_hash not in txids:
1229                             txids.append(tx_hash)
1230                             hist.append( (tx_hash, item['height']) )
1231
1232                     if len(hist) != len(result):
1233                         raise BaseException("error: server sent history with non-unique txid", result)
1234
1235                     # check that the status corresponds to what was announced
1236                     rs = requested_histories.pop(addr)
1237                     if self.wallet.get_status(hist) != rs:
1238                         raise BaseException("error: status mismatch: %s"%addr)
1239
1240                     # store received history
1241                     self.wallet.receive_history_callback(addr, hist)
1242
1243                     # request transactions that we don't have
1244                     for tx_hash, tx_height in hist:
1245                         if self.wallet.transactions.get(tx_hash) is None:
1246                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1247                                 missing_tx.append( (tx_hash, tx_height) )
1248
1249             elif method == 'blockchain.transaction.get':
1250                 tx_hash = params[0]
1251                 tx_height = params[1]
1252                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1253                 tx = Transaction(result)
1254                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1255                 self.was_updated = True
1256                 requested_tx.remove( (tx_hash, tx_height) )
1257                 print_error("received tx:", tx_hash, len(tx.raw))
1258
1259             elif method == 'blockchain.transaction.broadcast':
1260                 self.wallet.tx_result = result
1261                 self.wallet.tx_event.set()
1262
1263             else:
1264                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1265
1266             if self.was_updated and not requested_tx:
1267                 self.interface.trigger_callback('updated')
1268                 self.interface.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1269
1270
1271                 self.was_updated = False