lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, user_dir, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36
  37 # AES encryption
  38 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  39 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  40
  41 def pw_encode(s, password):
  42     if password:
  43         secret = Hash(password)
  44         return EncodeAES(secret, s)
  45     else:
  46         return s
  47
  48 def pw_decode(s, password):
  49     if password is not None:
  50         secret = Hash(password)
  51         try:
  52             d = DecodeAES(secret, s)
  53         except:
  54             raise BaseException('Invalid password')
  55         return d
  56     else:
  57         return s
  58
  59
  60
  61
  62
  63 from version import ELECTRUM_VERSION, SEED_VERSION
  64
  65
  66 class Wallet:
  67     def __init__(self, config={}):
  68
  69         self.config = config
  70         self.electrum_version = ELECTRUM_VERSION
  71         self.gap_limit_for_change = 3 # constant
  72
  73         # saved fields
  74         self.seed_version          = config.get('seed_version', SEED_VERSION)
  75         self.gap_limit             = config.get('gap_limit', 5)
  76         self.use_change            = config.get('use_change',True)
  77         self.fee                   = int(config.get('fee_per_kb',50000))
  78         self.num_zeros             = int(config.get('num_zeros',0))
  79         self.use_encryption        = config.get('use_encryption', False)
  80         self.seed                  = config.get('seed', '')               # encrypted
  81         self.labels                = config.get('labels', {})
  82         self.frozen_addresses      = config.get('frozen_addresses',[])
  83         self.prioritized_addresses = config.get('prioritized_addresses',[])
  84         self.addressbook           = config.get('contacts', [])
  85
  86         self.imported_keys         = config.get('imported_keys',{})
  87         self.history               = config.get('addr_history',{})        # address -> list(txid, height)
  88
  89
  90         self.master_public_keys = config.get('master_public_keys',{})
  91         self.master_private_keys = config.get('master_private_keys', {})
  92
  93         self.load_accounts(config)
  94
  95         self.transactions = {}
  96         tx = config.get('transactions',{})
  97         try:
  98             for k,v in tx.items(): self.transactions[k] = Transaction(v)
  99         except:
 100             print_msg("Warning: Cannot deserialize transactions. skipping")
 101
 102         # not saved
 103         self.prevout_values = {}     # my own transaction outputs
 104         self.spent_outputs = []
 105
 106         # spv
 107         self.verifier = None
 108
 109         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 110         # interface.is_up_to_date() returns true when all requests have been answered and processed
 111         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 112
 113         self.up_to_date = False
 114         self.lock = threading.Lock()
 115         self.transaction_lock = threading.Lock()
 116         self.tx_event = threading.Event()
 117
 118         if self.seed_version != SEED_VERSION:
 119             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
 120
 121         for tx_hash, tx in self.transactions.items():
 122             if self.check_new_tx(tx_hash, tx):
 123                 self.update_tx_outputs(tx_hash)
 124             else:
 125                 print_error("unreferenced tx", tx_hash)
 126                 self.transactions.pop(tx_hash)
 127
 128
 129     def set_up_to_date(self,b):
 130         with self.lock: self.up_to_date = b
 131
 132     def is_up_to_date(self):
 133         with self.lock: return self.up_to_date
 134
 135     def update(self):
 136         self.up_to_date = False
 137         self.interface.poke('synchronizer')
 138         while not self.is_up_to_date(): time.sleep(0.1)
 139
 140     def import_key(self, sec, password):
 141         # check password
 142         seed = self.decode_seed(password)
 143         try:
 144             address = address_from_private_key(sec)
 145         except:
 146             raise BaseException('Invalid private key')
 147
 148         if self.is_mine(address):
 149             raise BaseException('Address already in wallet')
 150
 151         # store the originally requested keypair into the imported keys table
 152         self.imported_keys[address] = pw_encode(sec, password )
 153         self.config.set_key('imported_keys', self.imported_keys, True)
 154         return address
 155
 156     def delete_imported_key(self, addr):
 157         if addr in self.imported_keys:
 158             self.imported_keys.pop(addr)
 159             self.config.set_key('imported_keys', self.imported_keys, True)
 160
 161
 162     def init_seed(self, seed):
 163         if self.seed: raise BaseException("a seed exists")
 164         if not seed:
 165             seed = random_seed(128)
 166         self.seed = seed
 167
 168
 169     def save_seed(self):
 170         self.config.set_key('seed', self.seed, True)
 171         self.config.set_key('seed_version', self.seed_version, True)
 172
 173         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 174
 175         # normal accounts
 176         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 177         # p2sh 2of2
 178         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 179         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 180         # p2sh 2of3
 181         k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 182         k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 183         k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 184
 185         self.master_public_keys = {
 186             "m/0'/": (c0, K0, cK0),
 187             "m/1'/": (c1, K1, cK1),
 188             "m/2'/": (c2, K2, cK2),
 189             "m/3'/": (c3, K3, cK3),
 190             "m/4'/": (c4, K4, cK4),
 191             "m/5'/": (c5, K5, cK5)
 192             }
 193
 194         self.master_private_keys = {
 195             "m/0'/": k0,
 196             "m/1'/": k1,
 197             "m/2'/": k2,
 198             "m/3'/": k3,
 199             "m/4'/": k4,
 200             "m/5'/": k5
 201             }
 202
 203         self.config.set_key('master_public_keys', self.master_public_keys, True)
 204         self.config.set_key('master_private_keys', self.master_private_keys, True)
 205
 206         # create default account
 207         self.create_account('Main account')
 208
 209
 210     def account_id(self, account_type, i):
 211         if account_type is None:
 212             return "m/0'/%d'"%i
 213         elif account_type == '2of2':
 214             return "m/1'/%d & m/2'/%d"%(i,i)
 215         elif account_type == '2of3':
 216             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 217         else:
 218             raise BaseException('unknown account type')
 219
 220
 221     def num_accounts(self, account_type):
 222         keys = self.accounts.keys()
 223         i = 0
 224         while True:
 225             account_id = self.account_id(account_type, i)
 226             if account_id not in keys: break
 227             i += 1
 228         return i
 229
 230
 231     def create_account(self, name, account_type = None):
 232         i = self.num_accounts(account_type)
 233         acount_id = self.account_id(account_type,i)
 234
 235         if account_type is None:
 236             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 237             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 238             account = BIP32_Account({ 'name':name, 'c':c0, 'K':K0, 'cK':cK0 })
 239
 240         elif account_type == '2of2':
 241             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 242             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 243             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 244             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 245             account = BIP32_Account_2of2({ 'name':name, 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 246
 247         elif account_type == '2of3':
 248             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 249             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 250             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 251             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 252             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 253             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 254             account = BIP32_Account_2of3({ 'name':name, 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 255
 256         self.accounts[account_id] = account
 257         self.save_accounts()
 258
 259
 260     def save_accounts(self):
 261         d = {}
 262         for k, v in self.accounts.items():
 263             d[k] = v.dump()
 264         self.config.set_key('accounts', d, True)
 265
 266
 267     def load_accounts(self, config):
 268         d = config.get('accounts', {})
 269         self.accounts = {}
 270         for k, v in d.items():
 271             if '&' in k:
 272                 self.accounts[k] = BIP32_Account_2of2(v)
 273             else:
 274                 self.accounts[k] = BIP32_Account(v)
 275
 276
 277
 278
 279     def addresses(self, include_change = True):
 280         o = self.get_account_addresses(-1, include_change)
 281         for a in self.accounts.keys():
 282             o += self.get_account_addresses(a, include_change)
 283         return o
 284
 285
 286     def is_mine(self, address):
 287         return address in self.addresses(True)
 288
 289     def is_change(self, address):
 290         if not self.is_mine(address): return False
 291         if address in self.imported_keys.keys(): return False
 292         acct, s = self.get_address_index(address)
 293         return s[0] == 1
 294
 295     def get_master_public_key(self):
 296         raise
 297         return self.config.get("master_public_key")
 298
 299     def get_master_private_key(self, account, password):
 300         master_k = pw_decode( self.master_private_keys[account], password)
 301         master_c, master_K, master_Kc = self.master_public_keys[account]
 302         try:
 303             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 304             assert K.encode('hex') == master_K
 305         except:
 306             raise BaseException("Invalid password")
 307         return master_k
 308
 309
 310     def get_address_index(self, address):
 311         if address in self.imported_keys.keys():
 312             return -1, None
 313         for account in self.accounts.keys():
 314             for for_change in [0,1]:
 315                 addresses = self.accounts[account].get_addresses(for_change)
 316                 for addr in addresses:
 317                     if address == addr:
 318                         return account, (for_change, addresses.index(addr))
 319         raise BaseException("not found")
 320
 321
 322     def get_public_key(self, address):
 323         account, sequence = self.get_address_index(address)
 324         return self.accounts[account].get_pubkey( *sequence )
 325
 326
 327     def decode_seed(self, password):
 328         seed = pw_decode(self.seed, password)
 329         #todo:  #self.sequences[0].check_seed(seed)
 330         return seed
 331
 332
 333     def get_private_key(self, address, password):
 334         out = []
 335         if address in self.imported_keys.keys():
 336             out.append( pw_decode( self.imported_keys[address], password ) )
 337         else:
 338             account, sequence = self.get_address_index(address)
 339             # assert address == self.accounts[account].get_address(*sequence)
 340             l = account.split("&")
 341             for s in l:
 342                 s = s.strip()
 343                 m = re.match("(m/\d+'/)(\d+)", s)
 344                 if m:
 345                     root = m.group(1)
 346                     if root not in self.master_private_keys.keys(): continue
 347                     num = int(m.group(2))
 348                     master_k = self.get_master_private_key(root, password)
 349                     master_c, _, _ = self.master_public_keys[root]
 350                     pk = bip32_private_key( (num,) + sequence, master_k.decode('hex'), master_c.decode('hex'))
 351                     out.append(pk)
 352
 353         return out
 354
 355
 356
 357
 358     def signrawtransaction(self, tx, input_info, private_keys, password):
 359         import deserialize
 360         unspent_coins = self.get_unspent_coins()
 361         seed = self.decode_seed(password)
 362
 363         # build a list of public/private keys
 364         keypairs = {}
 365         for sec in private_keys:
 366             pubkey = public_key_from_private_key(sec)
 367             keypairs[ pubkey ] = sec
 368
 369
 370         for txin in tx.inputs:
 371             # convert to own format
 372             txin['tx_hash'] = txin['prevout_hash']
 373             txin['index'] = txin['prevout_n']
 374
 375             for item in input_info:
 376                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 377                     txin['raw_output_script'] = item['scriptPubKey']
 378                     txin['redeemScript'] = item.get('redeemScript')
 379                     txin['KeyID'] = item.get('KeyID')
 380                     break
 381             else:
 382                 for item in unspent_coins:
 383                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 384                         txin['raw_output_script'] = item['raw_output_script']
 385                         break
 386                 else:
 387                     # if neither, we might want to get it from the server..
 388                     raise
 389
 390             # find the address and fill private_keys
 391             if txin.get('KeyID'):
 392                 account, name, sequence = txin.get('KeyID')
 393                 if name != 'BIP32': continue
 394                 sec = self.accounts[account].get_private_key(sequence, seed)
 395                 pubkey = self.accounts[account].get_pubkey(sequence)
 396                 txin['address'] = addr
 397                 keypairs[pubkey] = [sec]
 398
 399             redeem_script = txin.get("redeemScript")
 400             if redeem_script:
 401                 num, redeem_pubkeys = deserialize.parse_redeemScript(redeem_script)
 402                 addr = hash_160_to_bc_address(hash_160(redeem_script.decode('hex')), 5)
 403                 txin['address'] = addr
 404
 405             elif txin.get("raw_output_script"):
 406                 addr = deserialize.get_address_from_output_script(txin.get("raw_output_script").decode('hex'))
 407                 sec = self.get_private_key(addr, password)
 408                 pubkey = public_key_from_private_key(sec)
 409                 if sec:
 410                     keypairs[pubkey] = [sec]
 411                     txin['address'] = addr
 412
 413         tx.sign( keypairs )
 414
 415     def sign_message(self, address, message, password):
 416         sec = self.get_private_key(address, password)
 417         key = regenerate_key(sec)
 418         compressed = is_compressed(sec)
 419         return key.sign_message(message, compressed, address)
 420
 421     def verify_message(self, address, signature, message):
 422         try:
 423             EC_KEY.verify_message(address, signature, message)
 424             return True
 425         except BaseException as e:
 426             print_error("Verification error: {0}".format(e))
 427             return False
 428
 429
 430     def change_gap_limit(self, value):
 431         if value >= self.gap_limit:
 432             self.gap_limit = value
 433             self.config.set_key('gap_limit', self.gap_limit, True)
 434             self.interface.poke('synchronizer')
 435             return True
 436
 437         elif value >= self.min_acceptable_gap():
 438             for key, account in self.accounts.items():
 439                 addresses = account[0]
 440                 k = self.num_unused_trailing_addresses(addresses)
 441                 n = len(addresses) - k + value
 442                 addresses = addresses[0:n]
 443                 self.accounts[key][0] = addresses
 444
 445             self.gap_limit = value
 446             self.config.set_key('gap_limit', self.gap_limit, True)
 447             self.save_accounts()
 448             return True
 449         else:
 450             return False
 451
 452     def num_unused_trailing_addresses(self, addresses):
 453         k = 0
 454         for a in addresses[::-1]:
 455             if self.history.get(a):break
 456             k = k + 1
 457         return k
 458
 459     def min_acceptable_gap(self):
 460         # fixme: this assumes wallet is synchronized
 461         n = 0
 462         nmax = 0
 463
 464         for account in self.accounts.values():
 465             addresses = account.get_addresses(0)
 466             k = self.num_unused_trailing_addresses(addresses)
 467             for a in addresses[0:-k]:
 468                 if self.history.get(a):
 469                     n = 0
 470                 else:
 471                     n += 1
 472                     if n > nmax: nmax = n
 473         return nmax + 1
 474
 475
 476     def address_is_old(self, address):
 477         age = -1
 478         h = self.history.get(address, [])
 479         if h == ['*']:
 480             return True
 481         for tx_hash, tx_height in h:
 482             if tx_height == 0:
 483                 tx_age = 0
 484             else:
 485                 tx_age = self.verifier.height - tx_height + 1
 486             if tx_age > age:
 487                 age = tx_age
 488         return age > 2
 489
 490
 491     def synchronize_sequence(self, account, for_change):
 492         limit = self.gap_limit_for_change if for_change else self.gap_limit
 493         new_addresses = []
 494         while True:
 495             addresses = account.get_addresses(for_change)
 496             if len(addresses) < limit:
 497                 address = account.create_new_address(for_change)
 498                 self.history[address] = []
 499                 new_addresses.append( address )
 500                 continue
 501
 502             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 503                 break
 504             else:
 505                 address = account.create_new_address(for_change)
 506                 self.history[address] = []
 507                 new_addresses.append( address )
 508
 509         return new_addresses
 510
 511
 512     def synchronize_account(self, account):
 513         new = []
 514         new += self.synchronize_sequence(account, 0)
 515         new += self.synchronize_sequence(account, 1)
 516         return new
 517
 518     def synchronize(self):
 519         new = []
 520         for account in self.accounts.values():
 521             new += self.synchronize_account(account)
 522         if new:
 523             self.save_accounts()
 524             self.config.set_key('addr_history', self.history, True)
 525         return new
 526
 527
 528     def is_found(self):
 529         return self.history.values() != [[]] * len(self.history)
 530
 531
 532     def add_contact(self, address, label=None):
 533         self.addressbook.append(address)
 534         self.config.set_key('contacts', self.addressbook, True)
 535         if label:
 536             self.labels[address] = label
 537             self.config.set_key('labels', self.labels)
 538
 539     def delete_contact(self, addr):
 540         if addr in self.addressbook:
 541             self.addressbook.remove(addr)
 542             self.config.set_key('addressbook', self.addressbook, True)
 543
 544
 545     def fill_addressbook(self):
 546         for tx_hash, tx in self.transactions.items():
 547             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 548             if is_send:
 549                 for addr, v in tx.outputs:
 550                     if not self.is_mine(addr) and addr not in self.addressbook:
 551                         self.addressbook.append(addr)
 552         # redo labels
 553         # self.update_tx_labels()
 554
 555     def get_num_tx(self, address):
 556         n = 0
 557         for tx in self.transactions.values():
 558             if address in map(lambda x:x[0], tx.outputs): n += 1
 559         return n
 560
 561
 562     def get_address_flags(self, addr):
 563         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 564         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 565         return flags
 566
 567
 568     def get_tx_value(self, tx, account=None):
 569         domain = self.get_account_addresses(account)
 570         return tx.get_value(domain, self.prevout_values)
 571
 572
 573     def update_tx_outputs(self, tx_hash):
 574         tx = self.transactions.get(tx_hash)
 575
 576         for i, (addr, value) in enumerate(tx.outputs):
 577             key = tx_hash+ ':%d'%i
 578             self.prevout_values[key] = value
 579
 580         for item in tx.inputs:
 581             if self.is_mine(item.get('address')):
 582                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 583                 self.spent_outputs.append(key)
 584
 585
 586     def get_addr_balance(self, address):
 587         assert self.is_mine(address)
 588         h = self.history.get(address,[])
 589         if h == ['*']: return 0,0
 590         c = u = 0
 591         received_coins = []   # list of coins received at address
 592
 593         for tx_hash, tx_height in h:
 594             tx = self.transactions.get(tx_hash)
 595             if not tx: continue
 596
 597             for i, (addr, value) in enumerate(tx.outputs):
 598                 if addr == address:
 599                     key = tx_hash + ':%d'%i
 600                     received_coins.append(key)
 601
 602         for tx_hash, tx_height in h:
 603             tx = self.transactions.get(tx_hash)
 604             if not tx: continue
 605             v = 0
 606
 607             for item in tx.inputs:
 608                 addr = item.get('address')
 609                 if addr == address:
 610                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 611                     value = self.prevout_values.get( key )
 612                     if key in received_coins:
 613                         v -= value
 614
 615             for i, (addr, value) in enumerate(tx.outputs):
 616                 key = tx_hash + ':%d'%i
 617                 if addr == address:
 618                     v += value
 619
 620             if tx_height:
 621                 c += v
 622             else:
 623                 u += v
 624         return c, u
 625
 626
 627     def get_accounts(self):
 628         accounts = {}
 629         for k, account in self.accounts.items():
 630             accounts[k] = account.name
 631         if self.imported_keys:
 632             accounts[-1] = 'Imported keys'
 633         return accounts
 634
 635     def get_account_addresses(self, a, include_change=True):
 636         if a is None:
 637             o = self.addresses(True)
 638         elif a == -1:
 639             o = self.imported_keys.keys()
 640         else:
 641             ac = self.accounts[a]
 642             o = ac.get_addresses(0)
 643             if include_change: o += ac.get_addresses(1)
 644         return o
 645
 646     def get_imported_balance(self):
 647         cc = uu = 0
 648         for addr in self.imported_keys.keys():
 649             c, u = self.get_addr_balance(addr)
 650             cc += c
 651             uu += u
 652         return cc, uu
 653
 654     def get_account_balance(self, account):
 655         if account is None:
 656             return self.get_balance()
 657         elif account == -1:
 658             return self.get_imported_balance()
 659
 660         conf = unconf = 0
 661         for addr in self.get_account_addresses(account):
 662             c, u = self.get_addr_balance(addr)
 663             conf += c
 664             unconf += u
 665         return conf, unconf
 666
 667     def get_frozen_balance(self):
 668         conf = unconf = 0
 669         for addr in self.frozen_addresses:
 670             c, u = self.get_addr_balance(addr)
 671             conf += c
 672             unconf += u
 673         return conf, unconf
 674
 675
 676     def get_balance(self):
 677         cc = uu = 0
 678         for a in self.accounts.keys():
 679             c, u = self.get_account_balance(a)
 680             cc += c
 681             uu += u
 682         c, u = self.get_imported_balance()
 683         cc += c
 684         uu += u
 685         return cc, uu
 686
 687
 688     def get_unspent_coins(self, domain=None):
 689         coins = []
 690         if domain is None: domain = self.addresses(True)
 691         for addr in domain:
 692             h = self.history.get(addr, [])
 693             if h == ['*']: continue
 694             for tx_hash, tx_height in h:
 695                 tx = self.transactions.get(tx_hash)
 696                 if tx is None: raise BaseException("Wallet not synchronized")
 697                 for output in tx.d.get('outputs'):
 698                     if output.get('address') != addr: continue
 699                     key = tx_hash + ":%d" % output.get('index')
 700                     if key in self.spent_outputs: continue
 701                     output['tx_hash'] = tx_hash
 702                     coins.append(output)
 703         return coins
 704
 705
 706
 707     def choose_tx_inputs( self, amount, fixed_fee, account = None ):
 708         """ todo: minimize tx size """
 709         total = 0
 710         fee = self.fee if fixed_fee is None else fixed_fee
 711         domain = self.get_account_addresses(account)
 712         coins = []
 713         prioritized_coins = []
 714         for i in self.frozen_addresses:
 715             if i in domain: domain.remove(i)
 716
 717         for i in self.prioritized_addresses:
 718             if i in domain: domain.remove(i)
 719
 720         coins = self.get_unspent_coins(domain)
 721         prioritized_coins = self.get_unspent_coins(self.prioritized_addresses)
 722
 723         inputs = []
 724         coins = prioritized_coins + coins
 725
 726         for item in coins:
 727             addr = item.get('address')
 728             v = item.get('value')
 729             total += v
 730             inputs.append( item )
 731             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
 732             if total >= amount + fee: break
 733         else:
 734             inputs = []
 735
 736         return inputs, total, fee
 737
 738
 739     def estimated_fee(self, inputs):
 740         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
 741         fee = self.fee * int(round(estimated_size/1024.))
 742         if fee == 0: fee = self.fee
 743         return fee
 744
 745
 746     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None, account=0 ):
 747         "add change to a transaction"
 748         change_amount = total - ( amount + fee )
 749         if change_amount != 0:
 750             if not change_addr:
 751                 if account is None:
 752                     # send change to one of the accounts involved in the tx
 753                     address = inputs[0].get('address')
 754                     account, _ = self.get_address_index(address)
 755
 756                 if not self.use_change or account == -1:
 757                     change_addr = inputs[-1]['address']
 758                 else:
 759                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
 760
 761             # Insert the change output at a random position in the outputs
 762             posn = random.randint(0, len(outputs))
 763             outputs[posn:posn] = [( change_addr,  change_amount)]
 764         return outputs
 765
 766
 767     def get_history(self, address):
 768         with self.lock:
 769             return self.history.get(address)
 770
 771
 772     def get_status(self, h):
 773         if not h: return None
 774         if h == ['*']: return '*'
 775         status = ''
 776         for tx_hash, height in h:
 777             status += tx_hash + ':%d:' % height
 778         return hashlib.sha256( status ).digest().encode('hex')
 779
 780
 781     def receive_tx_callback(self, tx_hash, tx, tx_height):
 782         if not self.check_new_tx(tx_hash, tx):
 783             # may happen due to pruning
 784             print_error("received transaction that is no longer referenced in history", tx_hash)
 785             return
 786
 787         with self.transaction_lock:
 788             self.transactions[tx_hash] = tx
 789
 790             self.interface.pending_transactions_for_notifications.append(tx)
 791
 792             self.save_transactions()
 793             if self.verifier and tx_height>0:
 794                 self.verifier.add(tx_hash, tx_height)
 795             self.update_tx_outputs(tx_hash)
 796
 797
 798     def save_transactions(self):
 799         tx = {}
 800         for k,v in self.transactions.items():
 801             tx[k] = str(v)
 802         self.config.set_key('transactions', tx, True)
 803
 804     def receive_history_callback(self, addr, hist):
 805
 806         if not self.check_new_history(addr, hist):
 807             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
 808
 809         with self.lock:
 810             self.history[addr] = hist
 811             self.config.set_key('addr_history', self.history, True)
 812
 813         if hist != ['*']:
 814             for tx_hash, tx_height in hist:
 815                 if tx_height>0:
 816                     # add it in case it was previously unconfirmed
 817                     if self.verifier: self.verifier.add(tx_hash, tx_height)
 818
 819
 820     def get_tx_history(self, account=None):
 821         with self.transaction_lock:
 822             history = self.transactions.items()
 823             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
 824             result = []
 825
 826             balance = 0
 827             for tx_hash, tx in history:
 828                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
 829                 if v is not None: balance += v
 830
 831             c, u = self.get_account_balance(account)
 832
 833             if balance != c+u:
 834                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
 835
 836             balance = c + u - balance
 837             for tx_hash, tx in history:
 838                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
 839                 if not is_relevant:
 840                     continue
 841                 if value is not None:
 842                     balance += value
 843
 844                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
 845                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
 846
 847         return result
 848
 849
 850     def get_label(self, tx_hash):
 851         label = self.labels.get(tx_hash)
 852         is_default = (label == '') or (label is None)
 853         if is_default: label = self.get_default_label(tx_hash)
 854         return label, is_default
 855
 856
 857     def get_default_label(self, tx_hash):
 858         tx = self.transactions.get(tx_hash)
 859         default_label = ''
 860         if tx:
 861             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
 862             if is_mine:
 863                 for o in tx.outputs:
 864                     o_addr, _ = o
 865                     if not self.is_mine(o_addr):
 866                         try:
 867                             default_label = self.labels[o_addr]
 868                         except KeyError:
 869                             default_label = o_addr
 870                         break
 871                 else:
 872                     default_label = '(internal)'
 873             else:
 874                 for o in tx.outputs:
 875                     o_addr, _ = o
 876                     if self.is_mine(o_addr) and not self.is_change(o_addr):
 877                         break
 878                 else:
 879                     for o in tx.outputs:
 880                         o_addr, _ = o
 881                         if self.is_mine(o_addr):
 882                             break
 883                     else:
 884                         o_addr = None
 885
 886                 if o_addr:
 887                     dest_label = self.labels.get(o_addr)
 888                     try:
 889                         default_label = self.labels[o_addr]
 890                     except KeyError:
 891                         default_label = o_addr
 892
 893         return default_label
 894
 895
 896     def mktx(self, outputs, password, fee=None, change_addr=None, account=None ):
 897         """
 898         create a transaction
 899         account parameter:
 900            None means use all accounts
 901            -1 means imported keys
 902            0, 1, etc are seed accounts
 903         """
 904
 905         for address, x in outputs:
 906             assert is_valid(address)
 907
 908         amount = sum( map(lambda x:x[1], outputs) )
 909
 910         inputs, total, fee = self.choose_tx_inputs( amount, fee, account )
 911         if not inputs:
 912             raise ValueError("Not enough funds")
 913
 914         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr, account)
 915
 916         tx = Transaction.from_io(inputs, outputs)
 917
 918
 919         keypairs = {}
 920         for i, txin in enumerate(tx.inputs):
 921             address = txin['address']
 922             if address in self.imported_keys.keys():
 923                 pk_addresses.append(address)
 924                 continue
 925             account, sequence = self.get_address_index(address)
 926             txin['KeyID'] = (account, 'BIP32', sequence) # used by the server to find the key
 927
 928             redeemScript = self.accounts[account].redeem_script(sequence)
 929             if redeemScript:
 930                 txin['redeemScript'] = redeemScript
 931                 assert address == self.accounts[account].get_address(*sequence)
 932             else:
 933                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
 934
 935             private_keys = self.get_private_key(address, password)
 936             for sec in private_keys:
 937                 pubkey = public_key_from_private_key(sec)
 938                 keypairs[ pubkey ] = sec
 939
 940         tx.sign(keypairs)
 941
 942         for address, x in outputs:
 943             if address not in self.addressbook and not self.is_mine(address):
 944                 self.addressbook.append(address)
 945
 946         return tx
 947
 948
 949
 950     def sendtx(self, tx):
 951         # synchronous
 952         h = self.send_tx(tx)
 953         self.tx_event.wait()
 954         return self.receive_tx(h)
 955
 956     def send_tx(self, tx):
 957         # asynchronous
 958         self.tx_event.clear()
 959         self.interface.send([('blockchain.transaction.broadcast', [str(tx)])], 'synchronizer')
 960         return tx.hash()
 961
 962     def receive_tx(self,tx_hash):
 963         out = self.tx_result
 964         if out != tx_hash:
 965             return False, "error: " + out
 966         return True, out
 967
 968
 969
 970     def update_password(self, seed, old_password, new_password):
 971         if new_password == '': new_password = None
 972         # this will throw an exception if unicode cannot be converted
 973         self.seed = pw_encode( seed, new_password)
 974         self.config.set_key('seed', self.seed, True)
 975         self.use_encryption = (new_password != None)
 976         self.config.set_key('use_encryption', self.use_encryption,True)
 977         for k in self.imported_keys.keys():
 978             a = self.imported_keys[k]
 979             b = pw_decode(a, old_password)
 980             c = pw_encode(b, new_password)
 981             self.imported_keys[k] = c
 982         self.config.set_key('imported_keys', self.imported_keys, True)
 983
 984         for k, v in self.master_private_keys.items():
 985             b = pw_decode(v, old_password)
 986             c = pw_encode(b, new_password)
 987             self.master_private_keys[k] = c
 988         self.config.set_key('master_private_keys', self.master_private_keys, True)
 989
 990
 991     def freeze(self,addr):
 992         if self.is_mine(addr) and addr not in self.frozen_addresses:
 993             self.unprioritize(addr)
 994             self.frozen_addresses.append(addr)
 995             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 996             return True
 997         else:
 998             return False
 999
1000     def unfreeze(self,addr):
1001         if self.is_mine(addr) and addr in self.frozen_addresses:
1002             self.frozen_addresses.remove(addr)
1003             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
1004             return True
1005         else:
1006             return False
1007
1008     def prioritize(self,addr):
1009         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1010             self.unfreeze(addr)
1011             self.prioritized_addresses.append(addr)
1012             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1013             return True
1014         else:
1015             return False
1016
1017     def unprioritize(self,addr):
1018         if self.is_mine(addr) and addr in self.prioritized_addresses:
1019             self.prioritized_addresses.remove(addr)
1020             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1021             return True
1022         else:
1023             return False
1024
1025     def set_fee(self, fee):
1026         if self.fee != fee:
1027             self.fee = fee
1028             self.config.set_key('fee_per_kb', self.fee, True)
1029
1030
1031     def save(self):
1032         print_error("Warning: wallet.save() is deprecated")
1033         tx = {}
1034         for k,v in self.transactions.items():
1035             tx[k] = str(v)
1036
1037         s = {
1038             'use_change': self.use_change,
1039             'fee_per_kb': self.fee,
1040             'addr_history': self.history,
1041             'labels': self.labels,
1042             'contacts': self.addressbook,
1043             'num_zeros': self.num_zeros,
1044             'frozen_addresses': self.frozen_addresses,
1045             'prioritized_addresses': self.prioritized_addresses,
1046             'gap_limit': self.gap_limit,
1047             'transactions': tx,
1048         }
1049         for k, v in s.items():
1050             self.config.set_key(k,v)
1051         self.config.save()
1052
1053     def set_verifier(self, verifier):
1054         self.verifier = verifier
1055
1056         # review transactions that are in the history
1057         for addr, hist in self.history.items():
1058             if hist == ['*']: continue
1059             for tx_hash, tx_height in hist:
1060                 if tx_height>0:
1061                     # add it in case it was previously unconfirmed
1062                     self.verifier.add(tx_hash, tx_height)
1063
1064
1065         # if we are on a pruning server, remove unverified transactions
1066         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1067         for tx_hash in self.transactions.keys():
1068             if tx_hash not in vr:
1069                 self.transactions.pop(tx_hash)
1070
1071
1072
1073     def check_new_history(self, addr, hist):
1074
1075         # check that all tx in hist are relevant
1076         if hist != ['*']:
1077             for tx_hash, height in hist:
1078                 tx = self.transactions.get(tx_hash)
1079                 if not tx: continue
1080                 if not tx.has_address(addr):
1081                     return False
1082
1083         # check that we are not "orphaning" a transaction
1084         old_hist = self.history.get(addr,[])
1085         if old_hist == ['*']: return True
1086
1087         for tx_hash, height in old_hist:
1088             if tx_hash in map(lambda x:x[0], hist): continue
1089             found = False
1090             for _addr, _hist in self.history.items():
1091                 if _addr == addr: continue
1092                 if _hist == ['*']: continue
1093                 _tx_hist = map(lambda x:x[0], _hist)
1094                 if tx_hash in _tx_hist:
1095                     found = True
1096                     break
1097
1098             if not found:
1099                 tx = self.transactions.get(tx_hash)
1100                 # tx might not be there
1101                 if not tx: continue
1102
1103                 # already verified?
1104                 if self.verifier.get_height(tx_hash):
1105                     continue
1106                 # unconfirmed tx
1107                 print_error("new history is orphaning transaction:", tx_hash)
1108                 # check that all outputs are not mine, request histories
1109                 ext_requests = []
1110                 for _addr, _v in tx.outputs:
1111                     # assert not self.is_mine(_addr)
1112                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1113
1114                 ext_h = self.interface.synchronous_get(ext_requests)
1115                 print_error("sync:", ext_requests, ext_h)
1116                 height = None
1117                 for h in ext_h:
1118                     if h == ['*']: continue
1119                     for item in h:
1120                         if item.get('tx_hash') == tx_hash:
1121                             height = item.get('height')
1122                 if height:
1123                     print_error("found height for", tx_hash, height)
1124                     self.verifier.add(tx_hash, height)
1125                 else:
1126                     print_error("removing orphaned tx from history", tx_hash)
1127                     self.transactions.pop(tx_hash)
1128
1129         return True
1130
1131
1132
1133     def check_new_tx(self, tx_hash, tx):
1134         # 1 check that tx is referenced in addr_history.
1135         addresses = []
1136         for addr, hist in self.history.items():
1137             if hist == ['*']:continue
1138             for txh, height in hist:
1139                 if txh == tx_hash:
1140                     addresses.append(addr)
1141
1142         if not addresses:
1143             return False
1144
1145         # 2 check that referencing addresses are in the tx
1146         for addr in addresses:
1147             if not tx.has_address(addr):
1148                 return False
1149
1150         return True
1151
1152
1153
1154
1155 class WalletSynchronizer(threading.Thread):
1156
1157
1158     def __init__(self, wallet, config):
1159         threading.Thread.__init__(self)
1160         self.daemon = True
1161         self.wallet = wallet
1162         wallet.synchronizer = self
1163         self.interface = self.wallet.interface
1164         self.interface.register_channel('synchronizer')
1165         self.wallet.interface.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1166         self.was_updated = True
1167         self.running = False
1168         self.lock = threading.Lock()
1169
1170     def stop(self):
1171         with self.lock: self.running = False
1172         self.interface.poke('synchronizer')
1173
1174     def is_running(self):
1175         with self.lock: return self.running
1176
1177
1178     def subscribe_to_addresses(self, addresses):
1179         messages = []
1180         for addr in addresses:
1181             messages.append(('blockchain.address.subscribe', [addr]))
1182         self.interface.send( messages, 'synchronizer')
1183
1184
1185     def run(self):
1186         with self.lock: self.running = True
1187
1188         requested_tx = []
1189         missing_tx = []
1190         requested_histories = {}
1191
1192         # request any missing transactions
1193         for history in self.wallet.history.values():
1194             if history == ['*']: continue
1195             for tx_hash, tx_height in history:
1196                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1197                     missing_tx.append( (tx_hash, tx_height) )
1198         print_error("missing tx", missing_tx)
1199
1200         # wait until we are connected, in case the user is not connected
1201         while not self.interface.is_connected:
1202             time.sleep(1)
1203
1204         # subscriptions
1205         self.subscribe_to_addresses(self.wallet.addresses(True))
1206
1207         while self.is_running():
1208             # 1. create new addresses
1209             new_addresses = self.wallet.synchronize()
1210
1211             # request missing addresses
1212             if new_addresses:
1213                 self.subscribe_to_addresses(new_addresses)
1214
1215             # request missing transactions
1216             for tx_hash, tx_height in missing_tx:
1217                 if (tx_hash, tx_height) not in requested_tx:
1218                     self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
1219                     requested_tx.append( (tx_hash, tx_height) )
1220             missing_tx = []
1221
1222             # detect if situation has changed
1223             if not self.interface.is_up_to_date('synchronizer'):
1224                 if self.wallet.is_up_to_date():
1225                     self.wallet.set_up_to_date(False)
1226                     self.was_updated = True
1227             else:
1228                 if not self.wallet.is_up_to_date():
1229                     self.wallet.set_up_to_date(True)
1230                     self.was_updated = True
1231
1232             if self.was_updated:
1233                 self.interface.trigger_callback('updated')
1234                 self.was_updated = False
1235
1236             # 2. get a response
1237             r = self.interface.get_response('synchronizer')
1238
1239             # poke sends None. (needed during stop)
1240             if not r: continue
1241
1242             # 3. handle response
1243             method = r['method']
1244             params = r['params']
1245             result = r.get('result')
1246             error = r.get('error')
1247             if error:
1248                 print "error", r
1249                 continue
1250
1251             if method == 'blockchain.address.subscribe':
1252                 addr = params[0]
1253                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1254                     if requested_histories.get(addr) is None:
1255                         self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
1256                         requested_histories[addr] = result
1257
1258             elif method == 'blockchain.address.get_history':
1259                 addr = params[0]
1260                 print_error("receiving history", addr, result)
1261                 if result == ['*']:
1262                     assert requested_histories.pop(addr) == '*'
1263                     self.wallet.receive_history_callback(addr, result)
1264                 else:
1265                     hist = []
1266                     # check that txids are unique
1267                     txids = []
1268                     for item in result:
1269                         tx_hash = item['tx_hash']
1270                         if tx_hash not in txids:
1271                             txids.append(tx_hash)
1272                             hist.append( (tx_hash, item['height']) )
1273
1274                     if len(hist) != len(result):
1275                         raise BaseException("error: server sent history with non-unique txid", result)
1276
1277                     # check that the status corresponds to what was announced
1278                     rs = requested_histories.pop(addr)
1279                     if self.wallet.get_status(hist) != rs:
1280                         raise BaseException("error: status mismatch: %s"%addr)
1281
1282                     # store received history
1283                     self.wallet.receive_history_callback(addr, hist)
1284
1285                     # request transactions that we don't have
1286                     for tx_hash, tx_height in hist:
1287                         if self.wallet.transactions.get(tx_hash) is None:
1288                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1289                                 missing_tx.append( (tx_hash, tx_height) )
1290
1291             elif method == 'blockchain.transaction.get':
1292                 tx_hash = params[0]
1293                 tx_height = params[1]
1294                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1295                 tx = Transaction(result)
1296                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1297                 self.was_updated = True
1298                 requested_tx.remove( (tx_hash, tx_height) )
1299                 print_error("received tx:", tx_hash, len(tx.raw))
1300
1301             elif method == 'blockchain.transaction.broadcast':
1302                 self.wallet.tx_result = result
1303                 self.wallet.tx_event.set()
1304
1305             else:
1306                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1307
1308             if self.was_updated and not requested_tx:
1309                 self.interface.trigger_callback('updated')
1310                 self.interface.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1311
1312
1313                 self.was_updated = False