lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, user_dir, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36
  37 # AES encryption
  38 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  39 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  40
  41 def pw_encode(s, password):
  42     if password:
  43         secret = Hash(password)
  44         return EncodeAES(secret, s)
  45     else:
  46         return s
  47
  48 def pw_decode(s, password):
  49     if password is not None:
  50         secret = Hash(password)
  51         try:
  52             d = DecodeAES(secret, s)
  53         except:
  54             raise BaseException('Invalid password')
  55         return d
  56     else:
  57         return s
  58
  59
  60
  61
  62
  63 from version import ELECTRUM_VERSION, SEED_VERSION
  64
  65
  66 class Wallet:
  67     def __init__(self, config={}):
  68
  69         self.config = config
  70         self.electrum_version = ELECTRUM_VERSION
  71         self.gap_limit_for_change = 3 # constant
  72
  73         # saved fields
  74         self.seed_version          = config.get('seed_version', SEED_VERSION)
  75         self.gap_limit             = config.get('gap_limit', 5)
  76         self.use_change            = config.get('use_change',True)
  77         self.fee                   = int(config.get('fee_per_kb',50000))
  78         self.num_zeros             = int(config.get('num_zeros',0))
  79         self.use_encryption        = config.get('use_encryption', False)
  80         self.seed                  = config.get('seed', '')               # encrypted
  81         self.labels                = config.get('labels', {})
  82         self.frozen_addresses      = config.get('frozen_addresses',[])
  83         self.prioritized_addresses = config.get('prioritized_addresses',[])
  84         self.addressbook           = config.get('contacts', [])
  85
  86         self.imported_keys         = config.get('imported_keys',{})
  87         self.history               = config.get('addr_history',{})        # address -> list(txid, height)
  88
  89
  90         self.master_public_keys = config.get('master_public_keys',{})
  91         self.master_private_keys = config.get('master_private_keys', {})
  92
  93         self.load_accounts(config)
  94
  95         self.transactions = {}
  96         tx = config.get('transactions',{})
  97         try:
  98             for k,v in tx.items(): self.transactions[k] = Transaction(v)
  99         except:
 100             print_msg("Warning: Cannot deserialize transactions. skipping")
 101
 102         # not saved
 103         self.prevout_values = {}     # my own transaction outputs
 104         self.spent_outputs = []
 105
 106         # spv
 107         self.verifier = None
 108
 109         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 110         # interface.is_up_to_date() returns true when all requests have been answered and processed
 111         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 112
 113         self.up_to_date = False
 114         self.lock = threading.Lock()
 115         self.transaction_lock = threading.Lock()
 116         self.tx_event = threading.Event()
 117
 118         if self.seed_version != SEED_VERSION:
 119             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
 120
 121         for tx_hash, tx in self.transactions.items():
 122             if self.check_new_tx(tx_hash, tx):
 123                 self.update_tx_outputs(tx_hash)
 124             else:
 125                 print_error("unreferenced tx", tx_hash)
 126                 self.transactions.pop(tx_hash)
 127
 128
 129     def set_up_to_date(self,b):
 130         with self.lock: self.up_to_date = b
 131
 132     def is_up_to_date(self):
 133         with self.lock: return self.up_to_date
 134
 135     def update(self):
 136         self.up_to_date = False
 137         self.interface.poke('synchronizer')
 138         while not self.is_up_to_date(): time.sleep(0.1)
 139
 140     def import_key(self, sec, password):
 141         # check password
 142         seed = self.decode_seed(password)
 143         try:
 144             address = address_from_private_key(sec)
 145         except:
 146             raise BaseException('Invalid private key')
 147
 148         if self.is_mine(address):
 149             raise BaseException('Address already in wallet')
 150
 151         # store the originally requested keypair into the imported keys table
 152         self.imported_keys[address] = pw_encode(sec, password )
 153         self.config.set_key('imported_keys', self.imported_keys, True)
 154         return address
 155
 156     def delete_imported_key(self, addr):
 157         if addr in self.imported_keys:
 158             self.imported_keys.pop(addr)
 159             self.config.set_key('imported_keys', self.imported_keys, True)
 160
 161
 162     def init_seed(self, seed):
 163         if self.seed: raise BaseException("a seed exists")
 164         if not seed:
 165             seed = random_seed(128)
 166         self.seed = seed
 167
 168
 169     def save_seed(self):
 170         self.config.set_key('seed', self.seed, True)
 171         self.config.set_key('seed_version', self.seed_version, True)
 172
 173         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 174
 175         # normal accounts
 176         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 177         # p2sh 2of2
 178         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 179         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 180         # p2sh 2of3
 181         k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 182         k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 183         k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 184
 185         self.master_public_keys = {
 186             "m/0'/": (c0, K0, cK0),
 187             "m/1'/": (c1, K1, cK1),
 188             "m/2'/": (c2, K2, cK2),
 189             "m/3'/": (c3, K3, cK3),
 190             "m/4'/": (c4, K4, cK4),
 191             "m/5'/": (c5, K5, cK5)
 192             }
 193
 194         self.master_private_keys = {
 195             "m/0'/": k0,
 196             "m/1'/": k1,
 197             "m/2'/": k2,
 198             "m/3'/": k3,
 199             "m/4'/": k4,
 200             "m/5'/": k5
 201             }
 202
 203         self.config.set_key('master_public_keys', self.master_public_keys, True)
 204         self.config.set_key('master_private_keys', self.master_private_keys, True)
 205
 206         # create default account
 207         self.create_account('Main account')
 208
 209
 210     def account_id(self, account_type, i):
 211         if account_type is None:
 212             return "m/0'/%d'"%i
 213         elif account_type == '2of2':
 214             return "m/1'/%d & m/2'/%d"%(i,i)
 215         elif account_type == '2of3':
 216             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 217         else:
 218             raise BaseException('unknown account type')
 219
 220
 221     def num_accounts(self, account_type):
 222         keys = self.accounts.keys()
 223         i = 0
 224         while True:
 225             account_id = self.account_id(account_type, i)
 226             if account_id not in keys: break
 227             i += 1
 228         return i
 229
 230
 231     def create_account(self, name, account_type = None):
 232         i = self.num_accounts(account_type)
 233         acount_id = self.account_id(account_type,i)
 234
 235         if account_type is None:
 236             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 237             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 238             account = BIP32_Account({ 'name':name, 'c':c0, 'K':K0, 'cK':cK0 })
 239
 240         elif account_type == '2of2':
 241             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 242             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 243             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 244             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 245             account = BIP32_Account_2of2({ 'name':name, 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 246
 247         elif account_type == '2of3':
 248             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 249             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 250             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 251             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 252             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 253             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 254             account = BIP32_Account_2of3({ 'name':name, 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 255
 256         self.accounts[account_id] = account
 257         self.save_accounts()
 258
 259
 260     def save_accounts(self):
 261         d = {}
 262         for k, v in self.accounts.items():
 263             d[k] = v.dump()
 264         self.config.set_key('accounts', d, True)
 265
 266
 267     def load_accounts(self, config):
 268         d = config.get('accounts', {})
 269         self.accounts = {}
 270         for k, v in d.items():
 271             if '&' in k:
 272                 self.accounts[k] = BIP32_Account_2of2(v)
 273             else:
 274                 self.accounts[k] = BIP32_Account(v)
 275
 276
 277
 278
 279     def addresses(self, include_change = True):
 280         o = self.get_account_addresses(-1, include_change)
 281         for a in self.accounts.keys():
 282             o += self.get_account_addresses(a, include_change)
 283         return o
 284
 285
 286     def is_mine(self, address):
 287         return address in self.addresses(True)
 288
 289     def is_change(self, address):
 290         if not self.is_mine(address): return False
 291         if address in self.imported_keys.keys(): return False
 292         acct, s = self.get_address_index(address)
 293         return s[0] == 1
 294
 295     def get_master_public_key(self):
 296         raise
 297         return self.config.get("master_public_key")
 298
 299     def get_master_private_key(self, account, password):
 300         master_k = pw_decode( self.master_private_keys[account], password)
 301         master_c, master_K, master_Kc = self.master_public_keys[account]
 302         try:
 303             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 304             assert K.encode('hex') == master_K
 305         except:
 306             raise BaseException("Invalid password")
 307         return master_k
 308
 309
 310     def get_address_index(self, address):
 311         if address in self.imported_keys.keys():
 312             return -1, None
 313         for account in self.accounts.keys():
 314             for for_change in [0,1]:
 315                 addresses = self.accounts[account].get_addresses(for_change)
 316                 for addr in addresses:
 317                     if address == addr:
 318                         return account, (for_change, addresses.index(addr))
 319         raise BaseException("not found")
 320
 321
 322     def get_public_key(self, address):
 323         account, sequence = self.get_address_index(address)
 324         return self.accounts[account].get_pubkey( *sequence )
 325
 326
 327     def decode_seed(self, password):
 328         seed = pw_decode(self.seed, password)
 329         #todo:  #self.sequences[0].check_seed(seed)
 330         return seed
 331
 332
 333     def get_private_key(self, address, password):
 334         out = []
 335         if address in self.imported_keys.keys():
 336             out.append( pw_decode( self.imported_keys[address], password ) )
 337         else:
 338             account, sequence = self.get_address_index(address)
 339             # assert address == self.accounts[account].get_address(*sequence)
 340             l = account.split("&")
 341             for s in l:
 342                 s = s.strip()
 343                 m = re.match("(m/\d+'/)(\d+)", s)
 344                 if m:
 345                     root = m.group(1)
 346                     if root not in self.master_private_keys.keys(): continue
 347                     num = int(m.group(2))
 348                     master_k = self.get_master_private_key(root, password)
 349                     master_c, _, _ = self.master_public_keys[root]
 350                     pk = bip32_private_key( (num,) + sequence, master_k.decode('hex'), master_c.decode('hex'))
 351                     out.append(pk)
 352
 353         return out
 354
 355
 356
 357
 358     def signrawtransaction(self, tx, input_info, private_keys, password):
 359         import deserialize
 360         unspent_coins = self.get_unspent_coins()
 361         seed = self.decode_seed(password)
 362
 363         # build a list of public/private keys
 364         keypairs = {}
 365         for sec in private_keys:
 366             pubkey = public_key_from_private_key(sec)
 367             keypairs[ pubkey ] = sec
 368
 369
 370         for txin in tx.inputs:
 371             # convert to own format
 372             txin['tx_hash'] = txin['prevout_hash']
 373             txin['index'] = txin['prevout_n']
 374
 375             for item in input_info:
 376                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 377                     txin['raw_output_script'] = item['scriptPubKey']
 378                     txin['redeemScript'] = item.get('redeemScript')
 379                     txin['KeyID'] = item.get('KeyID')
 380                     break
 381             else:
 382                 for item in unspent_coins:
 383                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 384                         txin['raw_output_script'] = item['raw_output_script']
 385                         break
 386                 else:
 387                     # if neither, we might want to get it from the server..
 388                     raise
 389
 390             # find the address and fill private_keys
 391             if txin.get('KeyID'):
 392                 account, name, sequence = txin.get('KeyID')
 393                 if name != 'BIP32': continue
 394                 sec = self.accounts[account].get_private_key(sequence, seed)
 395                 pubkey = self.accounts[account].get_pubkey(sequence)
 396                 txin['address'] = addr
 397                 keypairs[pubkey] = [sec]
 398
 399             redeem_script = txin.get("redeemScript")
 400             if redeem_script:
 401                 num, redeem_pubkeys = deserialize.parse_redeemScript(redeem_script)
 402                 addr = hash_160_to_bc_address(hash_160(redeem_script.decode('hex')), 5)
 403                 txin['address'] = addr
 404
 405             elif txin.get("raw_output_script"):
 406                 addr = deserialize.get_address_from_output_script(txin.get("raw_output_script").decode('hex'))
 407                 sec = self.get_private_key(addr, password)
 408                 pubkey = public_key_from_private_key(sec)
 409                 if sec:
 410                     keypairs[pubkey] = [sec]
 411                     txin['address'] = addr
 412
 413         tx.sign( keypairs )
 414
 415     def sign_message(self, address, message, password):
 416         sec = self.get_private_key(address, password)
 417         key = regenerate_key(sec)
 418         compressed = is_compressed(sec)
 419         return key.sign_message(message, compressed, address)
 420
 421     def verify_message(self, address, signature, message):
 422         try:
 423             EC_KEY.verify_message(address, signature, message)
 424             return True
 425         except BaseException as e:
 426             print_error("Verification error: {0}".format(e))
 427             return False
 428
 429
 430     def change_gap_limit(self, value):
 431         if value >= self.gap_limit:
 432             self.gap_limit = value
 433             self.config.set_key('gap_limit', self.gap_limit, True)
 434             self.interface.poke('synchronizer')
 435             return True
 436
 437         elif value >= self.min_acceptable_gap():
 438             for key, account in self.accounts.items():
 439                 addresses = account[0]
 440                 k = self.num_unused_trailing_addresses(addresses)
 441                 n = len(addresses) - k + value
 442                 addresses = addresses[0:n]
 443                 self.accounts[key][0] = addresses
 444
 445             self.gap_limit = value
 446             self.config.set_key('gap_limit', self.gap_limit, True)
 447             self.save_accounts()
 448             return True
 449         else:
 450             return False
 451
 452     def num_unused_trailing_addresses(self, addresses):
 453         k = 0
 454         for a in addresses[::-1]:
 455             if self.history.get(a):break
 456             k = k + 1
 457         return k
 458
 459     def min_acceptable_gap(self):
 460         # fixme: this assumes wallet is synchronized
 461         n = 0
 462         nmax = 0
 463
 464         for account in self.accounts.values():
 465             addresses = account.get_addresses(0)
 466             k = self.num_unused_trailing_addresses(addresses)
 467             for a in addresses[0:-k]:
 468                 if self.history.get(a):
 469                     n = 0
 470                 else:
 471                     n += 1
 472                     if n > nmax: nmax = n
 473         return nmax + 1
 474
 475
 476     def address_is_old(self, address):
 477         age = -1
 478         h = self.history.get(address, [])
 479         if h == ['*']:
 480             return True
 481         for tx_hash, tx_height in h:
 482             if tx_height == 0:
 483                 tx_age = 0
 484             else:
 485                 tx_age = self.verifier.height - tx_height + 1
 486             if tx_age > age:
 487                 age = tx_age
 488         return age > 2
 489
 490
 491     def synchronize_sequence(self, account, for_change):
 492         limit = self.gap_limit_for_change if for_change else self.gap_limit
 493         new_addresses = []
 494         while True:
 495             addresses = account.get_addresses(for_change)
 496             if len(addresses) < limit:
 497                 address = account.create_new_address(for_change)
 498                 self.history[address] = []
 499                 new_addresses.append( address )
 500                 continue
 501
 502             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 503                 break
 504             else:
 505                 address = account.create_new_address(for_change)
 506                 self.history[address] = []
 507                 new_addresses.append( address )
 508
 509         return new_addresses
 510
 511
 512     def synchronize_account(self, account):
 513         new = []
 514         new += self.synchronize_sequence(account, 0)
 515         new += self.synchronize_sequence(account, 1)
 516         return new
 517
 518     def synchronize(self):
 519         new = []
 520         for account in self.accounts.values():
 521             new += self.synchronize_account(account)
 522         if new:
 523             self.save_accounts()
 524             self.config.set_key('addr_history', self.history, True)
 525         return new
 526
 527
 528     def is_found(self):
 529         return self.history.values() != [[]] * len(self.history)
 530
 531
 532     def add_contact(self, address, label=None):
 533         self.addressbook.append(address)
 534         self.config.set_key('contacts', self.addressbook, True)
 535         if label:
 536             self.labels[address] = label
 537             self.config.set_key('labels', self.labels)
 538
 539     def delete_contact(self, addr):
 540         if addr in self.addressbook:
 541             self.addressbook.remove(addr)
 542             self.config.set_key('addressbook', self.addressbook, True)
 543
 544
 545     def fill_addressbook(self):
 546         for tx_hash, tx in self.transactions.items():
 547             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 548             if is_send:
 549                 for addr, v in tx.outputs:
 550                     if not self.is_mine(addr) and addr not in self.addressbook:
 551                         self.addressbook.append(addr)
 552         # redo labels
 553         # self.update_tx_labels()
 554
 555     def get_num_tx(self, address):
 556         n = 0
 557         for tx in self.transactions.values():
 558             if address in map(lambda x:x[0], tx.outputs): n += 1
 559         return n
 560
 561
 562     def get_address_flags(self, addr):
 563         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 564         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 565         return flags
 566
 567
 568     def get_tx_value(self, tx, account=None):
 569         domain = self.get_account_addresses(account)
 570         return tx.get_value(domain, self.prevout_values)
 571
 572
 573     def update_tx_outputs(self, tx_hash):
 574         tx = self.transactions.get(tx_hash)
 575
 576         for i, (addr, value) in enumerate(tx.outputs):
 577             key = tx_hash+ ':%d'%i
 578             self.prevout_values[key] = value
 579
 580         for item in tx.inputs:
 581             if self.is_mine(item.get('address')):
 582                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 583                 self.spent_outputs.append(key)
 584
 585
 586     def get_addr_balance(self, address):
 587         assert self.is_mine(address)
 588         h = self.history.get(address,[])
 589         if h == ['*']: return 0,0
 590         c = u = 0
 591         received_coins = []   # list of coins received at address
 592
 593         for tx_hash, tx_height in h:
 594             tx = self.transactions.get(tx_hash)
 595             if not tx: continue
 596
 597             for i, (addr, value) in enumerate(tx.outputs):
 598                 if addr == address:
 599                     key = tx_hash + ':%d'%i
 600                     received_coins.append(key)
 601
 602         for tx_hash, tx_height in h:
 603             tx = self.transactions.get(tx_hash)
 604             if not tx: continue
 605             v = 0
 606
 607             for item in tx.inputs:
 608                 addr = item.get('address')
 609                 if addr == address:
 610                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 611                     value = self.prevout_values.get( key )
 612                     if key in received_coins:
 613                         v -= value
 614
 615             for i, (addr, value) in enumerate(tx.outputs):
 616                 key = tx_hash + ':%d'%i
 617                 if addr == address:
 618                     v += value
 619
 620             if tx_height:
 621                 c += v
 622             else:
 623                 u += v
 624         return c, u
 625
 626
 627     def get_accounts(self):
 628         accounts = {}
 629         for k, account in self.accounts.items():
 630             accounts[k] = account.name
 631         if self.imported_keys:
 632             accounts[-1] = 'Imported keys'
 633         return accounts
 634
 635     def get_account_addresses(self, a, include_change=True):
 636         if a is None:
 637             o = self.addresses(True)
 638         elif a == -1:
 639             o = self.imported_keys.keys()
 640         else:
 641             ac = self.accounts[a]
 642             o = ac.get_addresses(0)
 643             if include_change: o += ac.get_addresses(1)
 644         return o
 645
 646     def get_imported_balance(self):
 647         cc = uu = 0
 648         for addr in self.imported_keys.keys():
 649             c, u = self.get_addr_balance(addr)
 650             cc += c
 651             uu += u
 652         return cc, uu
 653
 654     def get_account_balance(self, account):
 655         if account is None:
 656             return self.get_balance()
 657         elif account == -1:
 658             return self.get_imported_balance()
 659
 660         conf = unconf = 0
 661         for addr in self.get_account_addresses(account):
 662             c, u = self.get_addr_balance(addr)
 663             conf += c
 664             unconf += u
 665         return conf, unconf
 666
 667     def get_frozen_balance(self):
 668         conf = unconf = 0
 669         for addr in self.frozen_addresses:
 670             c, u = self.get_addr_balance(addr)
 671             conf += c
 672             unconf += u
 673         return conf, unconf
 674
 675
 676     def get_balance(self):
 677         cc = uu = 0
 678         for a in self.accounts.keys():
 679             c, u = self.get_account_balance(a)
 680             cc += c
 681             uu += u
 682         c, u = self.get_imported_balance()
 683         cc += c
 684         uu += u
 685         return cc, uu
 686
 687
 688     def get_unspent_coins(self, domain=None):
 689         coins = []
 690         if domain is None: domain = self.addresses(True)
 691         for addr in domain:
 692             h = self.history.get(addr, [])
 693             if h == ['*']: continue
 694             for tx_hash, tx_height in h:
 695                 tx = self.transactions.get(tx_hash)
 696                 if tx is None: raise BaseException("Wallet not synchronized")
 697                 for output in tx.d.get('outputs'):
 698                     if output.get('address') != addr: continue
 699                     key = tx_hash + ":%d" % output.get('index')
 700                     if key in self.spent_outputs: continue
 701                     output['tx_hash'] = tx_hash
 702                     coins.append(output)
 703         return coins
 704
 705
 706
 707     def choose_tx_inputs( self, amount, fixed_fee, account = None ):
 708         """ todo: minimize tx size """
 709         total = 0
 710         fee = self.fee if fixed_fee is None else fixed_fee
 711         domain = self.get_account_addresses(account)
 712         coins = []
 713         prioritized_coins = []
 714         for i in self.frozen_addresses:
 715             if i in domain: domain.remove(i)
 716
 717         for i in self.prioritized_addresses:
 718             if i in domain: domain.remove(i)
 719
 720         coins = self.get_unspent_coins(domain)
 721         prioritized_coins = self.get_unspent_coins(self.prioritized_addresses)
 722
 723         inputs = []
 724         coins = prioritized_coins + coins
 725
 726         for item in coins:
 727             addr = item.get('address')
 728             v = item.get('value')
 729             total += v
 730             inputs.append( item )
 731             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
 732             if total >= amount + fee: break
 733         else:
 734             inputs = []
 735
 736         return inputs, total, fee
 737
 738
 739     def estimated_fee(self, inputs):
 740         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
 741         fee = self.fee * int(round(estimated_size/1024.))
 742         if fee == 0: fee = self.fee
 743         return fee
 744
 745
 746     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None, account=0 ):
 747         "add change to a transaction"
 748         change_amount = total - ( amount + fee )
 749         if change_amount != 0:
 750             if not change_addr:
 751                 if account is None:
 752                     # send change to one of the accounts involved in the tx
 753                     address = inputs[0].get('address')
 754                     account, _ = self.get_address_index(address)
 755
 756                 if not self.use_change or account == -1:
 757                     change_addr = inputs[-1]['address']
 758                 else:
 759                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
 760
 761             # Insert the change output at a random position in the outputs
 762             posn = random.randint(0, len(outputs))
 763             outputs[posn:posn] = [( change_addr,  change_amount)]
 764         return outputs
 765
 766
 767     def get_history(self, address):
 768         with self.lock:
 769             return self.history.get(address)
 770
 771
 772     def get_status(self, h):
 773         if not h: return None
 774         if h == ['*']: return '*'
 775         status = ''
 776         for tx_hash, height in h:
 777             status += tx_hash + ':%d:' % height
 778         return hashlib.sha256( status ).digest().encode('hex')
 779
 780
 781     def receive_tx_callback(self, tx_hash, tx, tx_height):
 782         if not self.check_new_tx(tx_hash, tx):
 783             # may happen due to pruning
 784             print_error("received transaction that is no longer referenced in history", tx_hash)
 785             return
 786
 787         with self.transaction_lock:
 788             self.transactions[tx_hash] = tx
 789
 790             self.interface.pending_transactions_for_notifications.append(tx)
 791
 792             self.save_transactions()
 793             if self.verifier and tx_height>0:
 794                 self.verifier.add(tx_hash, tx_height)
 795             self.update_tx_outputs(tx_hash)
 796
 797
 798     def save_transactions(self):
 799         tx = {}
 800         for k,v in self.transactions.items():
 801             tx[k] = str(v)
 802         self.config.set_key('transactions', tx, True)
 803
 804     def receive_history_callback(self, addr, hist):
 805
 806         if not self.check_new_history(addr, hist):
 807             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
 808
 809         with self.lock:
 810             self.history[addr] = hist
 811             self.config.set_key('addr_history', self.history, True)
 812
 813         if hist != ['*']:
 814             for tx_hash, tx_height in hist:
 815                 if tx_height>0:
 816                     # add it in case it was previously unconfirmed
 817                     if self.verifier: self.verifier.add(tx_hash, tx_height)
 818
 819
 820     def get_tx_history(self, account=None):
 821         with self.transaction_lock:
 822             history = self.transactions.items()
 823             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
 824             result = []
 825
 826             balance = 0
 827             for tx_hash, tx in history:
 828                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
 829                 if v is not None: balance += v
 830
 831             c, u = self.get_account_balance(account)
 832
 833             if balance != c+u:
 834                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
 835
 836             balance = c + u - balance
 837             for tx_hash, tx in history:
 838                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
 839                 if not is_relevant:
 840                     continue
 841                 if value is not None:
 842                     balance += value
 843
 844                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
 845                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
 846
 847         return result
 848
 849
 850     def get_label(self, tx_hash):
 851         label = self.labels.get(tx_hash)
 852         is_default = (label == '') or (label is None)
 853         if is_default: label = self.get_default_label(tx_hash)
 854         return label, is_default
 855
 856
 857     def get_default_label(self, tx_hash):
 858         tx = self.transactions.get(tx_hash)
 859         default_label = ''
 860         if tx:
 861             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
 862             if is_mine:
 863                 for o in tx.outputs:
 864                     o_addr, _ = o
 865                     if not self.is_mine(o_addr):
 866                         try:
 867                             default_label = self.labels[o_addr]
 868                         except KeyError:
 869                             default_label = o_addr
 870                         break
 871                 else:
 872                     default_label = '(internal)'
 873             else:
 874                 for o in tx.outputs:
 875                     o_addr, _ = o
 876                     if self.is_mine(o_addr) and not self.is_change(o_addr):
 877                         break
 878                 else:
 879                     for o in tx.outputs:
 880                         o_addr, _ = o
 881                         if self.is_mine(o_addr):
 882                             break
 883                     else:
 884                         o_addr = None
 885
 886                 if o_addr:
 887                     dest_label = self.labels.get(o_addr)
 888                     try:
 889                         default_label = self.labels[o_addr]
 890                     except KeyError:
 891                         default_label = o_addr
 892
 893         return default_label
 894
 895
 896     def mktx(self, outputs, password, fee=None, change_addr=None, account=None ):
 897         """
 898         create a transaction
 899         account parameter:
 900            None means use all accounts
 901            -1 means imported keys
 902            0, 1, etc are seed accounts
 903         """
 904
 905         for address, x in outputs:
 906             assert is_valid(address)
 907
 908         amount = sum( map(lambda x:x[1], outputs) )
 909
 910         inputs, total, fee = self.choose_tx_inputs( amount, fee, account )
 911         if not inputs:
 912             raise ValueError("Not enough funds")
 913
 914         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr, account)
 915
 916         tx = Transaction.from_io(inputs, outputs)
 917
 918
 919         keypairs = {}
 920         for i, txin in enumerate(tx.inputs):
 921             address = txin['address']
 922             if address in self.imported_keys.keys():
 923                 pk_addresses.append(address)
 924                 continue
 925             account, sequence = self.get_address_index(address)
 926             txin['KeyID'] = (account, 'BIP32', sequence) # used by the server to find the key
 927
 928             redeemScript = self.accounts[account].redeem_script(sequence)
 929             if redeemScript:
 930                 txin['redeemScript'] = redeemScript
 931                 assert address == self.accounts[account].get_address(*sequence)
 932             else:
 933                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
 934
 935             private_keys = self.get_private_key(address, password)
 936             for sec in private_keys:
 937                 compressed = is_compressed(sec)
 938                 pkey = regenerate_key(sec)
 939                 pubkey = GetPubKey(pkey.pubkey, compressed)
 940                 keypairs[ pubkey.encode('hex') ] = sec
 941
 942         tx.sign(keypairs)
 943
 944         for address, x in outputs:
 945             if address not in self.addressbook and not self.is_mine(address):
 946                 self.addressbook.append(address)
 947
 948         return tx
 949
 950
 951
 952     def sendtx(self, tx):
 953         # synchronous
 954         h = self.send_tx(tx)
 955         self.tx_event.wait()
 956         return self.receive_tx(h)
 957
 958     def send_tx(self, tx):
 959         # asynchronous
 960         self.tx_event.clear()
 961         self.interface.send([('blockchain.transaction.broadcast', [str(tx)])], 'synchronizer')
 962         return tx.hash()
 963
 964     def receive_tx(self,tx_hash):
 965         out = self.tx_result
 966         if out != tx_hash:
 967             return False, "error: " + out
 968         return True, out
 969
 970
 971
 972     def update_password(self, seed, old_password, new_password):
 973         if new_password == '': new_password = None
 974         # this will throw an exception if unicode cannot be converted
 975         self.seed = pw_encode( seed, new_password)
 976         self.config.set_key('seed', self.seed, True)
 977         self.use_encryption = (new_password != None)
 978         self.config.set_key('use_encryption', self.use_encryption,True)
 979         for k in self.imported_keys.keys():
 980             a = self.imported_keys[k]
 981             b = pw_decode(a, old_password)
 982             c = pw_encode(b, new_password)
 983             self.imported_keys[k] = c
 984         self.config.set_key('imported_keys', self.imported_keys, True)
 985
 986         for k, v in self.master_private_keys.items():
 987             b = pw_decode(v, old_password)
 988             c = pw_encode(b, new_password)
 989             self.master_private_keys[k] = c
 990         self.config.set_key('master_private_keys', self.master_private_keys, True)
 991
 992
 993     def freeze(self,addr):
 994         if self.is_mine(addr) and addr not in self.frozen_addresses:
 995             self.unprioritize(addr)
 996             self.frozen_addresses.append(addr)
 997             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 998             return True
 999         else:
1000             return False
1001
1002     def unfreeze(self,addr):
1003         if self.is_mine(addr) and addr in self.frozen_addresses:
1004             self.frozen_addresses.remove(addr)
1005             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
1006             return True
1007         else:
1008             return False
1009
1010     def prioritize(self,addr):
1011         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1012             self.unfreeze(addr)
1013             self.prioritized_addresses.append(addr)
1014             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1015             return True
1016         else:
1017             return False
1018
1019     def unprioritize(self,addr):
1020         if self.is_mine(addr) and addr in self.prioritized_addresses:
1021             self.prioritized_addresses.remove(addr)
1022             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1023             return True
1024         else:
1025             return False
1026
1027     def set_fee(self, fee):
1028         if self.fee != fee:
1029             self.fee = fee
1030             self.config.set_key('fee_per_kb', self.fee, True)
1031
1032
1033     def save(self):
1034         print_error("Warning: wallet.save() is deprecated")
1035         tx = {}
1036         for k,v in self.transactions.items():
1037             tx[k] = str(v)
1038
1039         s = {
1040             'use_change': self.use_change,
1041             'fee_per_kb': self.fee,
1042             'addr_history': self.history,
1043             'labels': self.labels,
1044             'contacts': self.addressbook,
1045             'num_zeros': self.num_zeros,
1046             'frozen_addresses': self.frozen_addresses,
1047             'prioritized_addresses': self.prioritized_addresses,
1048             'gap_limit': self.gap_limit,
1049             'transactions': tx,
1050         }
1051         for k, v in s.items():
1052             self.config.set_key(k,v)
1053         self.config.save()
1054
1055     def set_verifier(self, verifier):
1056         self.verifier = verifier
1057
1058         # review transactions that are in the history
1059         for addr, hist in self.history.items():
1060             if hist == ['*']: continue
1061             for tx_hash, tx_height in hist:
1062                 if tx_height>0:
1063                     # add it in case it was previously unconfirmed
1064                     self.verifier.add(tx_hash, tx_height)
1065
1066
1067         # if we are on a pruning server, remove unverified transactions
1068         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1069         for tx_hash in self.transactions.keys():
1070             if tx_hash not in vr:
1071                 self.transactions.pop(tx_hash)
1072
1073
1074
1075     def check_new_history(self, addr, hist):
1076
1077         # check that all tx in hist are relevant
1078         if hist != ['*']:
1079             for tx_hash, height in hist:
1080                 tx = self.transactions.get(tx_hash)
1081                 if not tx: continue
1082                 if not tx.has_address(addr):
1083                     return False
1084
1085         # check that we are not "orphaning" a transaction
1086         old_hist = self.history.get(addr,[])
1087         if old_hist == ['*']: return True
1088
1089         for tx_hash, height in old_hist:
1090             if tx_hash in map(lambda x:x[0], hist): continue
1091             found = False
1092             for _addr, _hist in self.history.items():
1093                 if _addr == addr: continue
1094                 if _hist == ['*']: continue
1095                 _tx_hist = map(lambda x:x[0], _hist)
1096                 if tx_hash in _tx_hist:
1097                     found = True
1098                     break
1099
1100             if not found:
1101                 tx = self.transactions.get(tx_hash)
1102                 # tx might not be there
1103                 if not tx: continue
1104
1105                 # already verified?
1106                 if self.verifier.get_height(tx_hash):
1107                     continue
1108                 # unconfirmed tx
1109                 print_error("new history is orphaning transaction:", tx_hash)
1110                 # check that all outputs are not mine, request histories
1111                 ext_requests = []
1112                 for _addr, _v in tx.outputs:
1113                     # assert not self.is_mine(_addr)
1114                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1115
1116                 ext_h = self.interface.synchronous_get(ext_requests)
1117                 print_error("sync:", ext_requests, ext_h)
1118                 height = None
1119                 for h in ext_h:
1120                     if h == ['*']: continue
1121                     for item in h:
1122                         if item.get('tx_hash') == tx_hash:
1123                             height = item.get('height')
1124                 if height:
1125                     print_error("found height for", tx_hash, height)
1126                     self.verifier.add(tx_hash, height)
1127                 else:
1128                     print_error("removing orphaned tx from history", tx_hash)
1129                     self.transactions.pop(tx_hash)
1130
1131         return True
1132
1133
1134
1135     def check_new_tx(self, tx_hash, tx):
1136         # 1 check that tx is referenced in addr_history.
1137         addresses = []
1138         for addr, hist in self.history.items():
1139             if hist == ['*']:continue
1140             for txh, height in hist:
1141                 if txh == tx_hash:
1142                     addresses.append(addr)
1143
1144         if not addresses:
1145             return False
1146
1147         # 2 check that referencing addresses are in the tx
1148         for addr in addresses:
1149             if not tx.has_address(addr):
1150                 return False
1151
1152         return True
1153
1154
1155
1156
1157 class WalletSynchronizer(threading.Thread):
1158
1159
1160     def __init__(self, wallet, config):
1161         threading.Thread.__init__(self)
1162         self.daemon = True
1163         self.wallet = wallet
1164         wallet.synchronizer = self
1165         self.interface = self.wallet.interface
1166         self.interface.register_channel('synchronizer')
1167         self.wallet.interface.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1168         self.was_updated = True
1169         self.running = False
1170         self.lock = threading.Lock()
1171
1172     def stop(self):
1173         with self.lock: self.running = False
1174         self.interface.poke('synchronizer')
1175
1176     def is_running(self):
1177         with self.lock: return self.running
1178
1179
1180     def subscribe_to_addresses(self, addresses):
1181         messages = []
1182         for addr in addresses:
1183             messages.append(('blockchain.address.subscribe', [addr]))
1184         self.interface.send( messages, 'synchronizer')
1185
1186
1187     def run(self):
1188         with self.lock: self.running = True
1189
1190         requested_tx = []
1191         missing_tx = []
1192         requested_histories = {}
1193
1194         # request any missing transactions
1195         for history in self.wallet.history.values():
1196             if history == ['*']: continue
1197             for tx_hash, tx_height in history:
1198                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1199                     missing_tx.append( (tx_hash, tx_height) )
1200         print_error("missing tx", missing_tx)
1201
1202         # wait until we are connected, in case the user is not connected
1203         while not self.interface.is_connected:
1204             time.sleep(1)
1205
1206         # subscriptions
1207         self.subscribe_to_addresses(self.wallet.addresses(True))
1208
1209         while self.is_running():
1210             # 1. create new addresses
1211             new_addresses = self.wallet.synchronize()
1212
1213             # request missing addresses
1214             if new_addresses:
1215                 self.subscribe_to_addresses(new_addresses)
1216
1217             # request missing transactions
1218             for tx_hash, tx_height in missing_tx:
1219                 if (tx_hash, tx_height) not in requested_tx:
1220                     self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
1221                     requested_tx.append( (tx_hash, tx_height) )
1222             missing_tx = []
1223
1224             # detect if situation has changed
1225             if not self.interface.is_up_to_date('synchronizer'):
1226                 if self.wallet.is_up_to_date():
1227                     self.wallet.set_up_to_date(False)
1228                     self.was_updated = True
1229             else:
1230                 if not self.wallet.is_up_to_date():
1231                     self.wallet.set_up_to_date(True)
1232                     self.was_updated = True
1233
1234             if self.was_updated:
1235                 self.interface.trigger_callback('updated')
1236                 self.was_updated = False
1237
1238             # 2. get a response
1239             r = self.interface.get_response('synchronizer')
1240
1241             # poke sends None. (needed during stop)
1242             if not r: continue
1243
1244             # 3. handle response
1245             method = r['method']
1246             params = r['params']
1247             result = r.get('result')
1248             error = r.get('error')
1249             if error:
1250                 print "error", r
1251                 continue
1252
1253             if method == 'blockchain.address.subscribe':
1254                 addr = params[0]
1255                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1256                     if requested_histories.get(addr) is None:
1257                         self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
1258                         requested_histories[addr] = result
1259
1260             elif method == 'blockchain.address.get_history':
1261                 addr = params[0]
1262                 print_error("receiving history", addr, result)
1263                 if result == ['*']:
1264                     assert requested_histories.pop(addr) == '*'
1265                     self.wallet.receive_history_callback(addr, result)
1266                 else:
1267                     hist = []
1268                     # check that txids are unique
1269                     txids = []
1270                     for item in result:
1271                         tx_hash = item['tx_hash']
1272                         if tx_hash not in txids:
1273                             txids.append(tx_hash)
1274                             hist.append( (tx_hash, item['height']) )
1275
1276                     if len(hist) != len(result):
1277                         raise BaseException("error: server sent history with non-unique txid", result)
1278
1279                     # check that the status corresponds to what was announced
1280                     rs = requested_histories.pop(addr)
1281                     if self.wallet.get_status(hist) != rs:
1282                         raise BaseException("error: status mismatch: %s"%addr)
1283
1284                     # store received history
1285                     self.wallet.receive_history_callback(addr, hist)
1286
1287                     # request transactions that we don't have
1288                     for tx_hash, tx_height in hist:
1289                         if self.wallet.transactions.get(tx_hash) is None:
1290                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1291                                 missing_tx.append( (tx_hash, tx_height) )
1292
1293             elif method == 'blockchain.transaction.get':
1294                 tx_hash = params[0]
1295                 tx_height = params[1]
1296                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1297                 tx = Transaction(result)
1298                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1299                 self.was_updated = True
1300                 requested_tx.remove( (tx_hash, tx_height) )
1301                 print_error("received tx:", tx_hash, len(tx.raw))
1302
1303             elif method == 'blockchain.transaction.broadcast':
1304                 self.wallet.tx_result = result
1305                 self.wallet.tx_event.set()
1306
1307             else:
1308                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1309
1310             if self.was_updated and not requested_tx:
1311                 self.interface.trigger_callback('updated')
1312                 self.interface.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1313
1314
1315                 self.was_updated = False