lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, user_dir, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36
  37 # AES encryption
  38 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  39 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  40
  41 def pw_encode(s, password):
  42     if password:
  43         secret = Hash(password)
  44         return EncodeAES(secret, s)
  45     else:
  46         return s
  47
  48 def pw_decode(s, password):
  49     if password is not None:
  50         secret = Hash(password)
  51         try:
  52             d = DecodeAES(secret, s)
  53         except:
  54             raise BaseException('Invalid password')
  55         return d
  56     else:
  57         return s
  58
  59
  60
  61
  62
  63 from version import ELECTRUM_VERSION, SEED_VERSION
  64
  65
  66 class Wallet:
  67     def __init__(self, config={}):
  68
  69         self.config = config
  70         self.electrum_version = ELECTRUM_VERSION
  71         self.gap_limit_for_change = 3 # constant
  72
  73         # saved fields
  74         self.seed_version          = config.get('seed_version', SEED_VERSION)
  75         self.gap_limit             = config.get('gap_limit', 5)
  76         self.use_change            = config.get('use_change',True)
  77         self.fee                   = int(config.get('fee_per_kb',50000))
  78         self.num_zeros             = int(config.get('num_zeros',0))
  79         self.use_encryption        = config.get('use_encryption', False)
  80         self.seed                  = config.get('seed', '')               # encrypted
  81         self.labels                = config.get('labels', {})
  82         self.frozen_addresses      = config.get('frozen_addresses',[])
  83         self.prioritized_addresses = config.get('prioritized_addresses',[])
  84         self.addressbook           = config.get('contacts', [])
  85
  86         self.imported_keys         = config.get('imported_keys',{})
  87         self.history               = config.get('addr_history',{})        # address -> list(txid, height)
  88
  89
  90         self.master_public_keys = config.get('master_public_keys',{})
  91         self.master_private_keys = config.get('master_private_keys', {})
  92
  93         self.load_accounts(config)
  94
  95         self.transactions = {}
  96         tx = config.get('transactions',{})
  97         try:
  98             for k,v in tx.items(): self.transactions[k] = Transaction(v)
  99         except:
 100             print_msg("Warning: Cannot deserialize transactions. skipping")
 101
 102         # not saved
 103         self.prevout_values = {}     # my own transaction outputs
 104         self.spent_outputs = []
 105
 106         # spv
 107         self.verifier = None
 108
 109         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 110         # interface.is_up_to_date() returns true when all requests have been answered and processed
 111         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 112
 113         self.up_to_date = False
 114         self.lock = threading.Lock()
 115         self.transaction_lock = threading.Lock()
 116         self.tx_event = threading.Event()
 117
 118         if self.seed_version != SEED_VERSION:
 119             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
 120
 121         for tx_hash, tx in self.transactions.items():
 122             if self.check_new_tx(tx_hash, tx):
 123                 self.update_tx_outputs(tx_hash)
 124             else:
 125                 print_error("unreferenced tx", tx_hash)
 126                 self.transactions.pop(tx_hash)
 127
 128
 129     def set_up_to_date(self,b):
 130         with self.lock: self.up_to_date = b
 131
 132     def is_up_to_date(self):
 133         with self.lock: return self.up_to_date
 134
 135     def update(self):
 136         self.up_to_date = False
 137         self.interface.poke('synchronizer')
 138         while not self.is_up_to_date(): time.sleep(0.1)
 139
 140     def import_key(self, sec, password):
 141         # check password
 142         seed = self.decode_seed(password)
 143         try:
 144             address = address_from_private_key(sec)
 145         except:
 146             raise BaseException('Invalid private key')
 147
 148         if self.is_mine(address):
 149             raise BaseException('Address already in wallet')
 150
 151         # store the originally requested keypair into the imported keys table
 152         self.imported_keys[address] = pw_encode(sec, password )
 153         self.config.set_key('imported_keys', self.imported_keys, True)
 154         return address
 155
 156     def delete_imported_key(self, addr):
 157         if addr in self.imported_keys:
 158             self.imported_keys.pop(addr)
 159             self.config.set_key('imported_keys', self.imported_keys, True)
 160
 161
 162     def init_seed(self, seed):
 163         if self.seed: raise BaseException("a seed exists")
 164         if not seed:
 165             seed = random_seed(128)
 166         self.seed = seed
 167
 168
 169     def save_seed(self):
 170         self.config.set_key('seed', self.seed, True)
 171         self.config.set_key('seed_version', self.seed_version, True)
 172
 173         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 174
 175         # normal accounts
 176         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 177         # p2sh 2of2
 178         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 179         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 180         # p2sh 2of3
 181         k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 182         k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 183         k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 184
 185         self.master_public_keys = {
 186             "m/0'/": (c0, K0, cK0),
 187             "m/1'/": (c1, K1, cK1),
 188             "m/2'/": (c2, K2, cK2),
 189             "m/3'/": (c3, K3, cK3),
 190             "m/4'/": (c4, K4, cK4),
 191             "m/5'/": (c5, K5, cK5)
 192             }
 193
 194         self.master_private_keys = {
 195             "m/0'/": k0,
 196             "m/1'/": k1,
 197             "m/2'/": k2,
 198             "m/3'/": k3,
 199             "m/4'/": k4,
 200             "m/5'/": k5
 201             }
 202
 203         self.config.set_key('master_public_keys', self.master_public_keys, True)
 204         self.config.set_key('master_private_keys', self.master_private_keys, True)
 205
 206         # create default account
 207         self.create_account('Main account')
 208
 209
 210     def account_id(self, account_type, i):
 211         if account_type is None:
 212             return "m/0'/%d'"%i
 213         elif account_type == '2of2':
 214             return "m/1'/%d & m/2'/%d"%(i,i)
 215         elif account_type == '2of3':
 216             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 217         else:
 218             raise BaseException('unknown account type')
 219
 220
 221     def num_accounts(self, account_type):
 222         keys = self.accounts.keys()
 223         i = 0
 224         while True:
 225             account_id = self.account_id(account_type, i)
 226             if account_id not in keys: break
 227             i += 1
 228         return i
 229
 230
 231     def create_account(self, name, account_type = None):
 232         i = self.num_accounts(account_type)
 233         acount_id = self.account_id(account_type,i)
 234
 235         if account_type is None:
 236             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 237             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 238             account = BIP32_Account({ 'name':name, 'c':c0, 'K':K0, 'cK':cK0 })
 239
 240         elif account_type == '2of2':
 241             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 242             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 243             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 244             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 245             account = BIP32_Account_2of2({ 'name':name, 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 246
 247         elif account_type == '2of3':
 248             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 249             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 250             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 251             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 252             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 253             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 254             account = BIP32_Account_2of3({ 'name':name, 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 255
 256         self.accounts[account_id] = account
 257         self.save_accounts()
 258
 259
 260     def save_accounts(self):
 261         d = {}
 262         for k, v in self.accounts.items():
 263             d[k] = v.dump()
 264         self.config.set_key('accounts', d, True)
 265
 266
 267     def load_accounts(self, config):
 268         d = config.get('accounts', {})
 269         self.accounts = {}
 270         for k, v in d.items():
 271             if '&' in k:
 272                 self.accounts[k] = BIP32_Account_2of2(v)
 273             else:
 274                 self.accounts[k] = BIP32_Account(v)
 275
 276
 277
 278
 279     def addresses(self, include_change = True):
 280         o = self.get_account_addresses(-1, include_change)
 281         for a in self.accounts.keys():
 282             o += self.get_account_addresses(a, include_change)
 283         return o
 284
 285
 286     def is_mine(self, address):
 287         return address in self.addresses(True)
 288
 289     def is_change(self, address):
 290         if not self.is_mine(address): return False
 291         if address in self.imported_keys.keys(): return False
 292         acct, s = self.get_address_index(address)
 293         return s[0] == 1
 294
 295     def get_master_public_key(self):
 296         raise
 297         return self.config.get("master_public_key")
 298
 299     def get_master_private_key(self, account, password):
 300         master_k = pw_decode( self.master_private_keys[account], password)
 301         master_c, master_K, master_Kc = self.master_public_keys[account]
 302         try:
 303             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 304             assert K.encode('hex') == master_K
 305         except:
 306             raise BaseException("Invalid password")
 307         return master_k
 308
 309
 310     def get_address_index(self, address):
 311         if address in self.imported_keys.keys():
 312             return -1, None
 313         for account in self.accounts.keys():
 314             for for_change in [0,1]:
 315                 addresses = self.accounts[account].get_addresses(for_change)
 316                 for addr in addresses:
 317                     if address == addr:
 318                         return account, (for_change, addresses.index(addr))
 319         raise BaseException("not found")
 320
 321
 322     def get_public_key(self, address):
 323         account, sequence = self.get_address_index(address)
 324         return self.accounts[account].get_pubkey( *sequence )
 325
 326
 327     def decode_seed(self, password):
 328         seed = pw_decode(self.seed, password)
 329         #todo:  #self.sequences[0].check_seed(seed)
 330         return seed
 331
 332
 333     def get_private_key(self, address, password):
 334         out = []
 335         if address in self.imported_keys.keys():
 336             out.append( pw_decode( self.imported_keys[address], password ) )
 337         else:
 338             account, sequence = self.get_address_index(address)
 339             # assert address == self.accounts[account].get_address(*sequence)
 340             l = account.split("&")
 341             for s in l:
 342                 s = s.strip()
 343                 m = re.match("(m/\d+'/)(\d+)", s)
 344                 if m:
 345                     root = m.group(1)
 346                     if root not in self.master_private_keys.keys(): continue
 347                     num = int(m.group(2))
 348                     master_k = self.get_master_private_key(root, password)
 349                     master_c, _, _ = self.master_public_keys[root]
 350                     pk = bip32_private_key( (num,) + sequence, master_k.decode('hex'), master_c.decode('hex'))
 351                     out.append(pk)
 352
 353         return out
 354
 355
 356     def get_private_keys(self, addresses, password):
 357         if not self.seed: return {}
 358         # decode seed in any case, in order to test the password
 359         seed = self.decode_seed(password)
 360         out = {}
 361         for address in addresses:
 362             pk = self.get_private_key(address, password)
 363             if pk: out[address] = pk
 364
 365         return out
 366
 367
 368     def signrawtransaction(self, tx, input_info, private_keys, password):
 369         unspent_coins = self.get_unspent_coins()
 370         seed = self.decode_seed(password)
 371
 372         # convert private_keys to dict
 373         pk = {}
 374         for sec in private_keys:
 375             address = address_from_private_key(sec)
 376             pk[address] = sec
 377         private_keys = pk
 378
 379         for txin in tx.inputs:
 380             # convert to own format
 381             txin['tx_hash'] = txin['prevout_hash']
 382             txin['index'] = txin['prevout_n']
 383
 384             for item in input_info:
 385                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 386                     txin['raw_output_script'] = item['scriptPubKey']
 387                     txin['redeemScript'] = item.get('redeemScript')
 388                     txin['KeyID'] = item.get('KeyID')
 389                     break
 390             else:
 391                 for item in unspent_coins:
 392                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 393                         txin['raw_output_script'] = item['raw_output_script']
 394                         break
 395                 else:
 396                     # if neither, we might want to get it from the server..
 397                     raise
 398
 399             # find the address:
 400             if txin.get('KeyID'):
 401                 account, name, sequence = txin.get('KeyID')
 402                 if name != 'Electrum': continue
 403                 sec = self.accounts[account].get_private_key(sequence, seed)
 404                 addr = self.accounts[account].get_address(sequence)
 405                 txin['address'] = addr
 406                 private_keys[addr] = sec
 407
 408             elif txin.get("redeemScript"):
 409                 txin['address'] = hash_160_to_bc_address(hash_160(txin.get("redeemScript").decode('hex')), 5)
 410
 411             elif txin.get("raw_output_script"):
 412                 import deserialize
 413                 addr = deserialize.get_address_from_output_script(txin.get("raw_output_script").decode('hex'))
 414                 sec = self.get_private_key(addr, password)
 415                 if sec:
 416                     private_keys[addr] = sec
 417                     txin['address'] = addr
 418
 419         tx.sign( private_keys )
 420
 421     def sign_message(self, address, message, password):
 422         sec = self.get_private_key(address, password)
 423         key = regenerate_key(sec)
 424         compressed = is_compressed(sec)
 425         return key.sign_message(message, compressed, address)
 426
 427     def verify_message(self, address, signature, message):
 428         try:
 429             EC_KEY.verify_message(address, signature, message)
 430             return True
 431         except BaseException as e:
 432             print_error("Verification error: {0}".format(e))
 433             return False
 434
 435
 436     def change_gap_limit(self, value):
 437         if value >= self.gap_limit:
 438             self.gap_limit = value
 439             self.config.set_key('gap_limit', self.gap_limit, True)
 440             self.interface.poke('synchronizer')
 441             return True
 442
 443         elif value >= self.min_acceptable_gap():
 444             for key, account in self.accounts.items():
 445                 addresses = account[0]
 446                 k = self.num_unused_trailing_addresses(addresses)
 447                 n = len(addresses) - k + value
 448                 addresses = addresses[0:n]
 449                 self.accounts[key][0] = addresses
 450
 451             self.gap_limit = value
 452             self.config.set_key('gap_limit', self.gap_limit, True)
 453             self.save_accounts()
 454             return True
 455         else:
 456             return False
 457
 458     def num_unused_trailing_addresses(self, addresses):
 459         k = 0
 460         for a in addresses[::-1]:
 461             if self.history.get(a):break
 462             k = k + 1
 463         return k
 464
 465     def min_acceptable_gap(self):
 466         # fixme: this assumes wallet is synchronized
 467         n = 0
 468         nmax = 0
 469
 470         for account in self.accounts.values():
 471             addresses = account.get_addresses(0)
 472             k = self.num_unused_trailing_addresses(addresses)
 473             for a in addresses[0:-k]:
 474                 if self.history.get(a):
 475                     n = 0
 476                 else:
 477                     n += 1
 478                     if n > nmax: nmax = n
 479         return nmax + 1
 480
 481
 482     def address_is_old(self, address):
 483         age = -1
 484         h = self.history.get(address, [])
 485         if h == ['*']:
 486             return True
 487         for tx_hash, tx_height in h:
 488             if tx_height == 0:
 489                 tx_age = 0
 490             else:
 491                 tx_age = self.verifier.height - tx_height + 1
 492             if tx_age > age:
 493                 age = tx_age
 494         return age > 2
 495
 496
 497     def synchronize_sequence(self, account, for_change):
 498         limit = self.gap_limit_for_change if for_change else self.gap_limit
 499         new_addresses = []
 500         while True:
 501             addresses = account.get_addresses(for_change)
 502             if len(addresses) < limit:
 503                 address = account.create_new_address(for_change)
 504                 self.history[address] = []
 505                 new_addresses.append( address )
 506                 continue
 507
 508             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 509                 break
 510             else:
 511                 address = account.create_new_address(for_change)
 512                 self.history[address] = []
 513                 new_addresses.append( address )
 514
 515         return new_addresses
 516
 517
 518     def synchronize_account(self, account):
 519         new = []
 520         new += self.synchronize_sequence(account, 0)
 521         new += self.synchronize_sequence(account, 1)
 522         return new
 523
 524     def synchronize(self):
 525         new = []
 526         for account in self.accounts.values():
 527             new += self.synchronize_account(account)
 528         if new:
 529             self.save_accounts()
 530             self.config.set_key('addr_history', self.history, True)
 531         return new
 532
 533
 534     def is_found(self):
 535         return self.history.values() != [[]] * len(self.history)
 536
 537
 538     def add_contact(self, address, label=None):
 539         self.addressbook.append(address)
 540         self.config.set_key('contacts', self.addressbook, True)
 541         if label:
 542             self.labels[address] = label
 543             self.config.set_key('labels', self.labels)
 544
 545     def delete_contact(self, addr):
 546         if addr in self.addressbook:
 547             self.addressbook.remove(addr)
 548             self.config.set_key('addressbook', self.addressbook, True)
 549
 550
 551     def fill_addressbook(self):
 552         for tx_hash, tx in self.transactions.items():
 553             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 554             if is_send:
 555                 for addr, v in tx.outputs:
 556                     if not self.is_mine(addr) and addr not in self.addressbook:
 557                         self.addressbook.append(addr)
 558         # redo labels
 559         # self.update_tx_labels()
 560
 561     def get_num_tx(self, address):
 562         n = 0
 563         for tx in self.transactions.values():
 564             if address in map(lambda x:x[0], tx.outputs): n += 1
 565         return n
 566
 567
 568     def get_address_flags(self, addr):
 569         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 570         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 571         return flags
 572
 573
 574     def get_tx_value(self, tx, account=None):
 575         domain = self.get_account_addresses(account)
 576         return tx.get_value(domain, self.prevout_values)
 577
 578
 579     def update_tx_outputs(self, tx_hash):
 580         tx = self.transactions.get(tx_hash)
 581
 582         for i, (addr, value) in enumerate(tx.outputs):
 583             key = tx_hash+ ':%d'%i
 584             self.prevout_values[key] = value
 585
 586         for item in tx.inputs:
 587             if self.is_mine(item.get('address')):
 588                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 589                 self.spent_outputs.append(key)
 590
 591
 592     def get_addr_balance(self, address):
 593         assert self.is_mine(address)
 594         h = self.history.get(address,[])
 595         if h == ['*']: return 0,0
 596         c = u = 0
 597         received_coins = []   # list of coins received at address
 598
 599         for tx_hash, tx_height in h:
 600             tx = self.transactions.get(tx_hash)
 601             if not tx: continue
 602
 603             for i, (addr, value) in enumerate(tx.outputs):
 604                 if addr == address:
 605                     key = tx_hash + ':%d'%i
 606                     received_coins.append(key)
 607
 608         for tx_hash, tx_height in h:
 609             tx = self.transactions.get(tx_hash)
 610             if not tx: continue
 611             v = 0
 612
 613             for item in tx.inputs:
 614                 addr = item.get('address')
 615                 if addr == address:
 616                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 617                     value = self.prevout_values.get( key )
 618                     if key in received_coins:
 619                         v -= value
 620
 621             for i, (addr, value) in enumerate(tx.outputs):
 622                 key = tx_hash + ':%d'%i
 623                 if addr == address:
 624                     v += value
 625
 626             if tx_height:
 627                 c += v
 628             else:
 629                 u += v
 630         return c, u
 631
 632
 633     def get_accounts(self):
 634         accounts = {}
 635         for k, account in self.accounts.items():
 636             accounts[k] = account.name
 637         if self.imported_keys:
 638             accounts[-1] = 'Imported keys'
 639         return accounts
 640
 641     def get_account_addresses(self, a, include_change=True):
 642         if a is None:
 643             o = self.addresses(True)
 644         elif a == -1:
 645             o = self.imported_keys.keys()
 646         else:
 647             ac = self.accounts[a]
 648             o = ac.get_addresses(0)
 649             if include_change: o += ac.get_addresses(1)
 650         return o
 651
 652     def get_imported_balance(self):
 653         cc = uu = 0
 654         for addr in self.imported_keys.keys():
 655             c, u = self.get_addr_balance(addr)
 656             cc += c
 657             uu += u
 658         return cc, uu
 659
 660     def get_account_balance(self, account):
 661         if account is None:
 662             return self.get_balance()
 663         elif account == -1:
 664             return self.get_imported_balance()
 665
 666         conf = unconf = 0
 667         for addr in self.get_account_addresses(account):
 668             c, u = self.get_addr_balance(addr)
 669             conf += c
 670             unconf += u
 671         return conf, unconf
 672
 673     def get_frozen_balance(self):
 674         conf = unconf = 0
 675         for addr in self.frozen_addresses:
 676             c, u = self.get_addr_balance(addr)
 677             conf += c
 678             unconf += u
 679         return conf, unconf
 680
 681
 682     def get_balance(self):
 683         cc = uu = 0
 684         for a in self.accounts.keys():
 685             c, u = self.get_account_balance(a)
 686             cc += c
 687             uu += u
 688         c, u = self.get_imported_balance()
 689         cc += c
 690         uu += u
 691         return cc, uu
 692
 693
 694     def get_unspent_coins(self, domain=None):
 695         coins = []
 696         if domain is None: domain = self.addresses(True)
 697         for addr in domain:
 698             h = self.history.get(addr, [])
 699             if h == ['*']: continue
 700             for tx_hash, tx_height in h:
 701                 tx = self.transactions.get(tx_hash)
 702                 if tx is None: raise BaseException("Wallet not synchronized")
 703                 for output in tx.d.get('outputs'):
 704                     if output.get('address') != addr: continue
 705                     key = tx_hash + ":%d" % output.get('index')
 706                     if key in self.spent_outputs: continue
 707                     output['tx_hash'] = tx_hash
 708                     coins.append(output)
 709         return coins
 710
 711
 712
 713     def choose_tx_inputs( self, amount, fixed_fee, account = None ):
 714         """ todo: minimize tx size """
 715         total = 0
 716         fee = self.fee if fixed_fee is None else fixed_fee
 717         domain = self.get_account_addresses(account)
 718         coins = []
 719         prioritized_coins = []
 720         for i in self.frozen_addresses:
 721             if i in domain: domain.remove(i)
 722
 723         for i in self.prioritized_addresses:
 724             if i in domain: domain.remove(i)
 725
 726         coins = self.get_unspent_coins(domain)
 727         prioritized_coins = self.get_unspent_coins(self.prioritized_addresses)
 728
 729         inputs = []
 730         coins = prioritized_coins + coins
 731
 732         for item in coins:
 733             addr = item.get('address')
 734             v = item.get('value')
 735             total += v
 736             inputs.append( item )
 737             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
 738             if total >= amount + fee: break
 739         else:
 740             inputs = []
 741
 742         return inputs, total, fee
 743
 744
 745     def estimated_fee(self, inputs):
 746         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
 747         fee = self.fee * int(round(estimated_size/1024.))
 748         if fee == 0: fee = self.fee
 749         return fee
 750
 751
 752     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None, account=0 ):
 753         "add change to a transaction"
 754         change_amount = total - ( amount + fee )
 755         if change_amount != 0:
 756             if not change_addr:
 757                 if account is None:
 758                     # send change to one of the accounts involved in the tx
 759                     address = inputs[0].get('address')
 760                     account, _ = self.get_address_index(address)
 761
 762                 if not self.use_change or account == -1:
 763                     change_addr = inputs[-1]['address']
 764                 else:
 765                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
 766
 767             # Insert the change output at a random position in the outputs
 768             posn = random.randint(0, len(outputs))
 769             outputs[posn:posn] = [( change_addr,  change_amount)]
 770         return outputs
 771
 772
 773     def get_history(self, address):
 774         with self.lock:
 775             return self.history.get(address)
 776
 777
 778     def get_status(self, h):
 779         if not h: return None
 780         if h == ['*']: return '*'
 781         status = ''
 782         for tx_hash, height in h:
 783             status += tx_hash + ':%d:' % height
 784         return hashlib.sha256( status ).digest().encode('hex')
 785
 786
 787     def receive_tx_callback(self, tx_hash, tx, tx_height):
 788         if not self.check_new_tx(tx_hash, tx):
 789             # may happen due to pruning
 790             print_error("received transaction that is no longer referenced in history", tx_hash)
 791             return
 792
 793         with self.transaction_lock:
 794             self.transactions[tx_hash] = tx
 795
 796             self.interface.pending_transactions_for_notifications.append(tx)
 797
 798             self.save_transactions()
 799             if self.verifier and tx_height>0:
 800                 self.verifier.add(tx_hash, tx_height)
 801             self.update_tx_outputs(tx_hash)
 802
 803
 804     def save_transactions(self):
 805         tx = {}
 806         for k,v in self.transactions.items():
 807             tx[k] = str(v)
 808         self.config.set_key('transactions', tx, True)
 809
 810     def receive_history_callback(self, addr, hist):
 811
 812         if not self.check_new_history(addr, hist):
 813             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
 814
 815         with self.lock:
 816             self.history[addr] = hist
 817             self.config.set_key('addr_history', self.history, True)
 818
 819         if hist != ['*']:
 820             for tx_hash, tx_height in hist:
 821                 if tx_height>0:
 822                     # add it in case it was previously unconfirmed
 823                     if self.verifier: self.verifier.add(tx_hash, tx_height)
 824
 825
 826     def get_tx_history(self, account=None):
 827         with self.transaction_lock:
 828             history = self.transactions.items()
 829             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
 830             result = []
 831
 832             balance = 0
 833             for tx_hash, tx in history:
 834                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
 835                 if v is not None: balance += v
 836
 837             c, u = self.get_account_balance(account)
 838
 839             if balance != c+u:
 840                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
 841
 842             balance = c + u - balance
 843             for tx_hash, tx in history:
 844                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
 845                 if not is_relevant:
 846                     continue
 847                 if value is not None:
 848                     balance += value
 849
 850                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
 851                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
 852
 853         return result
 854
 855
 856     def get_label(self, tx_hash):
 857         label = self.labels.get(tx_hash)
 858         is_default = (label == '') or (label is None)
 859         if is_default: label = self.get_default_label(tx_hash)
 860         return label, is_default
 861
 862
 863     def get_default_label(self, tx_hash):
 864         tx = self.transactions.get(tx_hash)
 865         default_label = ''
 866         if tx:
 867             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
 868             if is_mine:
 869                 for o in tx.outputs:
 870                     o_addr, _ = o
 871                     if not self.is_mine(o_addr):
 872                         try:
 873                             default_label = self.labels[o_addr]
 874                         except KeyError:
 875                             default_label = o_addr
 876                         break
 877                 else:
 878                     default_label = '(internal)'
 879             else:
 880                 for o in tx.outputs:
 881                     o_addr, _ = o
 882                     if self.is_mine(o_addr) and not self.is_change(o_addr):
 883                         break
 884                 else:
 885                     for o in tx.outputs:
 886                         o_addr, _ = o
 887                         if self.is_mine(o_addr):
 888                             break
 889                     else:
 890                         o_addr = None
 891
 892                 if o_addr:
 893                     dest_label = self.labels.get(o_addr)
 894                     try:
 895                         default_label = self.labels[o_addr]
 896                     except KeyError:
 897                         default_label = o_addr
 898
 899         return default_label
 900
 901
 902     def mktx(self, outputs, password, fee=None, change_addr=None, account=None ):
 903         """
 904         create a transaction
 905         account parameter:
 906            None means use all accounts
 907            -1 means imported keys
 908            0, 1, etc are seed accounts
 909         """
 910
 911         for address, x in outputs:
 912             assert is_valid(address)
 913
 914         amount = sum( map(lambda x:x[1], outputs) )
 915
 916         inputs, total, fee = self.choose_tx_inputs( amount, fee, account )
 917         if not inputs:
 918             raise ValueError("Not enough funds")
 919
 920         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr, account)
 921
 922         tx = Transaction.from_io(inputs, outputs)
 923
 924         private_keys = {}
 925         for i in range(len(tx.inputs)):
 926             txin = tx.inputs[i]
 927             address = txin['address']
 928             if address in self.imported_keys.keys():
 929                 pk_addresses.append(address)
 930                 continue
 931             account, sequence = self.get_address_index(address)
 932             txin['KeyID'] = (account, 'BIP32', sequence) # used by the server to find the key
 933
 934             redeemScript = self.accounts[account].redeem_script(sequence)
 935             if redeemScript:
 936                 txin['redeemScript'] = redeemScript
 937                 assert address == self.accounts[account].get_address(*sequence)
 938
 939             private_keys[address] = self.get_private_key(address, password)
 940
 941         print_error( "private keys", private_keys )
 942         tx.sign(private_keys)
 943
 944         for address, x in outputs:
 945             if address not in self.addressbook and not self.is_mine(address):
 946                 self.addressbook.append(address)
 947
 948         return tx
 949
 950
 951
 952     def sendtx(self, tx):
 953         # synchronous
 954         h = self.send_tx(tx)
 955         self.tx_event.wait()
 956         return self.receive_tx(h)
 957
 958     def send_tx(self, tx):
 959         # asynchronous
 960         self.tx_event.clear()
 961         self.interface.send([('blockchain.transaction.broadcast', [str(tx)])], 'synchronizer')
 962         return tx.hash()
 963
 964     def receive_tx(self,tx_hash):
 965         out = self.tx_result
 966         if out != tx_hash:
 967             return False, "error: " + out
 968         return True, out
 969
 970
 971
 972     def update_password(self, seed, old_password, new_password):
 973         if new_password == '': new_password = None
 974         # this will throw an exception if unicode cannot be converted
 975         self.seed = pw_encode( seed, new_password)
 976         self.config.set_key('seed', self.seed, True)
 977         self.use_encryption = (new_password != None)
 978         self.config.set_key('use_encryption', self.use_encryption,True)
 979         for k in self.imported_keys.keys():
 980             a = self.imported_keys[k]
 981             b = pw_decode(a, old_password)
 982             c = pw_encode(b, new_password)
 983             self.imported_keys[k] = c
 984         self.config.set_key('imported_keys', self.imported_keys, True)
 985
 986         for k, v in self.master_private_keys.items():
 987             b = pw_decode(v, old_password)
 988             c = pw_encode(b, new_password)
 989             self.master_private_keys[k] = c
 990         self.config.set_key('master_private_keys', self.master_private_keys, True)
 991
 992
 993     def freeze(self,addr):
 994         if self.is_mine(addr) and addr not in self.frozen_addresses:
 995             self.unprioritize(addr)
 996             self.frozen_addresses.append(addr)
 997             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 998             return True
 999         else:
1000             return False
1001
1002     def unfreeze(self,addr):
1003         if self.is_mine(addr) and addr in self.frozen_addresses:
1004             self.frozen_addresses.remove(addr)
1005             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
1006             return True
1007         else:
1008             return False
1009
1010     def prioritize(self,addr):
1011         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1012             self.unfreeze(addr)
1013             self.prioritized_addresses.append(addr)
1014             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1015             return True
1016         else:
1017             return False
1018
1019     def unprioritize(self,addr):
1020         if self.is_mine(addr) and addr in self.prioritized_addresses:
1021             self.prioritized_addresses.remove(addr)
1022             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1023             return True
1024         else:
1025             return False
1026
1027     def set_fee(self, fee):
1028         if self.fee != fee:
1029             self.fee = fee
1030             self.config.set_key('fee_per_kb', self.fee, True)
1031
1032
1033     def save(self):
1034         print_error("Warning: wallet.save() is deprecated")
1035         tx = {}
1036         for k,v in self.transactions.items():
1037             tx[k] = str(v)
1038
1039         s = {
1040             'use_change': self.use_change,
1041             'fee_per_kb': self.fee,
1042             'addr_history': self.history,
1043             'labels': self.labels,
1044             'contacts': self.addressbook,
1045             'num_zeros': self.num_zeros,
1046             'frozen_addresses': self.frozen_addresses,
1047             'prioritized_addresses': self.prioritized_addresses,
1048             'gap_limit': self.gap_limit,
1049             'transactions': tx,
1050         }
1051         for k, v in s.items():
1052             self.config.set_key(k,v)
1053         self.config.save()
1054
1055     def set_verifier(self, verifier):
1056         self.verifier = verifier
1057
1058         # review transactions that are in the history
1059         for addr, hist in self.history.items():
1060             if hist == ['*']: continue
1061             for tx_hash, tx_height in hist:
1062                 if tx_height>0:
1063                     # add it in case it was previously unconfirmed
1064                     self.verifier.add(tx_hash, tx_height)
1065
1066
1067         # if we are on a pruning server, remove unverified transactions
1068         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1069         for tx_hash in self.transactions.keys():
1070             if tx_hash not in vr:
1071                 self.transactions.pop(tx_hash)
1072
1073
1074
1075     def check_new_history(self, addr, hist):
1076
1077         # check that all tx in hist are relevant
1078         if hist != ['*']:
1079             for tx_hash, height in hist:
1080                 tx = self.transactions.get(tx_hash)
1081                 if not tx: continue
1082                 if not tx.has_address(addr):
1083                     return False
1084
1085         # check that we are not "orphaning" a transaction
1086         old_hist = self.history.get(addr,[])
1087         if old_hist == ['*']: return True
1088
1089         for tx_hash, height in old_hist:
1090             if tx_hash in map(lambda x:x[0], hist): continue
1091             found = False
1092             for _addr, _hist in self.history.items():
1093                 if _addr == addr: continue
1094                 if _hist == ['*']: continue
1095                 _tx_hist = map(lambda x:x[0], _hist)
1096                 if tx_hash in _tx_hist:
1097                     found = True
1098                     break
1099
1100             if not found:
1101                 tx = self.transactions.get(tx_hash)
1102                 # tx might not be there
1103                 if not tx: continue
1104
1105                 # already verified?
1106                 if self.verifier.get_height(tx_hash):
1107                     continue
1108                 # unconfirmed tx
1109                 print_error("new history is orphaning transaction:", tx_hash)
1110                 # check that all outputs are not mine, request histories
1111                 ext_requests = []
1112                 for _addr, _v in tx.outputs:
1113                     # assert not self.is_mine(_addr)
1114                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1115
1116                 ext_h = self.interface.synchronous_get(ext_requests)
1117                 print_error("sync:", ext_requests, ext_h)
1118                 height = None
1119                 for h in ext_h:
1120                     if h == ['*']: continue
1121                     for item in h:
1122                         if item.get('tx_hash') == tx_hash:
1123                             height = item.get('height')
1124                 if height:
1125                     print_error("found height for", tx_hash, height)
1126                     self.verifier.add(tx_hash, height)
1127                 else:
1128                     print_error("removing orphaned tx from history", tx_hash)
1129                     self.transactions.pop(tx_hash)
1130
1131         return True
1132
1133
1134
1135     def check_new_tx(self, tx_hash, tx):
1136         # 1 check that tx is referenced in addr_history.
1137         addresses = []
1138         for addr, hist in self.history.items():
1139             if hist == ['*']:continue
1140             for txh, height in hist:
1141                 if txh == tx_hash:
1142                     addresses.append(addr)
1143
1144         if not addresses:
1145             return False
1146
1147         # 2 check that referencing addresses are in the tx
1148         for addr in addresses:
1149             if not tx.has_address(addr):
1150                 return False
1151
1152         return True
1153
1154
1155
1156
1157 class WalletSynchronizer(threading.Thread):
1158
1159
1160     def __init__(self, wallet, config):
1161         threading.Thread.__init__(self)
1162         self.daemon = True
1163         self.wallet = wallet
1164         wallet.synchronizer = self
1165         self.interface = self.wallet.interface
1166         self.interface.register_channel('synchronizer')
1167         self.wallet.interface.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1168         self.was_updated = True
1169         self.running = False
1170         self.lock = threading.Lock()
1171
1172     def stop(self):
1173         with self.lock: self.running = False
1174         self.interface.poke('synchronizer')
1175
1176     def is_running(self):
1177         with self.lock: return self.running
1178
1179
1180     def subscribe_to_addresses(self, addresses):
1181         messages = []
1182         for addr in addresses:
1183             messages.append(('blockchain.address.subscribe', [addr]))
1184         self.interface.send( messages, 'synchronizer')
1185
1186
1187     def run(self):
1188         with self.lock: self.running = True
1189
1190         requested_tx = []
1191         missing_tx = []
1192         requested_histories = {}
1193
1194         # request any missing transactions
1195         for history in self.wallet.history.values():
1196             if history == ['*']: continue
1197             for tx_hash, tx_height in history:
1198                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1199                     missing_tx.append( (tx_hash, tx_height) )
1200         print_error("missing tx", missing_tx)
1201
1202         # wait until we are connected, in case the user is not connected
1203         while not self.interface.is_connected:
1204             time.sleep(1)
1205
1206         # subscriptions
1207         self.subscribe_to_addresses(self.wallet.addresses(True))
1208
1209         while self.is_running():
1210             # 1. create new addresses
1211             new_addresses = self.wallet.synchronize()
1212
1213             # request missing addresses
1214             if new_addresses:
1215                 self.subscribe_to_addresses(new_addresses)
1216
1217             # request missing transactions
1218             for tx_hash, tx_height in missing_tx:
1219                 if (tx_hash, tx_height) not in requested_tx:
1220                     self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
1221                     requested_tx.append( (tx_hash, tx_height) )
1222             missing_tx = []
1223
1224             # detect if situation has changed
1225             if not self.interface.is_up_to_date('synchronizer'):
1226                 if self.wallet.is_up_to_date():
1227                     self.wallet.set_up_to_date(False)
1228                     self.was_updated = True
1229             else:
1230                 if not self.wallet.is_up_to_date():
1231                     self.wallet.set_up_to_date(True)
1232                     self.was_updated = True
1233
1234             if self.was_updated:
1235                 self.interface.trigger_callback('updated')
1236                 self.was_updated = False
1237
1238             # 2. get a response
1239             r = self.interface.get_response('synchronizer')
1240
1241             # poke sends None. (needed during stop)
1242             if not r: continue
1243
1244             # 3. handle response
1245             method = r['method']
1246             params = r['params']
1247             result = r.get('result')
1248             error = r.get('error')
1249             if error:
1250                 print "error", r
1251                 continue
1252
1253             if method == 'blockchain.address.subscribe':
1254                 addr = params[0]
1255                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1256                     if requested_histories.get(addr) is None:
1257                         self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
1258                         requested_histories[addr] = result
1259
1260             elif method == 'blockchain.address.get_history':
1261                 addr = params[0]
1262                 print_error("receiving history", addr, result)
1263                 if result == ['*']:
1264                     assert requested_histories.pop(addr) == '*'
1265                     self.wallet.receive_history_callback(addr, result)
1266                 else:
1267                     hist = []
1268                     # check that txids are unique
1269                     txids = []
1270                     for item in result:
1271                         tx_hash = item['tx_hash']
1272                         if tx_hash not in txids:
1273                             txids.append(tx_hash)
1274                             hist.append( (tx_hash, item['height']) )
1275
1276                     if len(hist) != len(result):
1277                         raise BaseException("error: server sent history with non-unique txid", result)
1278
1279                     # check that the status corresponds to what was announced
1280                     rs = requested_histories.pop(addr)
1281                     if self.wallet.get_status(hist) != rs:
1282                         raise BaseException("error: status mismatch: %s"%addr)
1283
1284                     # store received history
1285                     self.wallet.receive_history_callback(addr, hist)
1286
1287                     # request transactions that we don't have
1288                     for tx_hash, tx_height in hist:
1289                         if self.wallet.transactions.get(tx_hash) is None:
1290                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1291                                 missing_tx.append( (tx_hash, tx_height) )
1292
1293             elif method == 'blockchain.transaction.get':
1294                 tx_hash = params[0]
1295                 tx_height = params[1]
1296                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1297                 tx = Transaction(result)
1298                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1299                 self.was_updated = True
1300                 requested_tx.remove( (tx_hash, tx_height) )
1301                 print_error("received tx:", tx_hash, len(tx.raw))
1302
1303             elif method == 'blockchain.transaction.broadcast':
1304                 self.wallet.tx_result = result
1305                 self.wallet.tx_event.set()
1306
1307             else:
1308                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1309
1310             if self.was_updated and not requested_tx:
1311                 self.interface.trigger_callback('updated')
1312                 self.interface.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1313
1314
1315                 self.was_updated = False