lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, user_dir, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36
  37 # AES encryption
  38 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  39 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  40
  41 def pw_encode(s, password):
  42     if password:
  43         secret = Hash(password)
  44         return EncodeAES(secret, s)
  45     else:
  46         return s
  47
  48 def pw_decode(s, password):
  49     if password is not None:
  50         secret = Hash(password)
  51         try:
  52             d = DecodeAES(secret, s)
  53         except:
  54             raise BaseException('Invalid password')
  55         return d
  56     else:
  57         return s
  58
  59
  60
  61
  62
  63 from version import ELECTRUM_VERSION, SEED_VERSION
  64
  65
  66 class Wallet:
  67     def __init__(self, config={}):
  68
  69         self.config = config
  70         self.electrum_version = ELECTRUM_VERSION
  71         self.gap_limit_for_change = 3 # constant
  72
  73         # saved fields
  74         self.seed_version          = config.get('seed_version', SEED_VERSION)
  75         self.gap_limit             = config.get('gap_limit', 5)
  76         self.use_change            = config.get('use_change',True)
  77         self.fee                   = int(config.get('fee_per_kb',20000))
  78         self.num_zeros             = int(config.get('num_zeros',0))
  79         self.use_encryption        = config.get('use_encryption', False)
  80         self.seed                  = config.get('seed', '')               # encrypted
  81         self.labels                = config.get('labels', {})
  82         self.frozen_addresses      = config.get('frozen_addresses',[])
  83         self.prioritized_addresses = config.get('prioritized_addresses',[])
  84         self.addressbook           = config.get('contacts', [])
  85
  86         self.imported_keys         = config.get('imported_keys',{})
  87         self.history               = config.get('addr_history',{})        # address -> list(txid, height)
  88
  89
  90         self.master_public_keys = config.get('master_public_keys',{})
  91         self.master_private_keys = config.get('master_private_keys', {})
  92
  93         self.first_addresses = config.get('first_addresses',{})
  94
  95         self.load_accounts(config)
  96
  97         self.transactions = {}
  98         tx = config.get('transactions',{})
  99         try:
 100             for k,v in tx.items(): self.transactions[k] = Transaction(v)
 101         except:
 102             print_msg("Warning: Cannot deserialize transactions. skipping")
 103
 104         # not saved
 105         self.prevout_values = {}     # my own transaction outputs
 106         self.spent_outputs = []
 107
 108         # spv
 109         self.verifier = None
 110
 111         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 112         # interface.is_up_to_date() returns true when all requests have been answered and processed
 113         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 114
 115         self.up_to_date = False
 116         self.lock = threading.Lock()
 117         self.transaction_lock = threading.Lock()
 118         self.tx_event = threading.Event()
 119
 120         if self.seed_version != SEED_VERSION:
 121             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
 122
 123         for tx_hash, tx in self.transactions.items():
 124             if self.check_new_tx(tx_hash, tx):
 125                 self.update_tx_outputs(tx_hash)
 126             else:
 127                 print_error("unreferenced tx", tx_hash)
 128                 self.transactions.pop(tx_hash)
 129
 130
 131     def set_up_to_date(self,b):
 132         with self.lock: self.up_to_date = b
 133
 134     def is_up_to_date(self):
 135         with self.lock: return self.up_to_date
 136
 137     def update(self):
 138         self.up_to_date = False
 139         self.interface.poke('synchronizer')
 140         while not self.is_up_to_date(): time.sleep(0.1)
 141
 142     def import_key(self, sec, password):
 143         # check password
 144         seed = self.decode_seed(password)
 145         try:
 146             address = address_from_private_key(sec)
 147         except:
 148             raise BaseException('Invalid private key')
 149
 150         if self.is_mine(address):
 151             raise BaseException('Address already in wallet')
 152
 153         # store the originally requested keypair into the imported keys table
 154         self.imported_keys[address] = pw_encode(sec, password )
 155         self.config.set_key('imported_keys', self.imported_keys, True)
 156         return address
 157
 158     def delete_imported_key(self, addr):
 159         if addr in self.imported_keys:
 160             self.imported_keys.pop(addr)
 161             self.config.set_key('imported_keys', self.imported_keys, True)
 162
 163
 164     def init_seed(self, seed):
 165         if self.seed: raise BaseException("a seed exists")
 166         if not seed:
 167             seed = random_seed(128)
 168         self.seed = seed
 169
 170
 171     def save_seed(self):
 172         self.config.set_key('seed', self.seed, True)
 173         self.config.set_key('seed_version', self.seed_version, True)
 174
 175         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 176
 177         # normal accounts
 178         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 179         # p2sh 2of2
 180         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 181         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 182         # p2sh 2of3
 183         k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 184         k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 185         k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 186
 187         self.master_public_keys = {
 188             "m/0'/": (c0, K0, cK0),
 189             "m/1'/": (c1, K1, cK1),
 190             "m/2'/": (c2, K2, cK2),
 191             "m/3'/": (c3, K3, cK3),
 192             "m/4'/": (c4, K4, cK4),
 193             "m/5'/": (c5, K5, cK5)
 194             }
 195
 196         self.master_private_keys = {
 197             "m/0'/": k0,
 198             "m/1'/": k1,
 199             "m/2'/": k2,
 200             "m/3'/": k3,
 201             "m/4'/": k4,
 202             "m/5'/": k5
 203             }
 204
 205         self.config.set_key('master_public_keys', self.master_public_keys, True)
 206         self.config.set_key('master_private_keys', self.master_private_keys, True)
 207
 208         # create default account
 209         self.create_account('1','Main account')
 210
 211
 212     def find_root_by_master_key(self, c, K):
 213         for key, v in self.master_public_keys.items():
 214             if key == "m/":continue
 215             cc, KK, _ = v
 216             if (c == cc) and (K == KK):
 217                 return key
 218
 219     def deseed_root(self, seed, password):
 220         # for safety, we ask the user to enter their seed
 221         assert seed == self.decode_seed(password)
 222         self.seed = ''
 223         self.config.set_key('seed', '', True)
 224
 225
 226     def deseed_branch(self, k):
 227         # check that parent has no seed
 228         assert self.seed == ''
 229         self.master_private_keys.pop(k)
 230         self.config.set_key('master_private_keys', self.master_private_keys, True)
 231
 232
 233     def account_id(self, account_type, i):
 234         if account_type == '1':
 235             return "m/0'/%d"%i
 236         elif account_type == '2of2':
 237             return "m/1'/%d & m/2'/%d"%(i,i)
 238         elif account_type == '2of3':
 239             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 240         else:
 241             raise BaseException('unknown account type')
 242
 243
 244     def num_accounts(self, account_type):
 245         keys = self.accounts.keys()
 246         i = 0
 247         while True:
 248             account_id = self.account_id(account_type, i)
 249             if account_id not in keys: break
 250             i += 1
 251         return i
 252
 253
 254     def new_account_address(self, account_type = '1'):
 255         i = self.num_accounts(account_type)
 256         k = self.account_id(account_type,i)
 257
 258         addr = self.first_addresses.get(k)
 259         if not addr:
 260             account_id, account = self.next_account(account_type)
 261             addr = account.first_address()
 262             self.first_addresses[k] = addr
 263             self.config.set_key('first_addresses',self.first_addresses)
 264
 265         return addr
 266
 267
 268     def next_account(self, account_type = '1'):
 269
 270         i = self.num_accounts(account_type)
 271         account_id = self.account_id(account_type,i)
 272
 273         if account_type is '1':
 274             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 275             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 276             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 277
 278         elif account_type == '2of2':
 279             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 280             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 281             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 282             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 283             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 284
 285         elif account_type == '2of3':
 286             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 287             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 288             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 289             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 290             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 291             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 292             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 293
 294         return account_id, account
 295
 296
 297     def create_account(self, account_type = '1', name = 'unnamed'):
 298         account_id, account = self.next_account(account_type)
 299         self.accounts[account_id] = account
 300         self.save_accounts()
 301         self.labels[account_id] = name
 302         self.config.set_key('labels', self.labels, True)
 303
 304
 305     def save_accounts(self):
 306         d = {}
 307         for k, v in self.accounts.items():
 308             d[k] = v.dump()
 309         self.config.set_key('accounts', d, True)
 310
 311
 312     def load_accounts(self, config):
 313         d = config.get('accounts', {})
 314         self.accounts = {}
 315         for k, v in d.items():
 316             if '&' in k:
 317                 self.accounts[k] = BIP32_Account_2of2(v)
 318             else:
 319                 self.accounts[k] = BIP32_Account(v)
 320
 321
 322     def addresses(self, include_change = True):
 323         o = self.get_account_addresses(-1, include_change)
 324         for a in self.accounts.keys():
 325             o += self.get_account_addresses(a, include_change)
 326         o += self.first_addresses.values()
 327         return o
 328
 329
 330     def is_mine(self, address):
 331         return address in self.addresses(True)
 332
 333
 334     def is_change(self, address):
 335         if not self.is_mine(address): return False
 336         if address in self.imported_keys.keys(): return False
 337         acct, s = self.get_address_index(address)
 338         return s[0] == 1
 339
 340     def get_master_public_key(self):
 341         raise
 342         return self.config.get("master_public_key")
 343
 344     def get_master_private_key(self, account, password):
 345         master_k = pw_decode( self.master_private_keys[account], password)
 346         master_c, master_K, master_Kc = self.master_public_keys[account]
 347         try:
 348             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 349             assert K.encode('hex') == master_K
 350         except:
 351             raise BaseException("Invalid password")
 352         return master_k
 353
 354
 355     def get_address_index(self, address):
 356         if address in self.imported_keys.keys():
 357             return -1, None
 358
 359         for account in self.accounts.keys():
 360             for for_change in [0,1]:
 361                 addresses = self.accounts[account].get_addresses(for_change)
 362                 for addr in addresses:
 363                     if address == addr:
 364                         return account, (for_change, addresses.index(addr))
 365
 366         raise BaseException("not found")
 367
 368
 369     def rebase_sequence(self, account, sequence):
 370         c, i = sequence
 371         dd = []
 372         for a in account.split('&'):
 373             s = a.strip()
 374             m = re.match("(m/\d+'/)(\d+)", s)
 375             root = m.group(1)
 376             num = int(m.group(2))
 377             dd.append( (root, [num,c,i] ) )
 378         return dd
 379
 380
 381     def get_keyID(self, account, sequence):
 382         rs = self.rebase_sequence(account, sequence)
 383         dd = []
 384         for root, public_sequence in rs:
 385             c, K, _ = self.master_public_keys[root]
 386             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 387             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 388         return '&'.join(dd)
 389
 390
 391     def get_public_key(self, address):
 392         account, sequence = self.get_address_index(address)
 393         return self.accounts[account].get_pubkey( *sequence )
 394
 395
 396     def decode_seed(self, password):
 397         seed = pw_decode(self.seed, password)
 398         #todo:  #self.sequences[0].check_seed(seed)
 399         return seed
 400
 401
 402     def get_private_key(self, address, password):
 403         out = []
 404         if address in self.imported_keys.keys():
 405             out.append( pw_decode( self.imported_keys[address], password ) )
 406         else:
 407             account, sequence = self.get_address_index(address)
 408             # assert address == self.accounts[account].get_address(*sequence)
 409             rs = self.rebase_sequence( account, sequence)
 410             for root, public_sequence in rs:
 411
 412                 if root not in self.master_private_keys.keys(): continue
 413                 master_k = self.get_master_private_key(root, password)
 414                 master_c, _, _ = self.master_public_keys[root]
 415                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 416                 out.append(pk)
 417
 418         return out
 419
 420
 421
 422
 423     def signrawtransaction(self, tx, input_info, private_keys, password):
 424         import deserialize
 425         unspent_coins = self.get_unspent_coins()
 426         seed = self.decode_seed(password)
 427
 428         # build a list of public/private keys
 429         keypairs = {}
 430         for sec in private_keys:
 431             pubkey = public_key_from_private_key(sec)
 432             keypairs[ pubkey ] = sec
 433
 434
 435         for txin in tx.inputs:
 436             # convert to own format
 437             txin['tx_hash'] = txin['prevout_hash']
 438             txin['index'] = txin['prevout_n']
 439
 440             for item in input_info:
 441                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 442                     txin['raw_output_script'] = item['scriptPubKey']
 443                     txin['redeemScript'] = item.get('redeemScript')
 444                     txin['KeyID'] = item.get('KeyID')
 445                     break
 446             else:
 447                 for item in unspent_coins:
 448                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 449                         print_error( "tx input is in unspent coins" )
 450                         txin['raw_output_script'] = item['raw_output_script']
 451                         account, sequence = self.get_address_index(item['address'])
 452                         if account != -1:
 453                             txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 454                         break
 455                 else:
 456                     raise BaseException("Unknown transaction input. Please provide the 'input_info' parameter, or synchronize this wallet")
 457
 458             # if available, derive private_keys from KeyID
 459             keyid = txin.get('KeyID')
 460             if keyid:
 461                 roots = []
 462                 for s in keyid.split('&'):
 463                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 464                     if not m: continue
 465                     c = m.group(1)
 466                     K = m.group(2)
 467                     sequence = m.group(3)
 468                     root = self.find_root_by_master_key(c,K)
 469                     if not root: continue
 470                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 471                     root = root + '%d'%sequence[0]
 472                     sequence = sequence[1:]
 473                     roots.append((root,sequence))
 474
 475                 account_id = " & ".join( map(lambda x:x[0], roots) )
 476                 account = self.accounts.get(account_id)
 477                 if not account: continue
 478                 addr = account.get_address(*sequence)
 479                 txin['address'] = addr
 480                 pk = self.get_private_key(addr, password)
 481                 for sec in pk:
 482                     pubkey = public_key_from_private_key(sec)
 483                     keypairs[pubkey] = sec
 484
 485             redeem_script = txin.get("redeemScript")
 486             print_error( "p2sh:", "yes" if redeem_script else "no")
 487             if redeem_script:
 488                 addr = hash_160_to_bc_address(hash_160(redeem_script.decode('hex')), 5)
 489             else:
 490                 addr = deserialize.get_address_from_output_script(txin["raw_output_script"].decode('hex'))
 491             txin['address'] = addr
 492
 493             # add private keys that are in the wallet
 494             pk = self.get_private_key(addr, password)
 495             for sec in pk:
 496                 pubkey = public_key_from_private_key(sec)
 497                 keypairs[pubkey] = sec
 498                 if not redeem_script:
 499                     txin['redeemPubkey'] = pubkey
 500
 501             print txin
 502
 503         tx.sign( keypairs )
 504
 505     def sign_message(self, address, message, password):
 506         sec = self.get_private_key(address, password)
 507         key = regenerate_key(sec)
 508         compressed = is_compressed(sec)
 509         return key.sign_message(message, compressed, address)
 510
 511     def verify_message(self, address, signature, message):
 512         try:
 513             EC_KEY.verify_message(address, signature, message)
 514             return True
 515         except BaseException as e:
 516             print_error("Verification error: {0}".format(e))
 517             return False
 518
 519
 520     def change_gap_limit(self, value):
 521         if value >= self.gap_limit:
 522             self.gap_limit = value
 523             self.config.set_key('gap_limit', self.gap_limit, True)
 524             self.interface.poke('synchronizer')
 525             return True
 526
 527         elif value >= self.min_acceptable_gap():
 528             for key, account in self.accounts.items():
 529                 addresses = account[0]
 530                 k = self.num_unused_trailing_addresses(addresses)
 531                 n = len(addresses) - k + value
 532                 addresses = addresses[0:n]
 533                 self.accounts[key][0] = addresses
 534
 535             self.gap_limit = value
 536             self.config.set_key('gap_limit', self.gap_limit, True)
 537             self.save_accounts()
 538             return True
 539         else:
 540             return False
 541
 542     def num_unused_trailing_addresses(self, addresses):
 543         k = 0
 544         for a in addresses[::-1]:
 545             if self.history.get(a):break
 546             k = k + 1
 547         return k
 548
 549     def min_acceptable_gap(self):
 550         # fixme: this assumes wallet is synchronized
 551         n = 0
 552         nmax = 0
 553
 554         for account in self.accounts.values():
 555             addresses = account.get_addresses(0)
 556             k = self.num_unused_trailing_addresses(addresses)
 557             for a in addresses[0:-k]:
 558                 if self.history.get(a):
 559                     n = 0
 560                 else:
 561                     n += 1
 562                     if n > nmax: nmax = n
 563         return nmax + 1
 564
 565
 566     def address_is_old(self, address):
 567         age = -1
 568         h = self.history.get(address, [])
 569         if h == ['*']:
 570             return True
 571         for tx_hash, tx_height in h:
 572             if tx_height == 0:
 573                 tx_age = 0
 574             else:
 575                 tx_age = self.verifier.height - tx_height + 1
 576             if tx_age > age:
 577                 age = tx_age
 578         return age > 2
 579
 580
 581     def synchronize_sequence(self, account, for_change):
 582         limit = self.gap_limit_for_change if for_change else self.gap_limit
 583         new_addresses = []
 584         while True:
 585             addresses = account.get_addresses(for_change)
 586             if len(addresses) < limit:
 587                 address = account.create_new_address(for_change)
 588                 self.history[address] = []
 589                 new_addresses.append( address )
 590                 continue
 591
 592             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 593                 break
 594             else:
 595                 address = account.create_new_address(for_change)
 596                 self.history[address] = []
 597                 new_addresses.append( address )
 598
 599         return new_addresses
 600
 601
 602
 603     def create_pending_accounts(self):
 604         for account_type in ['1','2of2','2of3']:
 605             a = self.new_account_address(account_type)
 606             if self.address_is_old(a):
 607                 print "creating account", a
 608                 self.create_account(account_type)
 609
 610
 611     def synchronize_account(self, account):
 612         new = []
 613         new += self.synchronize_sequence(account, 0)
 614         new += self.synchronize_sequence(account, 1)
 615         return new
 616
 617
 618     def synchronize(self):
 619         self.create_pending_accounts()
 620         new = []
 621         for account in self.accounts.values():
 622             new += self.synchronize_account(account)
 623         if new:
 624             self.save_accounts()
 625             self.config.set_key('addr_history', self.history, True)
 626         return new
 627
 628
 629     def is_found(self):
 630         return self.history.values() != [[]] * len(self.history)
 631
 632
 633     def add_contact(self, address, label=None):
 634         self.addressbook.append(address)
 635         self.config.set_key('contacts', self.addressbook, True)
 636         if label:
 637             self.labels[address] = label
 638             self.config.set_key('labels', self.labels, True)
 639
 640     def delete_contact(self, addr):
 641         if addr in self.addressbook:
 642             self.addressbook.remove(addr)
 643             self.config.set_key('addressbook', self.addressbook, True)
 644
 645
 646     def fill_addressbook(self):
 647         for tx_hash, tx in self.transactions.items():
 648             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 649             if is_send:
 650                 for addr, v in tx.outputs:
 651                     if not self.is_mine(addr) and addr not in self.addressbook:
 652                         self.addressbook.append(addr)
 653         # redo labels
 654         # self.update_tx_labels()
 655
 656     def get_num_tx(self, address):
 657         n = 0
 658         for tx in self.transactions.values():
 659             if address in map(lambda x:x[0], tx.outputs): n += 1
 660         return n
 661
 662
 663     def get_address_flags(self, addr):
 664         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 665         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 666         return flags
 667
 668
 669     def get_tx_value(self, tx, account=None):
 670         domain = self.get_account_addresses(account)
 671         return tx.get_value(domain, self.prevout_values)
 672
 673
 674     def update_tx_outputs(self, tx_hash):
 675         tx = self.transactions.get(tx_hash)
 676
 677         for i, (addr, value) in enumerate(tx.outputs):
 678             key = tx_hash+ ':%d'%i
 679             self.prevout_values[key] = value
 680
 681         for item in tx.inputs:
 682             if self.is_mine(item.get('address')):
 683                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 684                 self.spent_outputs.append(key)
 685
 686
 687     def get_addr_balance(self, address):
 688         assert self.is_mine(address)
 689         h = self.history.get(address,[])
 690         if h == ['*']: return 0,0
 691         c = u = 0
 692         received_coins = []   # list of coins received at address
 693
 694         for tx_hash, tx_height in h:
 695             tx = self.transactions.get(tx_hash)
 696             if not tx: continue
 697
 698             for i, (addr, value) in enumerate(tx.outputs):
 699                 if addr == address:
 700                     key = tx_hash + ':%d'%i
 701                     received_coins.append(key)
 702
 703         for tx_hash, tx_height in h:
 704             tx = self.transactions.get(tx_hash)
 705             if not tx: continue
 706             v = 0
 707
 708             for item in tx.inputs:
 709                 addr = item.get('address')
 710                 if addr == address:
 711                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 712                     value = self.prevout_values.get( key )
 713                     if key in received_coins:
 714                         v -= value
 715
 716             for i, (addr, value) in enumerate(tx.outputs):
 717                 key = tx_hash + ':%d'%i
 718                 if addr == address:
 719                     v += value
 720
 721             if tx_height:
 722                 c += v
 723             else:
 724                 u += v
 725         return c, u
 726
 727
 728     def get_accounts(self):
 729         accounts = {}
 730         for k, account in self.accounts.items():
 731             accounts[k] = self.labels.get(k, 'unnamed')
 732         if self.imported_keys:
 733             accounts[-1] = 'Imported keys'
 734         return accounts
 735
 736     def get_account_addresses(self, a, include_change=True):
 737         if a is None:
 738             o = self.addresses(True)
 739         elif a == -1:
 740             o = self.imported_keys.keys()
 741         else:
 742             ac = self.accounts[a]
 743             o = ac.get_addresses(0)
 744             if include_change: o += ac.get_addresses(1)
 745         return o
 746
 747     def get_imported_balance(self):
 748         cc = uu = 0
 749         for addr in self.imported_keys.keys():
 750             c, u = self.get_addr_balance(addr)
 751             cc += c
 752             uu += u
 753         return cc, uu
 754
 755     def get_account_balance(self, account):
 756         if account is None:
 757             return self.get_balance()
 758         elif account == -1:
 759             return self.get_imported_balance()
 760
 761         conf = unconf = 0
 762         for addr in self.get_account_addresses(account):
 763             c, u = self.get_addr_balance(addr)
 764             conf += c
 765             unconf += u
 766         return conf, unconf
 767
 768     def get_frozen_balance(self):
 769         conf = unconf = 0
 770         for addr in self.frozen_addresses:
 771             c, u = self.get_addr_balance(addr)
 772             conf += c
 773             unconf += u
 774         return conf, unconf
 775
 776
 777     def get_balance(self):
 778         cc = uu = 0
 779         for a in self.accounts.keys():
 780             c, u = self.get_account_balance(a)
 781             cc += c
 782             uu += u
 783         c, u = self.get_imported_balance()
 784         cc += c
 785         uu += u
 786         return cc, uu
 787
 788
 789     def get_unspent_coins(self, domain=None):
 790         coins = []
 791         if domain is None: domain = self.addresses(True)
 792         for addr in domain:
 793             h = self.history.get(addr, [])
 794             if h == ['*']: continue
 795             for tx_hash, tx_height in h:
 796                 tx = self.transactions.get(tx_hash)
 797                 if tx is None: raise BaseException("Wallet not synchronized")
 798                 for output in tx.d.get('outputs'):
 799                     if output.get('address') != addr: continue
 800                     key = tx_hash + ":%d" % output.get('index')
 801                     if key in self.spent_outputs: continue
 802                     output['tx_hash'] = tx_hash
 803                     coins.append(output)
 804         return coins
 805
 806
 807
 808     def choose_tx_inputs( self, amount, fixed_fee, account = None ):
 809         """ todo: minimize tx size """
 810         total = 0
 811         fee = self.fee if fixed_fee is None else fixed_fee
 812         domain = self.get_account_addresses(account)
 813         coins = []
 814         prioritized_coins = []
 815         for i in self.frozen_addresses:
 816             if i in domain: domain.remove(i)
 817
 818         for i in self.prioritized_addresses:
 819             if i in domain: domain.remove(i)
 820
 821         coins = self.get_unspent_coins(domain)
 822         prioritized_coins = self.get_unspent_coins(self.prioritized_addresses)
 823
 824         inputs = []
 825         coins = prioritized_coins + coins
 826
 827         for item in coins:
 828             addr = item.get('address')
 829             v = item.get('value')
 830             total += v
 831             inputs.append( item )
 832             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
 833             if total >= amount + fee: break
 834         else:
 835             inputs = []
 836
 837         return inputs, total, fee
 838
 839
 840     def estimated_fee(self, inputs):
 841         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
 842         fee = self.fee * int(round(estimated_size/1024.))
 843         if fee == 0: fee = self.fee
 844         return fee
 845
 846
 847     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None, account=0 ):
 848         "add change to a transaction"
 849         change_amount = total - ( amount + fee )
 850         if change_amount != 0:
 851             if not change_addr:
 852                 if account is None:
 853                     # send change to one of the accounts involved in the tx
 854                     address = inputs[0].get('address')
 855                     account, _ = self.get_address_index(address)
 856
 857                 if not self.use_change or account == -1:
 858                     change_addr = inputs[-1]['address']
 859                 else:
 860                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
 861
 862             # Insert the change output at a random position in the outputs
 863             posn = random.randint(0, len(outputs))
 864             outputs[posn:posn] = [( change_addr,  change_amount)]
 865         return outputs
 866
 867
 868     def get_history(self, address):
 869         with self.lock:
 870             return self.history.get(address)
 871
 872
 873     def get_status(self, h):
 874         if not h: return None
 875         if h == ['*']: return '*'
 876         status = ''
 877         for tx_hash, height in h:
 878             status += tx_hash + ':%d:' % height
 879         return hashlib.sha256( status ).digest().encode('hex')
 880
 881
 882     def receive_tx_callback(self, tx_hash, tx, tx_height):
 883         if not self.check_new_tx(tx_hash, tx):
 884             # may happen due to pruning
 885             print_error("received transaction that is no longer referenced in history", tx_hash)
 886             return
 887
 888         with self.transaction_lock:
 889             self.transactions[tx_hash] = tx
 890
 891             self.interface.pending_transactions_for_notifications.append(tx)
 892
 893             self.save_transactions()
 894             if self.verifier and tx_height>0:
 895                 self.verifier.add(tx_hash, tx_height)
 896             self.update_tx_outputs(tx_hash)
 897
 898
 899     def save_transactions(self):
 900         tx = {}
 901         for k,v in self.transactions.items():
 902             tx[k] = str(v)
 903         self.config.set_key('transactions', tx, True)
 904
 905     def receive_history_callback(self, addr, hist):
 906
 907         if not self.check_new_history(addr, hist):
 908             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
 909
 910         with self.lock:
 911             self.history[addr] = hist
 912             self.config.set_key('addr_history', self.history, True)
 913
 914         if hist != ['*']:
 915             for tx_hash, tx_height in hist:
 916                 if tx_height>0:
 917                     # add it in case it was previously unconfirmed
 918                     if self.verifier: self.verifier.add(tx_hash, tx_height)
 919
 920
 921     def get_tx_history(self, account=None):
 922         with self.transaction_lock:
 923             history = self.transactions.items()
 924             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
 925             result = []
 926
 927             balance = 0
 928             for tx_hash, tx in history:
 929                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
 930                 if v is not None: balance += v
 931
 932             c, u = self.get_account_balance(account)
 933
 934             if balance != c+u:
 935                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
 936
 937             balance = c + u - balance
 938             for tx_hash, tx in history:
 939                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
 940                 if not is_relevant:
 941                     continue
 942                 if value is not None:
 943                     balance += value
 944
 945                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
 946                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
 947
 948         return result
 949
 950
 951     def get_label(self, tx_hash):
 952         label = self.labels.get(tx_hash)
 953         is_default = (label == '') or (label is None)
 954         if is_default: label = self.get_default_label(tx_hash)
 955         return label, is_default
 956
 957
 958     def get_default_label(self, tx_hash):
 959         tx = self.transactions.get(tx_hash)
 960         default_label = ''
 961         if tx:
 962             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
 963             if is_mine:
 964                 for o in tx.outputs:
 965                     o_addr, _ = o
 966                     if not self.is_mine(o_addr):
 967                         try:
 968                             default_label = self.labels[o_addr]
 969                         except KeyError:
 970                             default_label = o_addr
 971                         break
 972                 else:
 973                     default_label = '(internal)'
 974             else:
 975                 for o in tx.outputs:
 976                     o_addr, _ = o
 977                     if self.is_mine(o_addr) and not self.is_change(o_addr):
 978                         break
 979                 else:
 980                     for o in tx.outputs:
 981                         o_addr, _ = o
 982                         if self.is_mine(o_addr):
 983                             break
 984                     else:
 985                         o_addr = None
 986
 987                 if o_addr:
 988                     dest_label = self.labels.get(o_addr)
 989                     try:
 990                         default_label = self.labels[o_addr]
 991                     except KeyError:
 992                         default_label = o_addr
 993
 994         return default_label
 995
 996
 997     def mktx(self, outputs, password, fee=None, change_addr=None, account=None ):
 998
 999         for address, x in outputs:
1000             assert is_valid(address)
1001
1002         amount = sum( map(lambda x:x[1], outputs) )
1003
1004         inputs, total, fee = self.choose_tx_inputs( amount, fee, account )
1005         if not inputs:
1006             raise ValueError("Not enough funds")
1007
1008         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr, account)
1009         tx = Transaction.from_io(inputs, outputs)
1010
1011         keypairs = {}
1012         for i, txin in enumerate(tx.inputs):
1013             address = txin['address']
1014
1015             account, sequence = self.get_address_index(address)
1016             txin['KeyID'] = self.get_keyID(account, sequence)
1017
1018             redeemScript = self.accounts[account].redeem_script(sequence)
1019             if redeemScript:
1020                 txin['redeemScript'] = redeemScript
1021             else:
1022                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1023
1024             private_keys = self.get_private_key(address, password)
1025
1026             for sec in private_keys:
1027                 pubkey = public_key_from_private_key(sec)
1028                 keypairs[ pubkey ] = sec
1029
1030         tx.sign(keypairs)
1031         for address, x in outputs:
1032             if address not in self.addressbook and not self.is_mine(address):
1033                 self.addressbook.append(address)
1034
1035         return tx
1036
1037
1038
1039     def sendtx(self, tx):
1040         # synchronous
1041         h = self.send_tx(tx)
1042         self.tx_event.wait()
1043         return self.receive_tx(h)
1044
1045     def send_tx(self, tx):
1046         # asynchronous
1047         self.tx_event.clear()
1048         self.interface.send([('blockchain.transaction.broadcast', [str(tx)])], 'synchronizer')
1049         return tx.hash()
1050
1051     def receive_tx(self,tx_hash):
1052         out = self.tx_result
1053         if out != tx_hash:
1054             return False, "error: " + out
1055         return True, out
1056
1057
1058
1059     def update_password(self, seed, old_password, new_password):
1060         if new_password == '': new_password = None
1061         # this will throw an exception if unicode cannot be converted
1062         self.seed = pw_encode( seed, new_password)
1063         self.config.set_key('seed', self.seed, True)
1064         self.use_encryption = (new_password != None)
1065         self.config.set_key('use_encryption', self.use_encryption,True)
1066         for k in self.imported_keys.keys():
1067             a = self.imported_keys[k]
1068             b = pw_decode(a, old_password)
1069             c = pw_encode(b, new_password)
1070             self.imported_keys[k] = c
1071         self.config.set_key('imported_keys', self.imported_keys, True)
1072
1073         for k, v in self.master_private_keys.items():
1074             b = pw_decode(v, old_password)
1075             c = pw_encode(b, new_password)
1076             self.master_private_keys[k] = c
1077         self.config.set_key('master_private_keys', self.master_private_keys, True)
1078
1079
1080     def freeze(self,addr):
1081         if self.is_mine(addr) and addr not in self.frozen_addresses:
1082             self.unprioritize(addr)
1083             self.frozen_addresses.append(addr)
1084             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
1085             return True
1086         else:
1087             return False
1088
1089     def unfreeze(self,addr):
1090         if self.is_mine(addr) and addr in self.frozen_addresses:
1091             self.frozen_addresses.remove(addr)
1092             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
1093             return True
1094         else:
1095             return False
1096
1097     def prioritize(self,addr):
1098         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1099             self.unfreeze(addr)
1100             self.prioritized_addresses.append(addr)
1101             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1102             return True
1103         else:
1104             return False
1105
1106     def unprioritize(self,addr):
1107         if self.is_mine(addr) and addr in self.prioritized_addresses:
1108             self.prioritized_addresses.remove(addr)
1109             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1110             return True
1111         else:
1112             return False
1113
1114     def set_fee(self, fee):
1115         if self.fee != fee:
1116             self.fee = fee
1117             self.config.set_key('fee_per_kb', self.fee, True)
1118
1119
1120     def save(self):
1121         print_error("Warning: wallet.save() is deprecated")
1122         tx = {}
1123         for k,v in self.transactions.items():
1124             tx[k] = str(v)
1125
1126         s = {
1127             'use_change': self.use_change,
1128             'fee_per_kb': self.fee,
1129             'addr_history': self.history,
1130             'labels': self.labels,
1131             'contacts': self.addressbook,
1132             'num_zeros': self.num_zeros,
1133             'frozen_addresses': self.frozen_addresses,
1134             'prioritized_addresses': self.prioritized_addresses,
1135             'gap_limit': self.gap_limit,
1136             'transactions': tx,
1137         }
1138         for k, v in s.items():
1139             self.config.set_key(k,v)
1140         self.config.save()
1141
1142     def set_verifier(self, verifier):
1143         self.verifier = verifier
1144
1145         # review transactions that are in the history
1146         for addr, hist in self.history.items():
1147             if hist == ['*']: continue
1148             for tx_hash, tx_height in hist:
1149                 if tx_height>0:
1150                     # add it in case it was previously unconfirmed
1151                     self.verifier.add(tx_hash, tx_height)
1152
1153
1154         # if we are on a pruning server, remove unverified transactions
1155         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1156         for tx_hash in self.transactions.keys():
1157             if tx_hash not in vr:
1158                 self.transactions.pop(tx_hash)
1159
1160
1161
1162     def check_new_history(self, addr, hist):
1163
1164         # check that all tx in hist are relevant
1165         if hist != ['*']:
1166             for tx_hash, height in hist:
1167                 tx = self.transactions.get(tx_hash)
1168                 if not tx: continue
1169                 if not tx.has_address(addr):
1170                     return False
1171
1172         # check that we are not "orphaning" a transaction
1173         old_hist = self.history.get(addr,[])
1174         if old_hist == ['*']: return True
1175
1176         for tx_hash, height in old_hist:
1177             if tx_hash in map(lambda x:x[0], hist): continue
1178             found = False
1179             for _addr, _hist in self.history.items():
1180                 if _addr == addr: continue
1181                 if _hist == ['*']: continue
1182                 _tx_hist = map(lambda x:x[0], _hist)
1183                 if tx_hash in _tx_hist:
1184                     found = True
1185                     break
1186
1187             if not found:
1188                 tx = self.transactions.get(tx_hash)
1189                 # tx might not be there
1190                 if not tx: continue
1191
1192                 # already verified?
1193                 if self.verifier.get_height(tx_hash):
1194                     continue
1195                 # unconfirmed tx
1196                 print_error("new history is orphaning transaction:", tx_hash)
1197                 # check that all outputs are not mine, request histories
1198                 ext_requests = []
1199                 for _addr, _v in tx.outputs:
1200                     # assert not self.is_mine(_addr)
1201                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1202
1203                 ext_h = self.interface.synchronous_get(ext_requests)
1204                 print_error("sync:", ext_requests, ext_h)
1205                 height = None
1206                 for h in ext_h:
1207                     if h == ['*']: continue
1208                     for item in h:
1209                         if item.get('tx_hash') == tx_hash:
1210                             height = item.get('height')
1211                 if height:
1212                     print_error("found height for", tx_hash, height)
1213                     self.verifier.add(tx_hash, height)
1214                 else:
1215                     print_error("removing orphaned tx from history", tx_hash)
1216                     self.transactions.pop(tx_hash)
1217
1218         return True
1219
1220
1221
1222     def check_new_tx(self, tx_hash, tx):
1223         # 1 check that tx is referenced in addr_history.
1224         addresses = []
1225         for addr, hist in self.history.items():
1226             if hist == ['*']:continue
1227             for txh, height in hist:
1228                 if txh == tx_hash:
1229                     addresses.append(addr)
1230
1231         if not addresses:
1232             return False
1233
1234         # 2 check that referencing addresses are in the tx
1235         for addr in addresses:
1236             if not tx.has_address(addr):
1237                 return False
1238
1239         return True
1240
1241
1242
1243
1244 class WalletSynchronizer(threading.Thread):
1245
1246
1247     def __init__(self, wallet, config):
1248         threading.Thread.__init__(self)
1249         self.daemon = True
1250         self.wallet = wallet
1251         wallet.synchronizer = self
1252         self.interface = self.wallet.interface
1253         self.interface.register_channel('synchronizer')
1254         self.wallet.interface.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1255         self.was_updated = True
1256         self.running = False
1257         self.lock = threading.Lock()
1258
1259     def stop(self):
1260         with self.lock: self.running = False
1261         self.interface.poke('synchronizer')
1262
1263     def is_running(self):
1264         with self.lock: return self.running
1265
1266
1267     def subscribe_to_addresses(self, addresses):
1268         messages = []
1269         for addr in addresses:
1270             messages.append(('blockchain.address.subscribe', [addr]))
1271         self.interface.send( messages, 'synchronizer')
1272
1273
1274     def run(self):
1275         with self.lock: self.running = True
1276
1277         requested_tx = []
1278         missing_tx = []
1279         requested_histories = {}
1280
1281         # request any missing transactions
1282         for history in self.wallet.history.values():
1283             if history == ['*']: continue
1284             for tx_hash, tx_height in history:
1285                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1286                     missing_tx.append( (tx_hash, tx_height) )
1287         print_error("missing tx", missing_tx)
1288
1289         # wait until we are connected, in case the user is not connected
1290         while not self.interface.is_connected:
1291             time.sleep(1)
1292
1293         # subscriptions
1294         self.subscribe_to_addresses(self.wallet.addresses(True))
1295
1296         while self.is_running():
1297             # 1. create new addresses
1298             new_addresses = self.wallet.synchronize()
1299
1300             # request missing addresses
1301             if new_addresses:
1302                 self.subscribe_to_addresses(new_addresses)
1303
1304             # request missing transactions
1305             for tx_hash, tx_height in missing_tx:
1306                 if (tx_hash, tx_height) not in requested_tx:
1307                     self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
1308                     requested_tx.append( (tx_hash, tx_height) )
1309             missing_tx = []
1310
1311             # detect if situation has changed
1312             if not self.interface.is_up_to_date('synchronizer'):
1313                 if self.wallet.is_up_to_date():
1314                     self.wallet.set_up_to_date(False)
1315                     self.was_updated = True
1316             else:
1317                 if not self.wallet.is_up_to_date():
1318                     self.wallet.set_up_to_date(True)
1319                     self.was_updated = True
1320
1321             if self.was_updated:
1322                 self.interface.trigger_callback('updated')
1323                 self.was_updated = False
1324
1325             # 2. get a response
1326             r = self.interface.get_response('synchronizer')
1327
1328             # poke sends None. (needed during stop)
1329             if not r: continue
1330
1331             # 3. handle response
1332             method = r['method']
1333             params = r['params']
1334             result = r.get('result')
1335             error = r.get('error')
1336             if error:
1337                 print "error", r
1338                 continue
1339
1340             if method == 'blockchain.address.subscribe':
1341                 addr = params[0]
1342                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1343                     if requested_histories.get(addr) is None:
1344                         self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
1345                         requested_histories[addr] = result
1346
1347             elif method == 'blockchain.address.get_history':
1348                 addr = params[0]
1349                 print_error("receiving history", addr, result)
1350                 if result == ['*']:
1351                     assert requested_histories.pop(addr) == '*'
1352                     self.wallet.receive_history_callback(addr, result)
1353                 else:
1354                     hist = []
1355                     # check that txids are unique
1356                     txids = []
1357                     for item in result:
1358                         tx_hash = item['tx_hash']
1359                         if tx_hash not in txids:
1360                             txids.append(tx_hash)
1361                             hist.append( (tx_hash, item['height']) )
1362
1363                     if len(hist) != len(result):
1364                         raise BaseException("error: server sent history with non-unique txid", result)
1365
1366                     # check that the status corresponds to what was announced
1367                     rs = requested_histories.pop(addr)
1368                     if self.wallet.get_status(hist) != rs:
1369                         raise BaseException("error: status mismatch: %s"%addr)
1370
1371                     # store received history
1372                     self.wallet.receive_history_callback(addr, hist)
1373
1374                     # request transactions that we don't have
1375                     for tx_hash, tx_height in hist:
1376                         if self.wallet.transactions.get(tx_hash) is None:
1377                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1378                                 missing_tx.append( (tx_hash, tx_height) )
1379
1380             elif method == 'blockchain.transaction.get':
1381                 tx_hash = params[0]
1382                 tx_height = params[1]
1383                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1384                 tx = Transaction(result)
1385                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1386                 self.was_updated = True
1387                 requested_tx.remove( (tx_hash, tx_height) )
1388                 print_error("received tx:", tx_hash, len(tx.raw))
1389
1390             elif method == 'blockchain.transaction.broadcast':
1391                 self.wallet.tx_result = result
1392                 self.wallet.tx_event.set()
1393
1394             else:
1395                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1396
1397             if self.was_updated and not requested_tx:
1398                 self.interface.trigger_callback('updated')
1399                 self.interface.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1400
1401
1402                 self.was_updated = False