lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37
  38 # AES encryption
  39 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  40 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  41
  42 def pw_encode(s, password):
  43     if password:
  44         secret = Hash(password)
  45         return EncodeAES(secret, s)
  46     else:
  47         return s
  48
  49 def pw_decode(s, password):
  50     if password is not None:
  51         secret = Hash(password)
  52         try:
  53             d = DecodeAES(secret, s)
  54         except:
  55             raise BaseException('Invalid password')
  56         return d
  57     else:
  58         return s
  59
  60
  61
  62
  63
  64 from version import ELECTRUM_VERSION, SEED_VERSION
  65
  66
  67 class WalletStorage:
  68
  69     def __init__(self, config):
  70         self.data = {}
  71         self.file_exists = False
  72         self.init_path(config)
  73         print_error( "wallet path", self.path )
  74         if self.path:
  75             self.read(self.path)
  76
  77
  78     def init_path(self, config):
  79         """Set the path of the wallet."""
  80
  81         path = config.get('wallet_path')
  82         if not path:
  83             path = config.get('default_wallet_path')
  84         if path is not None:
  85             self.path = path
  86             return
  87
  88         self.path = os.path.join(config.path, "electrum.dat")
  89
  90
  91     def read(self, path):
  92         """Read the contents of the wallet file."""
  93         try:
  94             with open(self.path, "r") as f:
  95                 data = f.read()
  96         except IOError:
  97             return
  98         try:
  99             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 100         except:
 101             raise IOError("Cannot read wallet file.")
 102
 103         self.data = d
 104         self.file_exists = True
 105
 106
 107     def get(self, key, default=None):
 108         return self.data.get(key, default)
 109
 110     def put(self, key, value, save = True):
 111
 112         if self.data.get(key) is not None:
 113             self.data[key] = value
 114         else:
 115             # add key to wallet config
 116             self.data[key] = value
 117
 118         if save:
 119             self.write()
 120
 121
 122     def write(self):
 123         s = repr(self.data)
 124         f = open(self.path,"w")
 125         f.write( s )
 126         f.close()
 127         if self.get('gui') != 'android':
 128             import stat
 129             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 130
 131
 132 class Wallet:
 133
 134     def __init__(self, storage):
 135
 136         self.storage = storage
 137         self.electrum_version = ELECTRUM_VERSION
 138         self.gap_limit_for_change = 3 # constant
 139
 140         # saved fields
 141         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 142
 143         self.gap_limit             = storage.get('gap_limit', 5)
 144         self.use_change            = storage.get('use_change',True)
 145         self.use_encryption        = storage.get('use_encryption', False)
 146         self.seed                  = storage.get('seed', '')               # encrypted
 147         self.labels                = storage.get('labels', {})
 148         self.frozen_addresses      = storage.get('frozen_addresses',[])
 149         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 150         self.addressbook           = storage.get('contacts', [])
 151
 152         self.imported_keys         = storage.get('imported_keys',{})
 153         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 154
 155         self.fee                   = int(storage.get('fee_per_kb',20000))
 156
 157         self.master_public_keys = storage.get('master_public_keys',{})
 158         self.master_private_keys = storage.get('master_private_keys', {})
 159
 160         self.next_addresses = storage.get('next_addresses',{})
 161
 162         if self.seed_version < 4:
 163             raise ValueError("This wallet seed is deprecated.")
 164
 165         self.load_accounts()
 166
 167         self.transactions = {}
 168         tx_list = self.storage.get('transactions',{})
 169         for k,v in tx_list.items():
 170             try:
 171                 tx = Transaction(v)
 172             except:
 173                 print_msg("Warning: Cannot deserialize transactions. skipping")
 174                 continue
 175
 176             self.add_extra_addresses(tx)
 177             self.transactions[k] = tx
 178
 179         for h,tx in self.transactions.items():
 180             if not self.check_new_tx(h, tx):
 181                 print_error("removing unreferenced tx", h)
 182                 self.transactions.pop(h)
 183
 184
 185         # not saved
 186         self.prevout_values = {}     # my own transaction outputs
 187         self.spent_outputs = []
 188
 189         # spv
 190         self.verifier = None
 191
 192         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 193         # interface.is_up_to_date() returns true when all requests have been answered and processed
 194         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 195
 196         self.up_to_date = False
 197         self.lock = threading.Lock()
 198         self.transaction_lock = threading.Lock()
 199         self.tx_event = threading.Event()
 200
 201         for tx_hash, tx in self.transactions.items():
 202             self.update_tx_outputs(tx_hash)
 203
 204
 205     def add_extra_addresses(self, tx):
 206         h = tx.hash()
 207         # find the address corresponding to pay-to-pubkey inputs
 208         tx.add_extra_addresses(self.transactions)
 209         for o in tx.d.get('outputs'):
 210             if o.get('is_pubkey'):
 211                 for tx2 in self.transactions.values():
 212                     tx2.add_extra_addresses({h:tx})
 213
 214
 215
 216
 217     def set_up_to_date(self,b):
 218         with self.lock: self.up_to_date = b
 219
 220
 221     def is_up_to_date(self):
 222         with self.lock: return self.up_to_date
 223
 224
 225     def update(self):
 226         self.up_to_date = False
 227         while not self.is_up_to_date():
 228             time.sleep(0.1)
 229
 230
 231     def import_key(self, sec, password):
 232         # check password
 233         seed = self.decode_seed(password)
 234         try:
 235             address = address_from_private_key(sec)
 236         except:
 237             raise BaseException('Invalid private key')
 238
 239         if self.is_mine(address):
 240             raise BaseException('Address already in wallet')
 241
 242         # store the originally requested keypair into the imported keys table
 243         self.imported_keys[address] = pw_encode(sec, password )
 244         self.storage.put('imported_keys', self.imported_keys, True)
 245         if self.synchronizer:
 246             self.synchronizer.subscribe_to_addresses([address])
 247         return address
 248
 249     def delete_imported_key(self, addr):
 250         if addr in self.imported_keys:
 251             self.imported_keys.pop(addr)
 252             self.storage.put('imported_keys', self.imported_keys, True)
 253
 254
 255     def init_seed(self, seed):
 256         if self.seed: raise BaseException("a seed exists")
 257         if not seed:
 258             seed = random_seed(128)
 259         self.seed = seed
 260
 261
 262     def save_seed(self):
 263         self.storage.put('seed', self.seed, True)
 264         self.storage.put('seed_version', self.seed_version, True)
 265
 266     def create_watching_only_wallet(self, c0, K0):
 267         cK0 = ""
 268         self.master_public_keys = {
 269             "m/0'/": (c0, K0, cK0),
 270             }
 271         self.storage.put('master_public_keys', self.master_public_keys, True)
 272         self.create_account('1','Main account')
 273
 274     def create_accounts(self):
 275
 276         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 277
 278         # normal accounts
 279         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 280         # p2sh 2of2
 281         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 282         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 283         # p2sh 2of3
 284         k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 285         k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 286         k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 287
 288         self.master_public_keys = {
 289             "m/0'/": (c0, K0, cK0),
 290             "m/1'/": (c1, K1, cK1),
 291             "m/2'/": (c2, K2, cK2),
 292             "m/3'/": (c3, K3, cK3),
 293             "m/4'/": (c4, K4, cK4),
 294             "m/5'/": (c5, K5, cK5)
 295             }
 296
 297         self.master_private_keys = {
 298             "m/0'/": k0,
 299             "m/1'/": k1,
 300             "m/2'/": k2,
 301             "m/3'/": k3,
 302             "m/4'/": k4,
 303             "m/5'/": k5
 304             }
 305
 306         self.storage.put('master_public_keys', self.master_public_keys, True)
 307         self.storage.put('master_private_keys', self.master_private_keys, True)
 308
 309         # create default account
 310         self.create_account('1','Main account')
 311
 312
 313     def find_root_by_master_key(self, c, K):
 314         for key, v in self.master_public_keys.items():
 315             if key == "m/":continue
 316             cc, KK, _ = v
 317             if (c == cc) and (K == KK):
 318                 return key
 319
 320     def deseed_root(self, seed, password):
 321         # for safety, we ask the user to enter their seed
 322         assert seed == self.decode_seed(password)
 323         self.seed = ''
 324         self.storage.put('seed', '', True)
 325
 326
 327     def deseed_branch(self, k):
 328         # check that parent has no seed
 329         assert self.seed == ''
 330         self.master_private_keys.pop(k)
 331         self.storage.put('master_private_keys', self.master_private_keys, True)
 332
 333
 334     def account_id(self, account_type, i):
 335         if account_type == '1':
 336             return "m/0'/%d"%i
 337         elif account_type == '2of2':
 338             return "m/1'/%d & m/2'/%d"%(i,i)
 339         elif account_type == '2of3':
 340             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 341         else:
 342             raise BaseException('unknown account type')
 343
 344
 345     def num_accounts(self, account_type):
 346         keys = self.accounts.keys()
 347         i = 0
 348         while True:
 349             account_id = self.account_id(account_type, i)
 350             if account_id not in keys: break
 351             i += 1
 352         return i
 353
 354
 355     def new_account_address(self, account_type = '1'):
 356         i = self.num_accounts(account_type)
 357         k = self.account_id(account_type,i)
 358
 359         addr = self.next_addresses.get(k)
 360         if not addr:
 361             account_id, account = self.next_account(account_type)
 362             addr = account.first_address()
 363             self.next_addresses[k] = addr
 364             self.storage.put('next_addresses',self.next_addresses)
 365
 366         return addr
 367
 368
 369     def next_account(self, account_type = '1'):
 370
 371         i = self.num_accounts(account_type)
 372         account_id = self.account_id(account_type,i)
 373
 374         if account_type is '1':
 375             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 376             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 377             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 378
 379         elif account_type == '2of2':
 380             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 381             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 382             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 383             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 384             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 385
 386         elif account_type == '2of3':
 387             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 388             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 389             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 390             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 391             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 392             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 393             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 394
 395         return account_id, account
 396
 397
 398     def set_label(self, key, value):
 399         self.labels[key] = value
 400         self.storage.put('labels', self.labels, True)
 401
 402
 403     def create_account(self, account_type = '1', name = None):
 404         account_id, account = self.next_account(account_type)
 405         self.accounts[account_id] = account
 406         self.save_accounts()
 407         if name:
 408             self.set_label(account_id, name)
 409
 410
 411     def create_old_account(self):
 412         mpk = OldAccount.mpk_from_seed(self.seed)
 413         self.storage.put('master_public_key', mpk, True)
 414         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 415         self.save_accounts()
 416
 417
 418     def save_accounts(self):
 419         d = {}
 420         for k, v in self.accounts.items():
 421             d[k] = v.dump()
 422         self.storage.put('accounts', d, True)
 423
 424
 425
 426     def load_accounts(self):
 427         d = self.storage.get('accounts', {})
 428         self.accounts = {}
 429         for k, v in d.items():
 430             if k == 0:
 431                 v['mpk'] = self.storage.get('master_public_key')
 432                 self.accounts[k] = OldAccount(v)
 433             elif '&' in k:
 434                 self.accounts[k] = BIP32_Account_2of2(v)
 435             else:
 436                 self.accounts[k] = BIP32_Account(v)
 437
 438
 439     def addresses(self, include_change = True, next=False):
 440         o = self.get_account_addresses(-1, include_change)
 441         for a in self.accounts.keys():
 442             o += self.get_account_addresses(a, include_change)
 443
 444         if next:
 445             for addr in self.next_addresses.values():
 446                 if addr not in o:
 447                     o += [addr]
 448         return o
 449
 450
 451     def is_mine(self, address):
 452         return address in self.addresses(True)
 453
 454
 455     def is_change(self, address):
 456         if not self.is_mine(address): return False
 457         if address in self.imported_keys.keys(): return False
 458         acct, s = self.get_address_index(address)
 459         if s is None: return False
 460         return s[0] == 1
 461
 462     def get_master_public_key(self):
 463         if self.seed_version == 4:
 464             return self.storage.get("master_public_key")
 465         else:
 466             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 467             return repr((c, K))
 468
 469     def get_master_private_key(self, account, password):
 470         master_k = pw_decode( self.master_private_keys[account], password)
 471         master_c, master_K, master_Kc = self.master_public_keys[account]
 472         try:
 473             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 474             assert K.encode('hex') == master_K
 475         except:
 476             raise BaseException("Invalid password")
 477         return master_k
 478
 479
 480     def get_address_index(self, address):
 481         if address in self.imported_keys.keys():
 482             return -1, None
 483
 484         for account in self.accounts.keys():
 485             for for_change in [0,1]:
 486                 addresses = self.accounts[account].get_addresses(for_change)
 487                 for addr in addresses:
 488                     if address == addr:
 489                         return account, (for_change, addresses.index(addr))
 490
 491         raise BaseException("Address not found", address)
 492
 493
 494     def rebase_sequence(self, account, sequence):
 495         c, i = sequence
 496         dd = []
 497         for a in account.split('&'):
 498             s = a.strip()
 499             m = re.match("(m/\d+'/)(\d+)", s)
 500             root = m.group(1)
 501             num = int(m.group(2))
 502             dd.append( (root, [num,c,i] ) )
 503         return dd
 504
 505
 506     def get_keyID(self, account, sequence):
 507         if account == 0:
 508             return 'old'
 509
 510         rs = self.rebase_sequence(account, sequence)
 511         dd = []
 512         for root, public_sequence in rs:
 513             c, K, _ = self.master_public_keys[root]
 514             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 515             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 516         return '&'.join(dd)
 517
 518
 519     def get_public_key(self, address):
 520         account, sequence = self.get_address_index(address)
 521         return self.accounts[account].get_pubkey( *sequence )
 522
 523
 524     def decode_seed(self, password):
 525         seed = pw_decode(self.seed, password)
 526         #todo:  #self.sequences[0].check_seed(seed)
 527         return seed
 528
 529
 530     def get_private_key(self, address, password):
 531         out = []
 532         if address in self.imported_keys.keys():
 533             out.append( pw_decode( self.imported_keys[address], password ) )
 534         else:
 535             account, sequence = self.get_address_index(address)
 536             if account == 0:
 537                 seed = self.decode_seed(password)
 538                 pk = self.accounts[account].get_private_key(seed, sequence)
 539                 out.append(pk)
 540                 return out
 541
 542             # assert address == self.accounts[account].get_address(*sequence)
 543             rs = self.rebase_sequence( account, sequence)
 544             for root, public_sequence in rs:
 545
 546                 if root not in self.master_private_keys.keys(): continue
 547                 master_k = self.get_master_private_key(root, password)
 548                 master_c, _, _ = self.master_public_keys[root]
 549                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 550                 out.append(pk)
 551
 552         return out
 553
 554
 555
 556
 557     def signrawtransaction(self, tx, input_info, private_keys, password):
 558
 559         unspent_coins = self.get_unspent_coins()
 560         seed = self.decode_seed(password)
 561
 562         # build a list of public/private keys
 563         keypairs = {}
 564         for sec in private_keys:
 565             pubkey = public_key_from_private_key(sec)
 566             keypairs[ pubkey ] = sec
 567
 568
 569         for txin in tx.inputs:
 570             # convert to own format
 571             txin['tx_hash'] = txin['prevout_hash']
 572             txin['index'] = txin['prevout_n']
 573
 574             for item in input_info:
 575                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 576                     txin['raw_output_script'] = item['scriptPubKey']
 577                     txin['redeemScript'] = item.get('redeemScript')
 578                     txin['KeyID'] = item.get('KeyID')
 579                     break
 580             else:
 581                 for item in unspent_coins:
 582                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 583                         print_error( "tx input is in unspent coins" )
 584                         txin['raw_output_script'] = item['raw_output_script']
 585                         account, sequence = self.get_address_index(item['address'])
 586                         if account != -1:
 587                             txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 588                         break
 589                 else:
 590                     raise BaseException("Unknown transaction input. Please provide the 'input_info' parameter, or synchronize this wallet")
 591
 592             # if available, derive private_keys from KeyID
 593             keyid = txin.get('KeyID')
 594             if keyid:
 595                 roots = []
 596                 for s in keyid.split('&'):
 597                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 598                     if not m: continue
 599                     c = m.group(1)
 600                     K = m.group(2)
 601                     sequence = m.group(3)
 602                     root = self.find_root_by_master_key(c,K)
 603                     if not root: continue
 604                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 605                     root = root + '%d'%sequence[0]
 606                     sequence = sequence[1:]
 607                     roots.append((root,sequence))
 608
 609                 account_id = " & ".join( map(lambda x:x[0], roots) )
 610                 account = self.accounts.get(account_id)
 611                 if not account: continue
 612                 addr = account.get_address(*sequence)
 613                 txin['address'] = addr
 614                 pk = self.get_private_key(addr, password)
 615                 for sec in pk:
 616                     pubkey = public_key_from_private_key(sec)
 617                     keypairs[pubkey] = sec
 618
 619             redeem_script = txin.get("redeemScript")
 620             print_error( "p2sh:", "yes" if redeem_script else "no")
 621             if redeem_script:
 622                 addr = hash_160_to_bc_address(hash_160(redeem_script.decode('hex')), 5)
 623             else:
 624                 import transaction
 625                 _, addr = transaction.get_address_from_output_script(txin["raw_output_script"].decode('hex'))
 626             txin['address'] = addr
 627
 628             # add private keys that are in the wallet
 629             pk = self.get_private_key(addr, password)
 630             for sec in pk:
 631                 pubkey = public_key_from_private_key(sec)
 632                 keypairs[pubkey] = sec
 633                 if not redeem_script:
 634                     txin['redeemPubkey'] = pubkey
 635
 636             print txin
 637
 638         tx.sign( keypairs )
 639
 640     def sign_message(self, address, message, password):
 641         keys = self.get_private_key(address, password)
 642         assert len(keys) == 1
 643         sec = keys[0]
 644         key = regenerate_key(sec)
 645         compressed = is_compressed(sec)
 646         return key.sign_message(message, compressed, address)
 647
 648     def verify_message(self, address, signature, message):
 649         try:
 650             EC_KEY.verify_message(address, signature, message)
 651             return True
 652         except BaseException as e:
 653             print_error("Verification error: {0}".format(e))
 654             return False
 655
 656
 657     def change_gap_limit(self, value):
 658         if value >= self.gap_limit:
 659             self.gap_limit = value
 660             self.storage.put('gap_limit', self.gap_limit, True)
 661             #self.interface.poke('synchronizer')
 662             return True
 663
 664         elif value >= self.min_acceptable_gap():
 665             for key, account in self.accounts.items():
 666                 addresses = account[0]
 667                 k = self.num_unused_trailing_addresses(addresses)
 668                 n = len(addresses) - k + value
 669                 addresses = addresses[0:n]
 670                 self.accounts[key][0] = addresses
 671
 672             self.gap_limit = value
 673             self.storage.put('gap_limit', self.gap_limit, True)
 674             self.save_accounts()
 675             return True
 676         else:
 677             return False
 678
 679     def num_unused_trailing_addresses(self, addresses):
 680         k = 0
 681         for a in addresses[::-1]:
 682             if self.history.get(a):break
 683             k = k + 1
 684         return k
 685
 686     def min_acceptable_gap(self):
 687         # fixme: this assumes wallet is synchronized
 688         n = 0
 689         nmax = 0
 690
 691         for account in self.accounts.values():
 692             addresses = account.get_addresses(0)
 693             k = self.num_unused_trailing_addresses(addresses)
 694             for a in addresses[0:-k]:
 695                 if self.history.get(a):
 696                     n = 0
 697                 else:
 698                     n += 1
 699                     if n > nmax: nmax = n
 700         return nmax + 1
 701
 702
 703     def address_is_old(self, address):
 704         age = -1
 705         h = self.history.get(address, [])
 706         if h == ['*']:
 707             return True
 708         for tx_hash, tx_height in h:
 709             if tx_height == 0:
 710                 tx_age = 0
 711             else:
 712                 tx_age = self.verifier.blockchain.height - tx_height + 1
 713             if tx_age > age:
 714                 age = tx_age
 715         return age > 2
 716
 717
 718     def synchronize_sequence(self, account, for_change):
 719         limit = self.gap_limit_for_change if for_change else self.gap_limit
 720         new_addresses = []
 721         while True:
 722             addresses = account.get_addresses(for_change)
 723             if len(addresses) < limit:
 724                 address = account.create_new_address(for_change)
 725                 self.history[address] = []
 726                 new_addresses.append( address )
 727                 continue
 728
 729             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 730                 break
 731             else:
 732                 address = account.create_new_address(for_change)
 733                 self.history[address] = []
 734                 new_addresses.append( address )
 735
 736         return new_addresses
 737
 738
 739
 740     def create_pending_accounts(self):
 741         for account_type in ['1','2of2','2of3']:
 742             a = self.new_account_address(account_type)
 743             if self.address_is_old(a):
 744                 print_error( "creating account", a )
 745                 self.create_account(account_type)
 746
 747
 748     def synchronize_account(self, account):
 749         new = []
 750         new += self.synchronize_sequence(account, 0)
 751         new += self.synchronize_sequence(account, 1)
 752         return new
 753
 754
 755     def synchronize(self):
 756         if self.master_public_keys:
 757             self.create_pending_accounts()
 758         new = []
 759         for account in self.accounts.values():
 760             new += self.synchronize_account(account)
 761         if new:
 762             self.save_accounts()
 763             self.storage.put('addr_history', self.history, True)
 764         return new
 765
 766
 767     def is_found(self):
 768         return self.history.values() != [[]] * len(self.history)
 769
 770
 771     def add_contact(self, address, label=None):
 772         self.addressbook.append(address)
 773         self.storage.put('contacts', self.addressbook, True)
 774         if label:
 775             self.set_label(address, label)
 776
 777
 778     def delete_contact(self, addr):
 779         if addr in self.addressbook:
 780             self.addressbook.remove(addr)
 781             self.storage.put('addressbook', self.addressbook, True)
 782
 783
 784     def fill_addressbook(self):
 785         for tx_hash, tx in self.transactions.items():
 786             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 787             if is_send:
 788                 for addr, v in tx.outputs:
 789                     if not self.is_mine(addr) and addr not in self.addressbook:
 790                         self.addressbook.append(addr)
 791         # redo labels
 792         # self.update_tx_labels()
 793
 794     def get_num_tx(self, address):
 795         n = 0
 796         for tx in self.transactions.values():
 797             if address in map(lambda x:x[0], tx.outputs): n += 1
 798         return n
 799
 800
 801     def get_address_flags(self, addr):
 802         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 803         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 804         return flags
 805
 806
 807     def get_tx_value(self, tx, account=None):
 808         domain = self.get_account_addresses(account)
 809         return tx.get_value(domain, self.prevout_values)
 810
 811
 812     def update_tx_outputs(self, tx_hash):
 813         tx = self.transactions.get(tx_hash)
 814
 815         for i, (addr, value) in enumerate(tx.outputs):
 816             key = tx_hash+ ':%d'%i
 817             self.prevout_values[key] = value
 818
 819         for item in tx.inputs:
 820             if self.is_mine(item.get('address')):
 821                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 822                 self.spent_outputs.append(key)
 823
 824
 825     def get_addr_balance(self, address):
 826         assert self.is_mine(address)
 827         h = self.history.get(address,[])
 828         if h == ['*']: return 0,0
 829         c = u = 0
 830         received_coins = []   # list of coins received at address
 831
 832         for tx_hash, tx_height in h:
 833             tx = self.transactions.get(tx_hash)
 834             if not tx: continue
 835
 836             for i, (addr, value) in enumerate(tx.outputs):
 837                 if addr == address:
 838                     key = tx_hash + ':%d'%i
 839                     received_coins.append(key)
 840
 841         for tx_hash, tx_height in h:
 842             tx = self.transactions.get(tx_hash)
 843             if not tx: continue
 844             v = 0
 845
 846             for item in tx.inputs:
 847                 addr = item.get('address')
 848                 if addr == address:
 849                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 850                     value = self.prevout_values.get( key )
 851                     if key in received_coins:
 852                         v -= value
 853
 854             for i, (addr, value) in enumerate(tx.outputs):
 855                 key = tx_hash + ':%d'%i
 856                 if addr == address:
 857                     v += value
 858
 859             if tx_height:
 860                 c += v
 861             else:
 862                 u += v
 863         return c, u
 864
 865
 866     def get_account_name(self, k):
 867         if k == 0:
 868             if self.seed_version == 4:
 869                 name = 'Main account'
 870             else:
 871                 name = 'Old account'
 872         else:
 873             name = self.labels.get(k, 'Unnamed account')
 874         return name
 875
 876     def get_account_names(self):
 877         accounts = {}
 878         for k, account in self.accounts.items():
 879             accounts[k] = self.get_account_name(k)
 880         if self.imported_keys:
 881             accounts[-1] = 'Imported keys'
 882         return accounts
 883
 884     def get_account_addresses(self, a, include_change=True):
 885         if a is None:
 886             o = self.addresses(True)
 887         elif a == -1:
 888             o = self.imported_keys.keys()
 889         else:
 890             ac = self.accounts[a]
 891             o = ac.get_addresses(0)
 892             if include_change: o += ac.get_addresses(1)
 893         return o
 894
 895     def get_imported_balance(self):
 896         cc = uu = 0
 897         for addr in self.imported_keys.keys():
 898             c, u = self.get_addr_balance(addr)
 899             cc += c
 900             uu += u
 901         return cc, uu
 902
 903     def get_account_balance(self, account):
 904         if account is None:
 905             return self.get_balance()
 906         elif account == -1:
 907             return self.get_imported_balance()
 908
 909         conf = unconf = 0
 910         for addr in self.get_account_addresses(account):
 911             c, u = self.get_addr_balance(addr)
 912             conf += c
 913             unconf += u
 914         return conf, unconf
 915
 916     def get_frozen_balance(self):
 917         conf = unconf = 0
 918         for addr in self.frozen_addresses:
 919             c, u = self.get_addr_balance(addr)
 920             conf += c
 921             unconf += u
 922         return conf, unconf
 923
 924
 925     def get_balance(self):
 926         cc = uu = 0
 927         for a in self.accounts.keys():
 928             c, u = self.get_account_balance(a)
 929             cc += c
 930             uu += u
 931         c, u = self.get_imported_balance()
 932         cc += c
 933         uu += u
 934         return cc, uu
 935
 936
 937     def get_unspent_coins(self, domain=None):
 938         coins = []
 939         if domain is None: domain = self.addresses(True)
 940         for addr in domain:
 941             h = self.history.get(addr, [])
 942             if h == ['*']: continue
 943             for tx_hash, tx_height in h:
 944                 tx = self.transactions.get(tx_hash)
 945                 if tx is None: raise BaseException("Wallet not synchronized")
 946                 for output in tx.d.get('outputs'):
 947                     if output.get('address') != addr: continue
 948                     key = tx_hash + ":%d" % output.get('index')
 949                     if key in self.spent_outputs: continue
 950                     output['tx_hash'] = tx_hash
 951                     output['height'] = tx_height
 952                     coins.append((tx_height, output))
 953
 954         # sort by age
 955         if coins:
 956             coins = sorted(coins)
 957             if coins[-1][0] != 0:
 958                 while coins[0][0] == 0:
 959                     coins = coins[1:] + [ coins[0] ]
 960         return [x[1] for x in coins]
 961
 962
 963
 964     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
 965         """ todo: minimize tx size """
 966         total = 0
 967         fee = self.fee if fixed_fee is None else fixed_fee
 968         if domain is None:
 969             domain = self.addresses()
 970
 971         for i in self.frozen_addresses:
 972             if i in domain: domain.remove(i)
 973
 974         prioritized = []
 975         for i in self.prioritized_addresses:
 976             if i in domain:
 977                 domain.remove(i)
 978                 prioritized.append(i)
 979
 980         coins = self.get_unspent_coins(domain)
 981         prioritized_coins = self.get_unspent_coins(prioritized)
 982
 983         inputs = []
 984         coins = prioritized_coins + coins
 985
 986         for item in coins:
 987             addr = item.get('address')
 988             v = item.get('value')
 989             total += v
 990             inputs.append( item )
 991             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
 992             if total >= amount + fee: break
 993         else:
 994             inputs = []
 995
 996         return inputs, total, fee
 997
 998
 999     def set_fee(self, fee):
1000         if self.fee != fee:
1001             self.fee = fee
1002             self.storage.put('fee_per_kb', self.fee, True)
1003
1004     def estimated_fee(self, inputs):
1005         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1006         fee = self.fee * int(round(estimated_size/1024.))
1007         if fee == 0: fee = self.fee
1008         return fee
1009
1010
1011     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1012         "add change to a transaction"
1013         change_amount = total - ( amount + fee )
1014         if change_amount != 0:
1015             if not change_addr:
1016
1017                 # send change to one of the accounts involved in the tx
1018                 address = inputs[0].get('address')
1019                 account, _ = self.get_address_index(address)
1020
1021                 if not self.use_change or account == -1:
1022                     change_addr = inputs[-1]['address']
1023                 else:
1024                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1025
1026             # Insert the change output at a random position in the outputs
1027             posn = random.randint(0, len(outputs))
1028             outputs[posn:posn] = [( change_addr,  change_amount)]
1029         return outputs
1030
1031
1032     def get_history(self, address):
1033         with self.lock:
1034             return self.history.get(address)
1035
1036
1037     def get_status(self, h):
1038         if not h: return None
1039         if h == ['*']: return '*'
1040         status = ''
1041         for tx_hash, height in h:
1042             status += tx_hash + ':%d:' % height
1043         return hashlib.sha256( status ).digest().encode('hex')
1044
1045
1046     def receive_tx_callback(self, tx_hash, tx, tx_height):
1047
1048         with self.transaction_lock:
1049             self.add_extra_addresses(tx)
1050             if not self.check_new_tx(tx_hash, tx):
1051                 # may happen due to pruning
1052                 print_error("received transaction that is no longer referenced in history", tx_hash)
1053                 return
1054             self.transactions[tx_hash] = tx
1055             self.network.interface.pending_transactions_for_notifications.append(tx)
1056             self.save_transactions()
1057             if self.verifier and tx_height>0:
1058                 self.verifier.add(tx_hash, tx_height)
1059             self.update_tx_outputs(tx_hash)
1060
1061
1062     def save_transactions(self):
1063         tx = {}
1064         for k,v in self.transactions.items():
1065             tx[k] = str(v)
1066         self.storage.put('transactions', tx, True)
1067
1068     def receive_history_callback(self, addr, hist):
1069
1070         if not self.check_new_history(addr, hist):
1071             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1072
1073         with self.lock:
1074             self.history[addr] = hist
1075             self.storage.put('addr_history', self.history, True)
1076
1077         if hist != ['*']:
1078             for tx_hash, tx_height in hist:
1079                 if tx_height>0:
1080                     # add it in case it was previously unconfirmed
1081                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1082
1083
1084     def get_tx_history(self, account=None):
1085         with self.transaction_lock:
1086             history = self.transactions.items()
1087             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1088             result = []
1089
1090             balance = 0
1091             for tx_hash, tx in history:
1092                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1093                 if v is not None: balance += v
1094
1095             c, u = self.get_account_balance(account)
1096
1097             if balance != c+u:
1098                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1099
1100             balance = c + u - balance
1101             for tx_hash, tx in history:
1102                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1103                 if not is_relevant:
1104                     continue
1105                 if value is not None:
1106                     balance += value
1107
1108                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1109                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1110
1111         return result
1112
1113
1114     def get_label(self, tx_hash):
1115         label = self.labels.get(tx_hash)
1116         is_default = (label == '') or (label is None)
1117         if is_default: label = self.get_default_label(tx_hash)
1118         return label, is_default
1119
1120
1121     def get_default_label(self, tx_hash):
1122         tx = self.transactions.get(tx_hash)
1123         default_label = ''
1124         if tx:
1125             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1126             if is_mine:
1127                 for o in tx.outputs:
1128                     o_addr, _ = o
1129                     if not self.is_mine(o_addr):
1130                         try:
1131                             default_label = self.labels[o_addr]
1132                         except KeyError:
1133                             default_label = o_addr
1134                         break
1135                 else:
1136                     default_label = '(internal)'
1137             else:
1138                 for o in tx.outputs:
1139                     o_addr, _ = o
1140                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1141                         break
1142                 else:
1143                     for o in tx.outputs:
1144                         o_addr, _ = o
1145                         if self.is_mine(o_addr):
1146                             break
1147                     else:
1148                         o_addr = None
1149
1150                 if o_addr:
1151                     dest_label = self.labels.get(o_addr)
1152                     try:
1153                         default_label = self.labels[o_addr]
1154                     except KeyError:
1155                         default_label = o_addr
1156
1157         return default_label
1158
1159
1160     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1161         for address, x in outputs:
1162             assert is_valid(address)
1163         amount = sum( map(lambda x:x[1], outputs) )
1164         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1165         if not inputs:
1166             raise ValueError("Not enough funds")
1167         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1168         return Transaction.from_io(inputs, outputs)
1169
1170
1171     def mktx_from_account(self, outputs, password, fee=None, change_addr=None, account=None):
1172         domain = self.get_account_addresses(account) if account else None
1173         return self.mktx(outputs, password, fee, change_addr, domain)
1174
1175
1176     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1177         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1178         self.sign_transaction(tx, password)
1179         return tx
1180
1181
1182     def sign_transaction(self, tx, password):
1183         keypairs = {}
1184         for i, txin in enumerate(tx.inputs):
1185             address = txin['address']
1186             account, sequence = self.get_address_index(address)
1187             txin['KeyID'] = self.get_keyID(account, sequence)
1188             redeemScript = self.accounts[account].redeem_script(sequence)
1189             if redeemScript:
1190                 txin['redeemScript'] = redeemScript
1191             else:
1192                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1193             private_keys = self.get_private_key(address, password)
1194             for sec in private_keys:
1195                 pubkey = public_key_from_private_key(sec)
1196                 keypairs[ pubkey ] = sec
1197         tx.sign(keypairs)
1198
1199
1200     def sendtx(self, tx):
1201         # synchronous
1202         h = self.send_tx(tx)
1203         self.tx_event.wait()
1204         return self.receive_tx(h)
1205
1206     def send_tx(self, tx):
1207         # asynchronous
1208         self.tx_event.clear()
1209         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1210         return tx.hash()
1211
1212     def on_broadcast(self, i, r):
1213         self.tx_result = r.get('result')
1214         self.tx_event.set()
1215
1216     def receive_tx(self,tx_hash):
1217         out = self.tx_result
1218         if out != tx_hash:
1219             return False, "error: " + out
1220         return True, out
1221
1222
1223
1224     def update_password(self, seed, old_password, new_password):
1225         if new_password == '': new_password = None
1226         # this will throw an exception if unicode cannot be converted
1227         self.seed = pw_encode( seed, new_password)
1228         self.storage.put('seed', self.seed, True)
1229         self.use_encryption = (new_password != None)
1230         self.storage.put('use_encryption', self.use_encryption,True)
1231         for k in self.imported_keys.keys():
1232             a = self.imported_keys[k]
1233             b = pw_decode(a, old_password)
1234             c = pw_encode(b, new_password)
1235             self.imported_keys[k] = c
1236         self.storage.put('imported_keys', self.imported_keys, True)
1237
1238         for k, v in self.master_private_keys.items():
1239             b = pw_decode(v, old_password)
1240             c = pw_encode(b, new_password)
1241             self.master_private_keys[k] = c
1242         self.storage.put('master_private_keys', self.master_private_keys, True)
1243
1244
1245     def freeze(self,addr):
1246         if self.is_mine(addr) and addr not in self.frozen_addresses:
1247             self.unprioritize(addr)
1248             self.frozen_addresses.append(addr)
1249             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1250             return True
1251         else:
1252             return False
1253
1254     def unfreeze(self,addr):
1255         if self.is_mine(addr) and addr in self.frozen_addresses:
1256             self.frozen_addresses.remove(addr)
1257             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1258             return True
1259         else:
1260             return False
1261
1262     def prioritize(self,addr):
1263         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1264             self.unfreeze(addr)
1265             self.prioritized_addresses.append(addr)
1266             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1267             return True
1268         else:
1269             return False
1270
1271     def unprioritize(self,addr):
1272         if self.is_mine(addr) and addr in self.prioritized_addresses:
1273             self.prioritized_addresses.remove(addr)
1274             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1275             return True
1276         else:
1277             return False
1278
1279
1280     def set_verifier(self, verifier):
1281         self.verifier = verifier
1282
1283         # review transactions that are in the history
1284         for addr, hist in self.history.items():
1285             if hist == ['*']: continue
1286             for tx_hash, tx_height in hist:
1287                 if tx_height>0:
1288                     # add it in case it was previously unconfirmed
1289                     self.verifier.add(tx_hash, tx_height)
1290
1291
1292         # if we are on a pruning server, remove unverified transactions
1293         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1294         for tx_hash in self.transactions.keys():
1295             if tx_hash not in vr:
1296                 self.transactions.pop(tx_hash)
1297
1298
1299
1300     def check_new_history(self, addr, hist):
1301
1302         # check that all tx in hist are relevant
1303         if hist != ['*']:
1304             for tx_hash, height in hist:
1305                 tx = self.transactions.get(tx_hash)
1306                 if not tx: continue
1307                 if not tx.has_address(addr):
1308                     return False
1309
1310         # check that we are not "orphaning" a transaction
1311         old_hist = self.history.get(addr,[])
1312         if old_hist == ['*']: return True
1313
1314         for tx_hash, height in old_hist:
1315             if tx_hash in map(lambda x:x[0], hist): continue
1316             found = False
1317             for _addr, _hist in self.history.items():
1318                 if _addr == addr: continue
1319                 if _hist == ['*']: continue
1320                 _tx_hist = map(lambda x:x[0], _hist)
1321                 if tx_hash in _tx_hist:
1322                     found = True
1323                     break
1324
1325             if not found:
1326                 tx = self.transactions.get(tx_hash)
1327                 # tx might not be there
1328                 if not tx: continue
1329
1330                 # already verified?
1331                 if self.verifier.get_height(tx_hash):
1332                     continue
1333                 # unconfirmed tx
1334                 print_error("new history is orphaning transaction:", tx_hash)
1335                 # check that all outputs are not mine, request histories
1336                 ext_requests = []
1337                 for _addr, _v in tx.outputs:
1338                     # assert not self.is_mine(_addr)
1339                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1340
1341                 ext_h = self.network.interface.synchronous_get(ext_requests)
1342                 print_error("sync:", ext_requests, ext_h)
1343                 height = None
1344                 for h in ext_h:
1345                     if h == ['*']: continue
1346                     for item in h:
1347                         if item.get('tx_hash') == tx_hash:
1348                             height = item.get('height')
1349                 if height:
1350                     print_error("found height for", tx_hash, height)
1351                     self.verifier.add(tx_hash, height)
1352                 else:
1353                     print_error("removing orphaned tx from history", tx_hash)
1354                     self.transactions.pop(tx_hash)
1355
1356         return True
1357
1358
1359
1360     def check_new_tx(self, tx_hash, tx):
1361         # 1 check that tx is referenced in addr_history.
1362         addresses = []
1363         for addr, hist in self.history.items():
1364             if hist == ['*']:continue
1365             for txh, height in hist:
1366                 if txh == tx_hash:
1367                     addresses.append(addr)
1368
1369         if not addresses:
1370             return False
1371
1372         # 2 check that referencing addresses are in the tx
1373         for addr in addresses:
1374             if not tx.has_address(addr):
1375                 return False
1376
1377         return True
1378
1379
1380     def start_threads(self, network):
1381         from verifier import TxVerifier
1382         self.network = network
1383         self.verifier = TxVerifier(self.network, self.storage)
1384         self.verifier.start()
1385         self.set_verifier(self.verifier)
1386         self.synchronizer = WalletSynchronizer(self)
1387         self.synchronizer.start()
1388
1389     def stop_threads(self):
1390         self.verifier.stop()
1391         self.synchronizer.stop()
1392
1393
1394
1395
1396
1397 class WalletSynchronizer(threading.Thread):
1398
1399
1400     def __init__(self, wallet):
1401         threading.Thread.__init__(self)
1402         self.daemon = True
1403         self.wallet = wallet
1404         wallet.synchronizer = self
1405         self.network = self.wallet.network
1406         #self.wallet.network.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1407         self.was_updated = True
1408         self.running = False
1409         self.lock = threading.Lock()
1410         self.queue = Queue.Queue()
1411
1412     def stop(self):
1413         with self.lock: self.running = False
1414
1415     def is_running(self):
1416         with self.lock: return self.running
1417
1418
1419     def subscribe_to_addresses(self, addresses):
1420         messages = []
1421         for addr in addresses:
1422             messages.append(('blockchain.address.subscribe', [addr]))
1423         self.network.interface.send( messages, lambda i,r: self.queue.put(r))
1424
1425
1426     def run(self):
1427         with self.lock:
1428             self.running = True
1429
1430         while self.is_running():
1431             interface = self.network.interface
1432             if not interface.is_connected:
1433                 print_error("synchronizer: waiting for interface")
1434                 interface.connect_event.wait()
1435
1436             self.run_interface(interface)
1437
1438
1439     def run_interface(self, interface):
1440
1441         print_error("synchronizer: connected to", interface.server)
1442
1443         requested_tx = []
1444         missing_tx = []
1445         requested_histories = {}
1446
1447         # request any missing transactions
1448         for history in self.wallet.history.values():
1449             if history == ['*']: continue
1450             for tx_hash, tx_height in history:
1451                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1452                     missing_tx.append( (tx_hash, tx_height) )
1453
1454         if missing_tx:
1455             print_error("missing tx", missing_tx)
1456
1457         # subscriptions
1458         self.subscribe_to_addresses(self.wallet.addresses(True, next=True))
1459
1460         while self.is_running():
1461             # 1. create new addresses
1462             new_addresses = self.wallet.synchronize()
1463
1464             # request missing addresses
1465             if new_addresses:
1466                 self.subscribe_to_addresses(new_addresses)
1467
1468             # request missing transactions
1469             for tx_hash, tx_height in missing_tx:
1470                 if (tx_hash, tx_height) not in requested_tx:
1471                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1472                     requested_tx.append( (tx_hash, tx_height) )
1473             missing_tx = []
1474
1475             # detect if situation has changed
1476             if interface.is_up_to_date() and self.queue.empty():
1477                 if not self.wallet.is_up_to_date():
1478                     self.wallet.set_up_to_date(True)
1479                     self.was_updated = True
1480             else:
1481                 if self.wallet.is_up_to_date():
1482                     self.wallet.set_up_to_date(False)
1483                     self.was_updated = True
1484
1485             if self.was_updated:
1486                 self.wallet.network.trigger_callback('updated')
1487                 self.was_updated = False
1488
1489             # 2. get a response
1490             try:
1491                 r = self.queue.get(block=True, timeout=1)
1492             except Queue.Empty:
1493                 continue
1494
1495             if interface != self.network.interface:
1496                 break
1497
1498             if not r:
1499                 continue
1500
1501             # 3. handle response
1502             method = r['method']
1503             params = r['params']
1504             result = r.get('result')
1505             error = r.get('error')
1506             if error:
1507                 print "error", r
1508                 continue
1509
1510             if method == 'blockchain.address.subscribe':
1511                 addr = params[0]
1512                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1513                     if requested_histories.get(addr) is None:
1514                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1515                         requested_histories[addr] = result
1516
1517             elif method == 'blockchain.address.get_history':
1518                 addr = params[0]
1519                 print_error("receiving history", addr, result)
1520                 if result == ['*']:
1521                     assert requested_histories.pop(addr) == '*'
1522                     self.wallet.receive_history_callback(addr, result)
1523                 else:
1524                     hist = []
1525                     # check that txids are unique
1526                     txids = []
1527                     for item in result:
1528                         tx_hash = item['tx_hash']
1529                         if tx_hash not in txids:
1530                             txids.append(tx_hash)
1531                             hist.append( (tx_hash, item['height']) )
1532
1533                     if len(hist) != len(result):
1534                         raise BaseException("error: server sent history with non-unique txid", result)
1535
1536                     # check that the status corresponds to what was announced
1537                     rs = requested_histories.pop(addr)
1538                     if self.wallet.get_status(hist) != rs:
1539                         raise BaseException("error: status mismatch: %s"%addr)
1540
1541                     # store received history
1542                     self.wallet.receive_history_callback(addr, hist)
1543
1544                     # request transactions that we don't have
1545                     for tx_hash, tx_height in hist:
1546                         if self.wallet.transactions.get(tx_hash) is None:
1547                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1548                                 missing_tx.append( (tx_hash, tx_height) )
1549
1550             elif method == 'blockchain.transaction.get':
1551                 tx_hash = params[0]
1552                 tx_height = params[1]
1553                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1554                 tx = Transaction(result)
1555                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1556                 self.was_updated = True
1557                 requested_tx.remove( (tx_hash, tx_height) )
1558                 print_error("received tx:", tx_hash, len(tx.raw))
1559
1560             else:
1561                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1562
1563             if self.was_updated and not requested_tx:
1564                 self.wallet.network.trigger_callback('updated')
1565                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1566
1567                 self.was_updated = False