lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37
  38 # AES encryption
  39 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  40 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  41
  42 def pw_encode(s, password):
  43     if password:
  44         secret = Hash(password)
  45         return EncodeAES(secret, s)
  46     else:
  47         return s
  48
  49 def pw_decode(s, password):
  50     if password is not None:
  51         secret = Hash(password)
  52         try:
  53             d = DecodeAES(secret, s)
  54         except:
  55             raise BaseException('Invalid password')
  56         return d
  57     else:
  58         return s
  59
  60
  61
  62
  63
  64 from version import ELECTRUM_VERSION, SEED_VERSION
  65
  66
  67 class WalletStorage:
  68
  69     def __init__(self, config):
  70         self.data = {}
  71         self.file_exists = False
  72         self.init_path(config)
  73         print_error( "wallet path", self.path )
  74         if self.path:
  75             self.read(self.path)
  76
  77
  78     def init_path(self, config):
  79         """Set the path of the wallet."""
  80
  81         path = config.get('wallet_path')
  82         if not path:
  83             path = config.get('default_wallet_path')
  84         if path is not None:
  85             self.path = path
  86             return
  87
  88         self.path = os.path.join(config.path, "electrum.dat")
  89
  90
  91     def read(self, path):
  92         """Read the contents of the wallet file."""
  93         try:
  94             with open(self.path, "r") as f:
  95                 data = f.read()
  96         except IOError:
  97             return
  98         try:
  99             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 100         except:
 101             raise IOError("Cannot read wallet file.")
 102
 103         self.data = d
 104         self.file_exists = True
 105
 106
 107     def get(self, key, default=None):
 108         return self.data.get(key, default)
 109
 110     def put(self, key, value, save = True):
 111
 112         if self.data.get(key) is not None:
 113             self.data[key] = value
 114         else:
 115             # add key to wallet config
 116             self.data[key] = value
 117
 118         if save:
 119             self.write()
 120
 121
 122     def write(self):
 123         s = repr(self.data)
 124         f = open(self.path,"w")
 125         f.write( s )
 126         f.close()
 127         if self.get('gui') != 'android':
 128             import stat
 129             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 130
 131
 132 class Wallet:
 133
 134     def __init__(self, storage):
 135
 136         self.storage = storage
 137         self.electrum_version = ELECTRUM_VERSION
 138         self.gap_limit_for_change = 3 # constant
 139
 140         # saved fields
 141         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 142
 143         self.gap_limit             = storage.get('gap_limit', 5)
 144         self.use_change            = storage.get('use_change',True)
 145         self.use_encryption        = storage.get('use_encryption', False)
 146         self.seed                  = storage.get('seed', '')               # encrypted
 147         self.labels                = storage.get('labels', {})
 148         self.frozen_addresses      = storage.get('frozen_addresses',[])
 149         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 150         self.addressbook           = storage.get('contacts', [])
 151
 152         self.imported_keys         = storage.get('imported_keys',{})
 153         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 154
 155         self.fee                   = int(storage.get('fee_per_kb',20000))
 156
 157         self.master_public_keys = storage.get('master_public_keys',{})
 158         self.master_private_keys = storage.get('master_private_keys', {})
 159
 160         self.next_addresses = storage.get('next_addresses',{})
 161
 162         if self.seed_version < 4:
 163             raise ValueError("This wallet seed is deprecated.")
 164
 165         self.load_accounts()
 166
 167         self.transactions = {}
 168         tx_list = self.storage.get('transactions',{})
 169         for k,v in tx_list.items():
 170             try:
 171                 tx = Transaction(v)
 172             except:
 173                 print_msg("Warning: Cannot deserialize transactions. skipping")
 174                 continue
 175
 176             self.add_extra_addresses(tx)
 177             self.transactions[k] = tx
 178
 179         for h,tx in self.transactions.items():
 180             if not self.check_new_tx(h, tx):
 181                 print_error("removing unreferenced tx", h)
 182                 self.transactions.pop(h)
 183
 184
 185         # not saved
 186         self.prevout_values = {}     # my own transaction outputs
 187         self.spent_outputs = []
 188
 189         # spv
 190         self.verifier = None
 191
 192         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 193         # interface.is_up_to_date() returns true when all requests have been answered and processed
 194         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 195
 196         self.up_to_date = False
 197         self.lock = threading.Lock()
 198         self.transaction_lock = threading.Lock()
 199         self.tx_event = threading.Event()
 200
 201         for tx_hash, tx in self.transactions.items():
 202             self.update_tx_outputs(tx_hash)
 203
 204
 205     def add_extra_addresses(self, tx):
 206         h = tx.hash()
 207         # find the address corresponding to pay-to-pubkey inputs
 208         tx.add_extra_addresses(self.transactions)
 209         for o in tx.d.get('outputs'):
 210             if o.get('is_pubkey'):
 211                 for tx2 in self.transactions.values():
 212                     tx2.add_extra_addresses({h:tx})
 213
 214
 215
 216
 217     def set_up_to_date(self,b):
 218         with self.lock: self.up_to_date = b
 219
 220
 221     def is_up_to_date(self):
 222         with self.lock: return self.up_to_date
 223
 224
 225     def update(self):
 226         self.up_to_date = False
 227         while not self.is_up_to_date():
 228             time.sleep(0.1)
 229
 230
 231     def import_key(self, sec, password):
 232         # check password
 233         seed = self.decode_seed(password)
 234         try:
 235             address = address_from_private_key(sec)
 236         except:
 237             raise BaseException('Invalid private key')
 238
 239         if self.is_mine(address):
 240             raise BaseException('Address already in wallet')
 241
 242         # store the originally requested keypair into the imported keys table
 243         self.imported_keys[address] = pw_encode(sec, password )
 244         self.storage.put('imported_keys', self.imported_keys, True)
 245         if self.synchronizer:
 246             self.synchronizer.subscribe_to_addresses([address])
 247         return address
 248
 249     def delete_imported_key(self, addr):
 250         if addr in self.imported_keys:
 251             self.imported_keys.pop(addr)
 252             self.storage.put('imported_keys', self.imported_keys, True)
 253
 254
 255     def init_seed(self, seed):
 256         if self.seed: raise BaseException("a seed exists")
 257         if not seed:
 258             seed = random_seed(128)
 259         self.seed = seed
 260
 261
 262     def save_seed(self):
 263         self.storage.put('seed', self.seed, True)
 264         self.storage.put('seed_version', self.seed_version, True)
 265
 266     def create_watching_only_wallet(self, c0, K0):
 267         cK0 = ""
 268         self.master_public_keys = {
 269             "m/0'/": (c0, K0, cK0),
 270             }
 271         self.storage.put('master_public_keys', self.master_public_keys, True)
 272         self.create_account('1','Main account')
 273
 274     def create_accounts(self):
 275
 276         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 277
 278         # normal accounts
 279         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 280         # p2sh 2of2
 281         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 282         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 283         # p2sh 2of3
 284         k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 285         k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 286         k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 287
 288         self.master_public_keys = {
 289             "m/0'/": (c0, K0, cK0),
 290             "m/1'/": (c1, K1, cK1),
 291             "m/2'/": (c2, K2, cK2),
 292             "m/3'/": (c3, K3, cK3),
 293             "m/4'/": (c4, K4, cK4),
 294             "m/5'/": (c5, K5, cK5)
 295             }
 296
 297         self.master_private_keys = {
 298             "m/0'/": k0,
 299             "m/1'/": k1,
 300             "m/2'/": k2,
 301             "m/3'/": k3,
 302             "m/4'/": k4,
 303             "m/5'/": k5
 304             }
 305
 306         self.storage.put('master_public_keys', self.master_public_keys, True)
 307         self.storage.put('master_private_keys', self.master_private_keys, True)
 308
 309         # create default account
 310         self.create_account('1','Main account')
 311
 312
 313     def find_root_by_master_key(self, c, K):
 314         for key, v in self.master_public_keys.items():
 315             if key == "m/":continue
 316             cc, KK, _ = v
 317             if (c == cc) and (K == KK):
 318                 return key
 319
 320     def deseed_root(self, seed, password):
 321         # for safety, we ask the user to enter their seed
 322         assert seed == self.decode_seed(password)
 323         self.seed = ''
 324         self.storage.put('seed', '', True)
 325
 326
 327     def deseed_branch(self, k):
 328         # check that parent has no seed
 329         assert self.seed == ''
 330         self.master_private_keys.pop(k)
 331         self.storage.put('master_private_keys', self.master_private_keys, True)
 332
 333
 334     def account_id(self, account_type, i):
 335         if account_type == '1':
 336             return "m/0'/%d"%i
 337         elif account_type == '2of2':
 338             return "m/1'/%d & m/2'/%d"%(i,i)
 339         elif account_type == '2of3':
 340             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 341         else:
 342             raise BaseException('unknown account type')
 343
 344
 345     def num_accounts(self, account_type):
 346         keys = self.accounts.keys()
 347         i = 0
 348         while True:
 349             account_id = self.account_id(account_type, i)
 350             if account_id not in keys: break
 351             i += 1
 352         return i
 353
 354
 355     def new_account_address(self, account_type = '1'):
 356         i = self.num_accounts(account_type)
 357         k = self.account_id(account_type,i)
 358
 359         addr = self.next_addresses.get(k)
 360         if not addr:
 361             account_id, account = self.next_account(account_type)
 362             addr = account.first_address()
 363             self.next_addresses[k] = addr
 364             self.storage.put('next_addresses',self.next_addresses)
 365
 366         return addr
 367
 368
 369     def next_account(self, account_type = '1'):
 370
 371         i = self.num_accounts(account_type)
 372         account_id = self.account_id(account_type,i)
 373
 374         if account_type is '1':
 375             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 376             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 377             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 378
 379         elif account_type == '2of2':
 380             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 381             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 382             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 383             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 384             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 385
 386         elif account_type == '2of3':
 387             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 388             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 389             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 390             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 391             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 392             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 393             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 394
 395         return account_id, account
 396
 397
 398     def set_label(self, key, value):
 399         self.labels[key] = value
 400         self.storage.put('labels', self.labels, True)
 401
 402
 403     def create_account(self, account_type = '1', name = None):
 404         account_id, account = self.next_account(account_type)
 405         self.accounts[account_id] = account
 406         self.save_accounts()
 407         if name:
 408             self.set_label(account_id, name)
 409
 410
 411     def create_old_account(self):
 412         mpk = OldAccount.mpk_from_seed(self.seed)
 413         self.storage.put('master_public_key', mpk, True)
 414         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 415         self.save_accounts()
 416
 417
 418     def save_accounts(self):
 419         d = {}
 420         for k, v in self.accounts.items():
 421             d[k] = v.dump()
 422         self.storage.put('accounts', d, True)
 423
 424
 425
 426     def load_accounts(self):
 427         d = self.storage.get('accounts', {})
 428         self.accounts = {}
 429         for k, v in d.items():
 430             if k == 0:
 431                 v['mpk'] = self.storage.get('master_public_key')
 432                 self.accounts[k] = OldAccount(v)
 433             elif '&' in k:
 434                 self.accounts[k] = BIP32_Account_2of2(v)
 435             else:
 436                 self.accounts[k] = BIP32_Account(v)
 437
 438
 439     def addresses(self, include_change = True, next=False):
 440         o = self.get_account_addresses(-1, include_change)
 441         for a in self.accounts.keys():
 442             o += self.get_account_addresses(a, include_change)
 443
 444         if next:
 445             for addr in self.next_addresses.values():
 446                 if addr not in o:
 447                     o += [addr]
 448         return o
 449
 450
 451     def is_mine(self, address):
 452         return address in self.addresses(True)
 453
 454
 455     def is_change(self, address):
 456         if not self.is_mine(address): return False
 457         if address in self.imported_keys.keys(): return False
 458         acct, s = self.get_address_index(address)
 459         if s is None: return False
 460         return s[0] == 1
 461
 462     def get_master_public_key(self):
 463         if self.seed_version == 4:
 464             return self.storage.get("master_public_key")
 465         else:
 466             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 467             return repr((c, K))
 468
 469     def get_master_private_key(self, account, password):
 470         master_k = pw_decode( self.master_private_keys[account], password)
 471         master_c, master_K, master_Kc = self.master_public_keys[account]
 472         try:
 473             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 474             assert K.encode('hex') == master_K
 475         except:
 476             raise BaseException("Invalid password")
 477         return master_k
 478
 479
 480     def get_address_index(self, address):
 481         if address in self.imported_keys.keys():
 482             return -1, None
 483
 484         for account in self.accounts.keys():
 485             for for_change in [0,1]:
 486                 addresses = self.accounts[account].get_addresses(for_change)
 487                 for addr in addresses:
 488                     if address == addr:
 489                         return account, (for_change, addresses.index(addr))
 490
 491         raise BaseException("Address not found", address)
 492
 493
 494     def get_roots(self, account):
 495         roots = []
 496         for a in account.split('&'):
 497             s = a.strip()
 498             m = re.match("(m/\d+'/)(\d+)", s)
 499             roots.append( m.group(1) )
 500         return roots
 501
 502     def is_seeded(self, account):
 503         for root in self.get_roots(account):
 504             if root not in self.master_private_keys.keys():
 505                 return False
 506         return True
 507
 508     def rebase_sequence(self, account, sequence):
 509         c, i = sequence
 510         dd = []
 511         for a in account.split('&'):
 512             s = a.strip()
 513             m = re.match("(m/\d+'/)(\d+)", s)
 514             root = m.group(1)
 515             num = int(m.group(2))
 516             dd.append( (root, [num,c,i] ) )
 517         return dd
 518
 519
 520     def get_keyID(self, account, sequence):
 521         if account == 0:
 522             return 'old'
 523
 524         rs = self.rebase_sequence(account, sequence)
 525         dd = []
 526         for root, public_sequence in rs:
 527             c, K, _ = self.master_public_keys[root]
 528             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 529             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 530         return '&'.join(dd)
 531
 532
 533     def get_public_key(self, address):
 534         account, sequence = self.get_address_index(address)
 535         return self.accounts[account].get_pubkey( *sequence )
 536
 537
 538     def decode_seed(self, password):
 539         seed = pw_decode(self.seed, password)
 540         #todo:  #self.sequences[0].check_seed(seed)
 541         return seed
 542
 543
 544     def get_private_key(self, address, password):
 545         out = []
 546         if address in self.imported_keys.keys():
 547             out.append( pw_decode( self.imported_keys[address], password ) )
 548         else:
 549             account, sequence = self.get_address_index(address)
 550             if account == 0:
 551                 seed = self.decode_seed(password)
 552                 pk = self.accounts[account].get_private_key(seed, sequence)
 553                 out.append(pk)
 554                 return out
 555
 556             # assert address == self.accounts[account].get_address(*sequence)
 557             rs = self.rebase_sequence( account, sequence)
 558             for root, public_sequence in rs:
 559
 560                 if root not in self.master_private_keys.keys(): continue
 561                 master_k = self.get_master_private_key(root, password)
 562                 master_c, _, _ = self.master_public_keys[root]
 563                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 564                 out.append(pk)
 565
 566         return out
 567
 568
 569
 570
 571     def signrawtransaction(self, tx, input_info, private_keys, password):
 572
 573         unspent_coins = self.get_unspent_coins()
 574         seed = self.decode_seed(password)
 575
 576         # build a list of public/private keys
 577         keypairs = {}
 578         for sec in private_keys:
 579             pubkey = public_key_from_private_key(sec)
 580             keypairs[ pubkey ] = sec
 581
 582
 583         for txin in tx.inputs:
 584             # convert to own format
 585             txin['tx_hash'] = txin['prevout_hash']
 586             txin['index'] = txin['prevout_n']
 587
 588             for item in input_info:
 589                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 590                     txin['raw_output_script'] = item['scriptPubKey']
 591                     txin['redeemScript'] = item.get('redeemScript')
 592                     txin['KeyID'] = item.get('KeyID')
 593                     break
 594             else:
 595                 for item in unspent_coins:
 596                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 597                         print_error( "tx input is in unspent coins" )
 598                         txin['raw_output_script'] = item['raw_output_script']
 599                         account, sequence = self.get_address_index(item['address'])
 600                         if account != -1:
 601                             txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 602                         break
 603                 else:
 604                     raise BaseException("Unknown transaction input. Please provide the 'input_info' parameter, or synchronize this wallet")
 605
 606             # if available, derive private_keys from KeyID
 607             keyid = txin.get('KeyID')
 608             if keyid:
 609                 roots = []
 610                 for s in keyid.split('&'):
 611                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 612                     if not m: continue
 613                     c = m.group(1)
 614                     K = m.group(2)
 615                     sequence = m.group(3)
 616                     root = self.find_root_by_master_key(c,K)
 617                     if not root: continue
 618                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 619                     root = root + '%d'%sequence[0]
 620                     sequence = sequence[1:]
 621                     roots.append((root,sequence))
 622
 623                 account_id = " & ".join( map(lambda x:x[0], roots) )
 624                 account = self.accounts.get(account_id)
 625                 if not account: continue
 626                 addr = account.get_address(*sequence)
 627                 txin['address'] = addr
 628                 pk = self.get_private_key(addr, password)
 629                 for sec in pk:
 630                     pubkey = public_key_from_private_key(sec)
 631                     keypairs[pubkey] = sec
 632
 633             redeem_script = txin.get("redeemScript")
 634             print_error( "p2sh:", "yes" if redeem_script else "no")
 635             if redeem_script:
 636                 addr = hash_160_to_bc_address(hash_160(redeem_script.decode('hex')), 5)
 637             else:
 638                 import transaction
 639                 _, addr = transaction.get_address_from_output_script(txin["raw_output_script"].decode('hex'))
 640             txin['address'] = addr
 641
 642             # add private keys that are in the wallet
 643             pk = self.get_private_key(addr, password)
 644             for sec in pk:
 645                 pubkey = public_key_from_private_key(sec)
 646                 keypairs[pubkey] = sec
 647                 if not redeem_script:
 648                     txin['redeemPubkey'] = pubkey
 649
 650             print txin
 651
 652         tx.sign( keypairs )
 653
 654     def sign_message(self, address, message, password):
 655         keys = self.get_private_key(address, password)
 656         assert len(keys) == 1
 657         sec = keys[0]
 658         key = regenerate_key(sec)
 659         compressed = is_compressed(sec)
 660         return key.sign_message(message, compressed, address)
 661
 662     def verify_message(self, address, signature, message):
 663         try:
 664             EC_KEY.verify_message(address, signature, message)
 665             return True
 666         except BaseException as e:
 667             print_error("Verification error: {0}".format(e))
 668             return False
 669
 670
 671     def change_gap_limit(self, value):
 672         if value >= self.gap_limit:
 673             self.gap_limit = value
 674             self.storage.put('gap_limit', self.gap_limit, True)
 675             #self.interface.poke('synchronizer')
 676             return True
 677
 678         elif value >= self.min_acceptable_gap():
 679             for key, account in self.accounts.items():
 680                 addresses = account[0]
 681                 k = self.num_unused_trailing_addresses(addresses)
 682                 n = len(addresses) - k + value
 683                 addresses = addresses[0:n]
 684                 self.accounts[key][0] = addresses
 685
 686             self.gap_limit = value
 687             self.storage.put('gap_limit', self.gap_limit, True)
 688             self.save_accounts()
 689             return True
 690         else:
 691             return False
 692
 693     def num_unused_trailing_addresses(self, addresses):
 694         k = 0
 695         for a in addresses[::-1]:
 696             if self.history.get(a):break
 697             k = k + 1
 698         return k
 699
 700     def min_acceptable_gap(self):
 701         # fixme: this assumes wallet is synchronized
 702         n = 0
 703         nmax = 0
 704
 705         for account in self.accounts.values():
 706             addresses = account.get_addresses(0)
 707             k = self.num_unused_trailing_addresses(addresses)
 708             for a in addresses[0:-k]:
 709                 if self.history.get(a):
 710                     n = 0
 711                 else:
 712                     n += 1
 713                     if n > nmax: nmax = n
 714         return nmax + 1
 715
 716
 717     def address_is_old(self, address):
 718         age = -1
 719         h = self.history.get(address, [])
 720         if h == ['*']:
 721             return True
 722         for tx_hash, tx_height in h:
 723             if tx_height == 0:
 724                 tx_age = 0
 725             else:
 726                 tx_age = self.verifier.blockchain.height - tx_height + 1
 727             if tx_age > age:
 728                 age = tx_age
 729         return age > 2
 730
 731
 732     def synchronize_sequence(self, account, for_change):
 733         limit = self.gap_limit_for_change if for_change else self.gap_limit
 734         new_addresses = []
 735         while True:
 736             addresses = account.get_addresses(for_change)
 737             if len(addresses) < limit:
 738                 address = account.create_new_address(for_change)
 739                 self.history[address] = []
 740                 new_addresses.append( address )
 741                 continue
 742
 743             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 744                 break
 745             else:
 746                 address = account.create_new_address(for_change)
 747                 self.history[address] = []
 748                 new_addresses.append( address )
 749
 750         return new_addresses
 751
 752
 753
 754     def create_pending_accounts(self):
 755         for account_type in ['1','2of2','2of3']:
 756             a = self.new_account_address(account_type)
 757             if self.address_is_old(a):
 758                 print_error( "creating account", a )
 759                 self.create_account(account_type)
 760
 761
 762     def synchronize_account(self, account):
 763         new = []
 764         new += self.synchronize_sequence(account, 0)
 765         new += self.synchronize_sequence(account, 1)
 766         return new
 767
 768
 769     def synchronize(self):
 770         if self.master_public_keys:
 771             self.create_pending_accounts()
 772         new = []
 773         for account in self.accounts.values():
 774             new += self.synchronize_account(account)
 775         if new:
 776             self.save_accounts()
 777             self.storage.put('addr_history', self.history, True)
 778         return new
 779
 780
 781     def is_found(self):
 782         return self.history.values() != [[]] * len(self.history)
 783
 784
 785     def add_contact(self, address, label=None):
 786         self.addressbook.append(address)
 787         self.storage.put('contacts', self.addressbook, True)
 788         if label:
 789             self.set_label(address, label)
 790
 791
 792     def delete_contact(self, addr):
 793         if addr in self.addressbook:
 794             self.addressbook.remove(addr)
 795             self.storage.put('addressbook', self.addressbook, True)
 796
 797
 798     def fill_addressbook(self):
 799         for tx_hash, tx in self.transactions.items():
 800             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 801             if is_send:
 802                 for addr, v in tx.outputs:
 803                     if not self.is_mine(addr) and addr not in self.addressbook:
 804                         self.addressbook.append(addr)
 805         # redo labels
 806         # self.update_tx_labels()
 807
 808     def get_num_tx(self, address):
 809         n = 0
 810         for tx in self.transactions.values():
 811             if address in map(lambda x:x[0], tx.outputs): n += 1
 812         return n
 813
 814
 815     def get_address_flags(self, addr):
 816         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 817         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 818         return flags
 819
 820
 821     def get_tx_value(self, tx, account=None):
 822         domain = self.get_account_addresses(account)
 823         return tx.get_value(domain, self.prevout_values)
 824
 825
 826     def update_tx_outputs(self, tx_hash):
 827         tx = self.transactions.get(tx_hash)
 828
 829         for i, (addr, value) in enumerate(tx.outputs):
 830             key = tx_hash+ ':%d'%i
 831             self.prevout_values[key] = value
 832
 833         for item in tx.inputs:
 834             if self.is_mine(item.get('address')):
 835                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 836                 self.spent_outputs.append(key)
 837
 838
 839     def get_addr_balance(self, address):
 840         assert self.is_mine(address)
 841         h = self.history.get(address,[])
 842         if h == ['*']: return 0,0
 843         c = u = 0
 844         received_coins = []   # list of coins received at address
 845
 846         for tx_hash, tx_height in h:
 847             tx = self.transactions.get(tx_hash)
 848             if not tx: continue
 849
 850             for i, (addr, value) in enumerate(tx.outputs):
 851                 if addr == address:
 852                     key = tx_hash + ':%d'%i
 853                     received_coins.append(key)
 854
 855         for tx_hash, tx_height in h:
 856             tx = self.transactions.get(tx_hash)
 857             if not tx: continue
 858             v = 0
 859
 860             for item in tx.inputs:
 861                 addr = item.get('address')
 862                 if addr == address:
 863                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 864                     value = self.prevout_values.get( key )
 865                     if key in received_coins:
 866                         v -= value
 867
 868             for i, (addr, value) in enumerate(tx.outputs):
 869                 key = tx_hash + ':%d'%i
 870                 if addr == address:
 871                     v += value
 872
 873             if tx_height:
 874                 c += v
 875             else:
 876                 u += v
 877         return c, u
 878
 879
 880     def get_account_name(self, k):
 881         if k == 0:
 882             if self.seed_version == 4:
 883                 name = 'Main account'
 884             else:
 885                 name = 'Old account'
 886         else:
 887             name = self.labels.get(k, 'Unnamed account')
 888         return name
 889
 890     def get_account_names(self):
 891         accounts = {}
 892         for k, account in self.accounts.items():
 893             accounts[k] = self.get_account_name(k)
 894         if self.imported_keys:
 895             accounts[-1] = 'Imported keys'
 896         return accounts
 897
 898     def get_account_addresses(self, a, include_change=True):
 899         if a is None:
 900             o = self.addresses(True)
 901         elif a == -1:
 902             o = self.imported_keys.keys()
 903         else:
 904             ac = self.accounts[a]
 905             o = ac.get_addresses(0)
 906             if include_change: o += ac.get_addresses(1)
 907         return o
 908
 909     def get_imported_balance(self):
 910         cc = uu = 0
 911         for addr in self.imported_keys.keys():
 912             c, u = self.get_addr_balance(addr)
 913             cc += c
 914             uu += u
 915         return cc, uu
 916
 917     def get_account_balance(self, account):
 918         if account is None:
 919             return self.get_balance()
 920         elif account == -1:
 921             return self.get_imported_balance()
 922
 923         conf = unconf = 0
 924         for addr in self.get_account_addresses(account):
 925             c, u = self.get_addr_balance(addr)
 926             conf += c
 927             unconf += u
 928         return conf, unconf
 929
 930     def get_frozen_balance(self):
 931         conf = unconf = 0
 932         for addr in self.frozen_addresses:
 933             c, u = self.get_addr_balance(addr)
 934             conf += c
 935             unconf += u
 936         return conf, unconf
 937
 938
 939     def get_balance(self):
 940         cc = uu = 0
 941         for a in self.accounts.keys():
 942             c, u = self.get_account_balance(a)
 943             cc += c
 944             uu += u
 945         c, u = self.get_imported_balance()
 946         cc += c
 947         uu += u
 948         return cc, uu
 949
 950
 951     def get_unspent_coins(self, domain=None):
 952         coins = []
 953         if domain is None: domain = self.addresses(True)
 954         for addr in domain:
 955             h = self.history.get(addr, [])
 956             if h == ['*']: continue
 957             for tx_hash, tx_height in h:
 958                 tx = self.transactions.get(tx_hash)
 959                 if tx is None: raise BaseException("Wallet not synchronized")
 960                 for output in tx.d.get('outputs'):
 961                     if output.get('address') != addr: continue
 962                     key = tx_hash + ":%d" % output.get('index')
 963                     if key in self.spent_outputs: continue
 964                     output['tx_hash'] = tx_hash
 965                     output['height'] = tx_height
 966                     coins.append((tx_height, output))
 967
 968         # sort by age
 969         if coins:
 970             coins = sorted(coins)
 971             if coins[-1][0] != 0:
 972                 while coins[0][0] == 0:
 973                     coins = coins[1:] + [ coins[0] ]
 974         return [x[1] for x in coins]
 975
 976
 977
 978     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
 979         """ todo: minimize tx size """
 980         total = 0
 981         fee = self.fee if fixed_fee is None else fixed_fee
 982         if domain is None:
 983             domain = self.addresses()
 984
 985         for i in self.frozen_addresses:
 986             if i in domain: domain.remove(i)
 987
 988         prioritized = []
 989         for i in self.prioritized_addresses:
 990             if i in domain:
 991                 domain.remove(i)
 992                 prioritized.append(i)
 993
 994         coins = self.get_unspent_coins(domain)
 995         prioritized_coins = self.get_unspent_coins(prioritized)
 996
 997         inputs = []
 998         coins = prioritized_coins + coins
 999
1000         for item in coins:
1001             addr = item.get('address')
1002             v = item.get('value')
1003             total += v
1004             inputs.append( item )
1005             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1006             if total >= amount + fee: break
1007         else:
1008             inputs = []
1009
1010         return inputs, total, fee
1011
1012
1013     def set_fee(self, fee):
1014         if self.fee != fee:
1015             self.fee = fee
1016             self.storage.put('fee_per_kb', self.fee, True)
1017
1018     def estimated_fee(self, inputs):
1019         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1020         fee = self.fee * int(round(estimated_size/1024.))
1021         if fee == 0: fee = self.fee
1022         return fee
1023
1024
1025     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1026         "add change to a transaction"
1027         change_amount = total - ( amount + fee )
1028         if change_amount != 0:
1029             if not change_addr:
1030
1031                 # send change to one of the accounts involved in the tx
1032                 address = inputs[0].get('address')
1033                 account, _ = self.get_address_index(address)
1034
1035                 if not self.use_change or account == -1:
1036                     change_addr = inputs[-1]['address']
1037                 else:
1038                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1039
1040             # Insert the change output at a random position in the outputs
1041             posn = random.randint(0, len(outputs))
1042             outputs[posn:posn] = [( change_addr,  change_amount)]
1043         return outputs
1044
1045
1046     def get_history(self, address):
1047         with self.lock:
1048             return self.history.get(address)
1049
1050
1051     def get_status(self, h):
1052         if not h: return None
1053         if h == ['*']: return '*'
1054         status = ''
1055         for tx_hash, height in h:
1056             status += tx_hash + ':%d:' % height
1057         return hashlib.sha256( status ).digest().encode('hex')
1058
1059
1060     def receive_tx_callback(self, tx_hash, tx, tx_height):
1061
1062         with self.transaction_lock:
1063             self.add_extra_addresses(tx)
1064             if not self.check_new_tx(tx_hash, tx):
1065                 # may happen due to pruning
1066                 print_error("received transaction that is no longer referenced in history", tx_hash)
1067                 return
1068             self.transactions[tx_hash] = tx
1069             self.network.interface.pending_transactions_for_notifications.append(tx)
1070             self.save_transactions()
1071             if self.verifier and tx_height>0:
1072                 self.verifier.add(tx_hash, tx_height)
1073             self.update_tx_outputs(tx_hash)
1074
1075
1076     def save_transactions(self):
1077         tx = {}
1078         for k,v in self.transactions.items():
1079             tx[k] = str(v)
1080         self.storage.put('transactions', tx, True)
1081
1082     def receive_history_callback(self, addr, hist):
1083
1084         if not self.check_new_history(addr, hist):
1085             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1086
1087         with self.lock:
1088             self.history[addr] = hist
1089             self.storage.put('addr_history', self.history, True)
1090
1091         if hist != ['*']:
1092             for tx_hash, tx_height in hist:
1093                 if tx_height>0:
1094                     # add it in case it was previously unconfirmed
1095                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1096
1097
1098     def get_tx_history(self, account=None):
1099         with self.transaction_lock:
1100             history = self.transactions.items()
1101             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1102             result = []
1103
1104             balance = 0
1105             for tx_hash, tx in history:
1106                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1107                 if v is not None: balance += v
1108
1109             c, u = self.get_account_balance(account)
1110
1111             if balance != c+u:
1112                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1113
1114             balance = c + u - balance
1115             for tx_hash, tx in history:
1116                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1117                 if not is_relevant:
1118                     continue
1119                 if value is not None:
1120                     balance += value
1121
1122                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1123                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1124
1125         return result
1126
1127
1128     def get_label(self, tx_hash):
1129         label = self.labels.get(tx_hash)
1130         is_default = (label == '') or (label is None)
1131         if is_default: label = self.get_default_label(tx_hash)
1132         return label, is_default
1133
1134
1135     def get_default_label(self, tx_hash):
1136         tx = self.transactions.get(tx_hash)
1137         default_label = ''
1138         if tx:
1139             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1140             if is_mine:
1141                 for o in tx.outputs:
1142                     o_addr, _ = o
1143                     if not self.is_mine(o_addr):
1144                         try:
1145                             default_label = self.labels[o_addr]
1146                         except KeyError:
1147                             default_label = o_addr
1148                         break
1149                 else:
1150                     default_label = '(internal)'
1151             else:
1152                 for o in tx.outputs:
1153                     o_addr, _ = o
1154                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1155                         break
1156                 else:
1157                     for o in tx.outputs:
1158                         o_addr, _ = o
1159                         if self.is_mine(o_addr):
1160                             break
1161                     else:
1162                         o_addr = None
1163
1164                 if o_addr:
1165                     dest_label = self.labels.get(o_addr)
1166                     try:
1167                         default_label = self.labels[o_addr]
1168                     except KeyError:
1169                         default_label = o_addr
1170
1171         return default_label
1172
1173
1174     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1175         for address, x in outputs:
1176             assert is_valid(address)
1177         amount = sum( map(lambda x:x[1], outputs) )
1178         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1179         if not inputs:
1180             raise ValueError("Not enough funds")
1181         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1182         return Transaction.from_io(inputs, outputs)
1183
1184
1185     def mktx_from_account(self, outputs, password, fee=None, change_addr=None, account=None):
1186         domain = self.get_account_addresses(account) if account else None
1187         return self.mktx(outputs, password, fee, change_addr, domain)
1188
1189
1190     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1191         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1192         self.sign_transaction(tx, password)
1193         return tx
1194
1195
1196     def sign_transaction(self, tx, password):
1197         keypairs = {}
1198         for i, txin in enumerate(tx.inputs):
1199             address = txin['address']
1200             account, sequence = self.get_address_index(address)
1201             txin['KeyID'] = self.get_keyID(account, sequence)
1202             redeemScript = self.accounts[account].redeem_script(sequence)
1203             if redeemScript:
1204                 txin['redeemScript'] = redeemScript
1205             else:
1206                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1207             private_keys = self.get_private_key(address, password)
1208             for sec in private_keys:
1209                 pubkey = public_key_from_private_key(sec)
1210                 keypairs[ pubkey ] = sec
1211         tx.sign(keypairs)
1212
1213
1214     def sendtx(self, tx):
1215         # synchronous
1216         h = self.send_tx(tx)
1217         self.tx_event.wait()
1218         return self.receive_tx(h)
1219
1220     def send_tx(self, tx):
1221         # asynchronous
1222         self.tx_event.clear()
1223         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1224         return tx.hash()
1225
1226     def on_broadcast(self, i, r):
1227         self.tx_result = r.get('result')
1228         self.tx_event.set()
1229
1230     def receive_tx(self,tx_hash):
1231         out = self.tx_result
1232         if out != tx_hash:
1233             return False, "error: " + out
1234         return True, out
1235
1236
1237
1238     def update_password(self, seed, old_password, new_password):
1239         if new_password == '': new_password = None
1240         # this will throw an exception if unicode cannot be converted
1241         self.seed = pw_encode( seed, new_password)
1242         self.storage.put('seed', self.seed, True)
1243         self.use_encryption = (new_password != None)
1244         self.storage.put('use_encryption', self.use_encryption,True)
1245         for k in self.imported_keys.keys():
1246             a = self.imported_keys[k]
1247             b = pw_decode(a, old_password)
1248             c = pw_encode(b, new_password)
1249             self.imported_keys[k] = c
1250         self.storage.put('imported_keys', self.imported_keys, True)
1251
1252         for k, v in self.master_private_keys.items():
1253             b = pw_decode(v, old_password)
1254             c = pw_encode(b, new_password)
1255             self.master_private_keys[k] = c
1256         self.storage.put('master_private_keys', self.master_private_keys, True)
1257
1258
1259     def freeze(self,addr):
1260         if self.is_mine(addr) and addr not in self.frozen_addresses:
1261             self.unprioritize(addr)
1262             self.frozen_addresses.append(addr)
1263             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1264             return True
1265         else:
1266             return False
1267
1268     def unfreeze(self,addr):
1269         if self.is_mine(addr) and addr in self.frozen_addresses:
1270             self.frozen_addresses.remove(addr)
1271             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1272             return True
1273         else:
1274             return False
1275
1276     def prioritize(self,addr):
1277         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1278             self.unfreeze(addr)
1279             self.prioritized_addresses.append(addr)
1280             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1281             return True
1282         else:
1283             return False
1284
1285     def unprioritize(self,addr):
1286         if self.is_mine(addr) and addr in self.prioritized_addresses:
1287             self.prioritized_addresses.remove(addr)
1288             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1289             return True
1290         else:
1291             return False
1292
1293
1294     def set_verifier(self, verifier):
1295         self.verifier = verifier
1296
1297         # review transactions that are in the history
1298         for addr, hist in self.history.items():
1299             if hist == ['*']: continue
1300             for tx_hash, tx_height in hist:
1301                 if tx_height>0:
1302                     # add it in case it was previously unconfirmed
1303                     self.verifier.add(tx_hash, tx_height)
1304
1305
1306         # if we are on a pruning server, remove unverified transactions
1307         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1308         for tx_hash in self.transactions.keys():
1309             if tx_hash not in vr:
1310                 self.transactions.pop(tx_hash)
1311
1312
1313
1314     def check_new_history(self, addr, hist):
1315
1316         # check that all tx in hist are relevant
1317         if hist != ['*']:
1318             for tx_hash, height in hist:
1319                 tx = self.transactions.get(tx_hash)
1320                 if not tx: continue
1321                 if not tx.has_address(addr):
1322                     return False
1323
1324         # check that we are not "orphaning" a transaction
1325         old_hist = self.history.get(addr,[])
1326         if old_hist == ['*']: return True
1327
1328         for tx_hash, height in old_hist:
1329             if tx_hash in map(lambda x:x[0], hist): continue
1330             found = False
1331             for _addr, _hist in self.history.items():
1332                 if _addr == addr: continue
1333                 if _hist == ['*']: continue
1334                 _tx_hist = map(lambda x:x[0], _hist)
1335                 if tx_hash in _tx_hist:
1336                     found = True
1337                     break
1338
1339             if not found:
1340                 tx = self.transactions.get(tx_hash)
1341                 # tx might not be there
1342                 if not tx: continue
1343
1344                 # already verified?
1345                 if self.verifier.get_height(tx_hash):
1346                     continue
1347                 # unconfirmed tx
1348                 print_error("new history is orphaning transaction:", tx_hash)
1349                 # check that all outputs are not mine, request histories
1350                 ext_requests = []
1351                 for _addr, _v in tx.outputs:
1352                     # assert not self.is_mine(_addr)
1353                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1354
1355                 ext_h = self.network.interface.synchronous_get(ext_requests)
1356                 print_error("sync:", ext_requests, ext_h)
1357                 height = None
1358                 for h in ext_h:
1359                     if h == ['*']: continue
1360                     for item in h:
1361                         if item.get('tx_hash') == tx_hash:
1362                             height = item.get('height')
1363                 if height:
1364                     print_error("found height for", tx_hash, height)
1365                     self.verifier.add(tx_hash, height)
1366                 else:
1367                     print_error("removing orphaned tx from history", tx_hash)
1368                     self.transactions.pop(tx_hash)
1369
1370         return True
1371
1372
1373
1374     def check_new_tx(self, tx_hash, tx):
1375         # 1 check that tx is referenced in addr_history.
1376         addresses = []
1377         for addr, hist in self.history.items():
1378             if hist == ['*']:continue
1379             for txh, height in hist:
1380                 if txh == tx_hash:
1381                     addresses.append(addr)
1382
1383         if not addresses:
1384             return False
1385
1386         # 2 check that referencing addresses are in the tx
1387         for addr in addresses:
1388             if not tx.has_address(addr):
1389                 return False
1390
1391         return True
1392
1393
1394     def start_threads(self, network):
1395         from verifier import TxVerifier
1396         self.network = network
1397         self.verifier = TxVerifier(self.network, self.storage)
1398         self.verifier.start()
1399         self.set_verifier(self.verifier)
1400         self.synchronizer = WalletSynchronizer(self)
1401         self.synchronizer.start()
1402
1403     def stop_threads(self):
1404         self.verifier.stop()
1405         self.synchronizer.stop()
1406
1407
1408
1409
1410
1411 class WalletSynchronizer(threading.Thread):
1412
1413
1414     def __init__(self, wallet):
1415         threading.Thread.__init__(self)
1416         self.daemon = True
1417         self.wallet = wallet
1418         wallet.synchronizer = self
1419         self.network = self.wallet.network
1420         #self.wallet.network.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1421         self.was_updated = True
1422         self.running = False
1423         self.lock = threading.Lock()
1424         self.queue = Queue.Queue()
1425
1426     def stop(self):
1427         with self.lock: self.running = False
1428
1429     def is_running(self):
1430         with self.lock: return self.running
1431
1432
1433     def subscribe_to_addresses(self, addresses):
1434         messages = []
1435         for addr in addresses:
1436             messages.append(('blockchain.address.subscribe', [addr]))
1437         self.network.interface.send( messages, lambda i,r: self.queue.put(r))
1438
1439
1440     def run(self):
1441         with self.lock:
1442             self.running = True
1443
1444         while self.is_running():
1445             interface = self.network.interface
1446             if not interface.is_connected:
1447                 print_error("synchronizer: waiting for interface")
1448                 interface.connect_event.wait()
1449
1450             self.run_interface(interface)
1451
1452
1453     def run_interface(self, interface):
1454
1455         print_error("synchronizer: connected to", interface.server)
1456
1457         requested_tx = []
1458         missing_tx = []
1459         requested_histories = {}
1460
1461         # request any missing transactions
1462         for history in self.wallet.history.values():
1463             if history == ['*']: continue
1464             for tx_hash, tx_height in history:
1465                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1466                     missing_tx.append( (tx_hash, tx_height) )
1467
1468         if missing_tx:
1469             print_error("missing tx", missing_tx)
1470
1471         # subscriptions
1472         self.subscribe_to_addresses(self.wallet.addresses(True, next=True))
1473
1474         while self.is_running():
1475             # 1. create new addresses
1476             new_addresses = self.wallet.synchronize()
1477
1478             # request missing addresses
1479             if new_addresses:
1480                 self.subscribe_to_addresses(new_addresses)
1481
1482             # request missing transactions
1483             for tx_hash, tx_height in missing_tx:
1484                 if (tx_hash, tx_height) not in requested_tx:
1485                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1486                     requested_tx.append( (tx_hash, tx_height) )
1487             missing_tx = []
1488
1489             # detect if situation has changed
1490             if interface.is_up_to_date() and self.queue.empty():
1491                 if not self.wallet.is_up_to_date():
1492                     self.wallet.set_up_to_date(True)
1493                     self.was_updated = True
1494             else:
1495                 if self.wallet.is_up_to_date():
1496                     self.wallet.set_up_to_date(False)
1497                     self.was_updated = True
1498
1499             if self.was_updated:
1500                 self.wallet.network.trigger_callback('updated')
1501                 self.was_updated = False
1502
1503             # 2. get a response
1504             try:
1505                 r = self.queue.get(block=True, timeout=1)
1506             except Queue.Empty:
1507                 continue
1508
1509             if interface != self.network.interface:
1510                 break
1511
1512             if not r:
1513                 continue
1514
1515             # 3. handle response
1516             method = r['method']
1517             params = r['params']
1518             result = r.get('result')
1519             error = r.get('error')
1520             if error:
1521                 print "error", r
1522                 continue
1523
1524             if method == 'blockchain.address.subscribe':
1525                 addr = params[0]
1526                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1527                     if requested_histories.get(addr) is None:
1528                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1529                         requested_histories[addr] = result
1530
1531             elif method == 'blockchain.address.get_history':
1532                 addr = params[0]
1533                 print_error("receiving history", addr, result)
1534                 if result == ['*']:
1535                     assert requested_histories.pop(addr) == '*'
1536                     self.wallet.receive_history_callback(addr, result)
1537                 else:
1538                     hist = []
1539                     # check that txids are unique
1540                     txids = []
1541                     for item in result:
1542                         tx_hash = item['tx_hash']
1543                         if tx_hash not in txids:
1544                             txids.append(tx_hash)
1545                             hist.append( (tx_hash, item['height']) )
1546
1547                     if len(hist) != len(result):
1548                         raise BaseException("error: server sent history with non-unique txid", result)
1549
1550                     # check that the status corresponds to what was announced
1551                     rs = requested_histories.pop(addr)
1552                     if self.wallet.get_status(hist) != rs:
1553                         raise BaseException("error: status mismatch: %s"%addr)
1554
1555                     # store received history
1556                     self.wallet.receive_history_callback(addr, hist)
1557
1558                     # request transactions that we don't have
1559                     for tx_hash, tx_height in hist:
1560                         if self.wallet.transactions.get(tx_hash) is None:
1561                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1562                                 missing_tx.append( (tx_hash, tx_height) )
1563
1564             elif method == 'blockchain.transaction.get':
1565                 tx_hash = params[0]
1566                 tx_height = params[1]
1567                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1568                 tx = Transaction(result)
1569                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1570                 self.was_updated = True
1571                 requested_tx.remove( (tx_hash, tx_height) )
1572                 print_error("received tx:", tx_hash, len(tx.raw))
1573
1574             else:
1575                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1576
1577             if self.was_updated and not requested_tx:
1578                 self.wallet.network.trigger_callback('updated')
1579                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1580
1581                 self.was_updated = False