lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37
  38 # AES encryption
  39 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  40 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  41
  42 def pw_encode(s, password):
  43     if password:
  44         secret = Hash(password)
  45         return EncodeAES(secret, s)
  46     else:
  47         return s
  48
  49 def pw_decode(s, password):
  50     if password is not None:
  51         secret = Hash(password)
  52         try:
  53             d = DecodeAES(secret, s)
  54         except:
  55             raise BaseException('Invalid password')
  56         return d
  57     else:
  58         return s
  59
  60
  61
  62
  63
  64 from version import ELECTRUM_VERSION, SEED_VERSION
  65
  66
  67 class WalletStorage:
  68
  69     def __init__(self, config):
  70         self.data = {}
  71         self.file_exists = False
  72         self.init_path(config)
  73         print_error( "wallet path", self.path )
  74         if self.path:
  75             self.read(self.path)
  76
  77
  78     def init_path(self, config):
  79         """Set the path of the wallet."""
  80
  81         path = config.get('wallet_path')
  82         if not path:
  83             path = config.get('default_wallet_path')
  84         if path is not None:
  85             self.path = path
  86             return
  87
  88         self.path = os.path.join(config.path, "electrum.dat")
  89
  90
  91     def read(self, path):
  92         """Read the contents of the wallet file."""
  93         try:
  94             with open(self.path, "r") as f:
  95                 data = f.read()
  96         except IOError:
  97             return
  98         try:
  99             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 100         except:
 101             raise IOError("Cannot read wallet file.")
 102
 103         self.data = d
 104         self.file_exists = True
 105
 106
 107     def get(self, key, default=None):
 108         return self.data.get(key, default)
 109
 110     def put(self, key, value, save = True):
 111
 112         if self.data.get(key) is not None:
 113             self.data[key] = value
 114         else:
 115             # add key to wallet config
 116             self.data[key] = value
 117
 118         if save:
 119             self.write()
 120
 121
 122     def write(self):
 123         s = repr(self.data)
 124         f = open(self.path,"w")
 125         f.write( s )
 126         f.close()
 127         if self.get('gui') != 'android':
 128             import stat
 129             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 130
 131
 132 class Wallet:
 133
 134     def __init__(self, storage):
 135
 136         self.storage = storage
 137         self.electrum_version = ELECTRUM_VERSION
 138         self.gap_limit_for_change = 3 # constant
 139
 140         # saved fields
 141         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 142
 143         self.gap_limit             = storage.get('gap_limit', 5)
 144         self.use_change            = storage.get('use_change',True)
 145         self.use_encryption        = storage.get('use_encryption', False)
 146         self.seed                  = storage.get('seed', '')               # encrypted
 147         self.labels                = storage.get('labels', {})
 148         self.frozen_addresses      = storage.get('frozen_addresses',[])
 149         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 150         self.addressbook           = storage.get('contacts', [])
 151
 152         self.imported_keys         = storage.get('imported_keys',{})
 153         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 154
 155         self.fee                   = int(storage.get('fee_per_kb',20000))
 156
 157         self.master_public_keys = storage.get('master_public_keys',{})
 158         self.master_private_keys = storage.get('master_private_keys', {})
 159
 160         self.next_addresses = storage.get('next_addresses',{})
 161
 162         if self.seed_version < 4:
 163             raise ValueError("This wallet seed is deprecated.")
 164
 165         self.load_accounts()
 166
 167         self.transactions = {}
 168         tx_list = self.storage.get('transactions',{})
 169         for k,v in tx_list.items():
 170             try:
 171                 tx = Transaction(v)
 172             except:
 173                 print_msg("Warning: Cannot deserialize transactions. skipping")
 174                 continue
 175
 176             self.add_extra_addresses(tx)
 177             self.transactions[k] = tx
 178
 179         for h,tx in self.transactions.items():
 180             if not self.check_new_tx(h, tx):
 181                 print_error("removing unreferenced tx", h)
 182                 self.transactions.pop(h)
 183
 184
 185         # not saved
 186         self.prevout_values = {}     # my own transaction outputs
 187         self.spent_outputs = []
 188
 189         # spv
 190         self.verifier = None
 191
 192         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 193         # interface.is_up_to_date() returns true when all requests have been answered and processed
 194         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 195
 196         self.up_to_date = False
 197         self.lock = threading.Lock()
 198         self.transaction_lock = threading.Lock()
 199         self.tx_event = threading.Event()
 200
 201         for tx_hash, tx in self.transactions.items():
 202             self.update_tx_outputs(tx_hash)
 203
 204
 205     def add_extra_addresses(self, tx):
 206         h = tx.hash()
 207         # find the address corresponding to pay-to-pubkey inputs
 208         tx.add_extra_addresses(self.transactions)
 209         for o in tx.d.get('outputs'):
 210             if o.get('is_pubkey'):
 211                 for tx2 in self.transactions.values():
 212                     tx2.add_extra_addresses({h:tx})
 213
 214
 215
 216
 217     def set_up_to_date(self,b):
 218         with self.lock: self.up_to_date = b
 219
 220
 221     def is_up_to_date(self):
 222         with self.lock: return self.up_to_date
 223
 224
 225     def update(self):
 226         self.up_to_date = False
 227         while not self.is_up_to_date():
 228             time.sleep(0.1)
 229
 230
 231     def import_key(self, sec, password):
 232         # check password
 233         seed = self.decode_seed(password)
 234         try:
 235             address = address_from_private_key(sec)
 236         except:
 237             raise BaseException('Invalid private key')
 238
 239         if self.is_mine(address):
 240             raise BaseException('Address already in wallet')
 241
 242         # store the originally requested keypair into the imported keys table
 243         self.imported_keys[address] = pw_encode(sec, password )
 244         self.storage.put('imported_keys', self.imported_keys, True)
 245         return address
 246
 247     def delete_imported_key(self, addr):
 248         if addr in self.imported_keys:
 249             self.imported_keys.pop(addr)
 250             self.storage.put('imported_keys', self.imported_keys, True)
 251
 252
 253     def init_seed(self, seed):
 254         if self.seed: raise BaseException("a seed exists")
 255         if not seed:
 256             seed = random_seed(128)
 257         self.seed = seed
 258
 259
 260     def save_seed(self):
 261         self.storage.put('seed', self.seed, True)
 262         self.storage.put('seed_version', self.seed_version, True)
 263
 264     def create_watching_only_wallet(self, c0, K0):
 265         cK0 = ""
 266         self.master_public_keys = {
 267             "m/0'/": (c0, K0, cK0),
 268             }
 269         self.storage.put('master_public_keys', self.master_public_keys, True)
 270         self.create_account('1','Main account')
 271
 272     def create_accounts(self):
 273
 274         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 275
 276         # normal accounts
 277         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 278         # p2sh 2of2
 279         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 280         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 281         # p2sh 2of3
 282         k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 283         k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 284         k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 285
 286         self.master_public_keys = {
 287             "m/0'/": (c0, K0, cK0),
 288             "m/1'/": (c1, K1, cK1),
 289             "m/2'/": (c2, K2, cK2),
 290             "m/3'/": (c3, K3, cK3),
 291             "m/4'/": (c4, K4, cK4),
 292             "m/5'/": (c5, K5, cK5)
 293             }
 294
 295         self.master_private_keys = {
 296             "m/0'/": k0,
 297             "m/1'/": k1,
 298             "m/2'/": k2,
 299             "m/3'/": k3,
 300             "m/4'/": k4,
 301             "m/5'/": k5
 302             }
 303
 304         self.storage.put('master_public_keys', self.master_public_keys, True)
 305         self.storage.put('master_private_keys', self.master_private_keys, True)
 306
 307         # create default account
 308         self.create_account('1','Main account')
 309
 310
 311     def find_root_by_master_key(self, c, K):
 312         for key, v in self.master_public_keys.items():
 313             if key == "m/":continue
 314             cc, KK, _ = v
 315             if (c == cc) and (K == KK):
 316                 return key
 317
 318     def deseed_root(self, seed, password):
 319         # for safety, we ask the user to enter their seed
 320         assert seed == self.decode_seed(password)
 321         self.seed = ''
 322         self.storage.put('seed', '', True)
 323
 324
 325     def deseed_branch(self, k):
 326         # check that parent has no seed
 327         assert self.seed == ''
 328         self.master_private_keys.pop(k)
 329         self.storage.put('master_private_keys', self.master_private_keys, True)
 330
 331
 332     def account_id(self, account_type, i):
 333         if account_type == '1':
 334             return "m/0'/%d"%i
 335         elif account_type == '2of2':
 336             return "m/1'/%d & m/2'/%d"%(i,i)
 337         elif account_type == '2of3':
 338             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 339         else:
 340             raise BaseException('unknown account type')
 341
 342
 343     def num_accounts(self, account_type):
 344         keys = self.accounts.keys()
 345         i = 0
 346         while True:
 347             account_id = self.account_id(account_type, i)
 348             if account_id not in keys: break
 349             i += 1
 350         return i
 351
 352
 353     def new_account_address(self, account_type = '1'):
 354         i = self.num_accounts(account_type)
 355         k = self.account_id(account_type,i)
 356
 357         addr = self.next_addresses.get(k)
 358         if not addr:
 359             account_id, account = self.next_account(account_type)
 360             addr = account.first_address()
 361             self.next_addresses[k] = addr
 362             self.storage.put('next_addresses',self.next_addresses)
 363
 364         return addr
 365
 366
 367     def next_account(self, account_type = '1'):
 368
 369         i = self.num_accounts(account_type)
 370         account_id = self.account_id(account_type,i)
 371
 372         if account_type is '1':
 373             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 374             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 375             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 376
 377         elif account_type == '2of2':
 378             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 379             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 380             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 381             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 382             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 383
 384         elif account_type == '2of3':
 385             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 386             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 387             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 388             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 389             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 390             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 391             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 392
 393         return account_id, account
 394
 395
 396     def set_label(self, key, value):
 397         self.labels[key] = value
 398         self.storage.put('labels', self.labels, True)
 399
 400
 401     def create_account(self, account_type = '1', name = None):
 402         account_id, account = self.next_account(account_type)
 403         self.accounts[account_id] = account
 404         self.save_accounts()
 405         if name:
 406             self.set_label(account_id, name)
 407
 408
 409     def create_old_account(self):
 410         mpk = OldAccount.mpk_from_seed(self.seed)
 411         self.storage.put('master_public_key', mpk, True)
 412         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 413         self.save_accounts()
 414
 415
 416     def save_accounts(self):
 417         d = {}
 418         for k, v in self.accounts.items():
 419             d[k] = v.dump()
 420         self.storage.put('accounts', d, True)
 421
 422
 423
 424     def load_accounts(self):
 425         d = self.storage.get('accounts', {})
 426         self.accounts = {}
 427         for k, v in d.items():
 428             if k == 0:
 429                 v['mpk'] = self.storage.get('master_public_key')
 430                 self.accounts[k] = OldAccount(v)
 431             elif '&' in k:
 432                 self.accounts[k] = BIP32_Account_2of2(v)
 433             else:
 434                 self.accounts[k] = BIP32_Account(v)
 435
 436
 437     def addresses(self, include_change = True, next=False):
 438         o = self.get_account_addresses(-1, include_change)
 439         for a in self.accounts.keys():
 440             o += self.get_account_addresses(a, include_change)
 441
 442         if next:
 443             for addr in self.next_addresses.values():
 444                 if addr not in o:
 445                     o += [addr]
 446         return o
 447
 448
 449     def is_mine(self, address):
 450         return address in self.addresses(True)
 451
 452
 453     def is_change(self, address):
 454         if not self.is_mine(address): return False
 455         if address in self.imported_keys.keys(): return False
 456         acct, s = self.get_address_index(address)
 457         if s is None: return False
 458         return s[0] == 1
 459
 460     def get_master_public_key(self):
 461         if self.seed_version == 4:
 462             return self.storage.get("master_public_key")
 463         else:
 464             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 465             return repr((c, K))
 466
 467     def get_master_private_key(self, account, password):
 468         master_k = pw_decode( self.master_private_keys[account], password)
 469         master_c, master_K, master_Kc = self.master_public_keys[account]
 470         try:
 471             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 472             assert K.encode('hex') == master_K
 473         except:
 474             raise BaseException("Invalid password")
 475         return master_k
 476
 477
 478     def get_address_index(self, address):
 479         if address in self.imported_keys.keys():
 480             return -1, None
 481
 482         for account in self.accounts.keys():
 483             for for_change in [0,1]:
 484                 addresses = self.accounts[account].get_addresses(for_change)
 485                 for addr in addresses:
 486                     if address == addr:
 487                         return account, (for_change, addresses.index(addr))
 488
 489         raise BaseException("Address not found", address)
 490
 491
 492     def rebase_sequence(self, account, sequence):
 493         c, i = sequence
 494         dd = []
 495         for a in account.split('&'):
 496             s = a.strip()
 497             m = re.match("(m/\d+'/)(\d+)", s)
 498             root = m.group(1)
 499             num = int(m.group(2))
 500             dd.append( (root, [num,c,i] ) )
 501         return dd
 502
 503
 504     def get_keyID(self, account, sequence):
 505         if account == 0:
 506             return 'old'
 507
 508         rs = self.rebase_sequence(account, sequence)
 509         dd = []
 510         for root, public_sequence in rs:
 511             c, K, _ = self.master_public_keys[root]
 512             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 513             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 514         return '&'.join(dd)
 515
 516
 517     def get_public_key(self, address):
 518         account, sequence = self.get_address_index(address)
 519         return self.accounts[account].get_pubkey( *sequence )
 520
 521
 522     def decode_seed(self, password):
 523         seed = pw_decode(self.seed, password)
 524         #todo:  #self.sequences[0].check_seed(seed)
 525         return seed
 526
 527
 528     def get_private_key(self, address, password):
 529         out = []
 530         if address in self.imported_keys.keys():
 531             out.append( pw_decode( self.imported_keys[address], password ) )
 532         else:
 533             account, sequence = self.get_address_index(address)
 534             if account == 0:
 535                 seed = self.decode_seed(password)
 536                 pk = self.accounts[account].get_private_key(seed, sequence)
 537                 out.append(pk)
 538                 return out
 539
 540             # assert address == self.accounts[account].get_address(*sequence)
 541             rs = self.rebase_sequence( account, sequence)
 542             for root, public_sequence in rs:
 543
 544                 if root not in self.master_private_keys.keys(): continue
 545                 master_k = self.get_master_private_key(root, password)
 546                 master_c, _, _ = self.master_public_keys[root]
 547                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 548                 out.append(pk)
 549
 550         return out
 551
 552
 553
 554
 555     def signrawtransaction(self, tx, input_info, private_keys, password):
 556
 557         unspent_coins = self.get_unspent_coins()
 558         seed = self.decode_seed(password)
 559
 560         # build a list of public/private keys
 561         keypairs = {}
 562         for sec in private_keys:
 563             pubkey = public_key_from_private_key(sec)
 564             keypairs[ pubkey ] = sec
 565
 566
 567         for txin in tx.inputs:
 568             # convert to own format
 569             txin['tx_hash'] = txin['prevout_hash']
 570             txin['index'] = txin['prevout_n']
 571
 572             for item in input_info:
 573                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 574                     txin['raw_output_script'] = item['scriptPubKey']
 575                     txin['redeemScript'] = item.get('redeemScript')
 576                     txin['KeyID'] = item.get('KeyID')
 577                     break
 578             else:
 579                 for item in unspent_coins:
 580                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 581                         print_error( "tx input is in unspent coins" )
 582                         txin['raw_output_script'] = item['raw_output_script']
 583                         account, sequence = self.get_address_index(item['address'])
 584                         if account != -1:
 585                             txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 586                         break
 587                 else:
 588                     raise BaseException("Unknown transaction input. Please provide the 'input_info' parameter, or synchronize this wallet")
 589
 590             # if available, derive private_keys from KeyID
 591             keyid = txin.get('KeyID')
 592             if keyid:
 593                 roots = []
 594                 for s in keyid.split('&'):
 595                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 596                     if not m: continue
 597                     c = m.group(1)
 598                     K = m.group(2)
 599                     sequence = m.group(3)
 600                     root = self.find_root_by_master_key(c,K)
 601                     if not root: continue
 602                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 603                     root = root + '%d'%sequence[0]
 604                     sequence = sequence[1:]
 605                     roots.append((root,sequence))
 606
 607                 account_id = " & ".join( map(lambda x:x[0], roots) )
 608                 account = self.accounts.get(account_id)
 609                 if not account: continue
 610                 addr = account.get_address(*sequence)
 611                 txin['address'] = addr
 612                 pk = self.get_private_key(addr, password)
 613                 for sec in pk:
 614                     pubkey = public_key_from_private_key(sec)
 615                     keypairs[pubkey] = sec
 616
 617             redeem_script = txin.get("redeemScript")
 618             print_error( "p2sh:", "yes" if redeem_script else "no")
 619             if redeem_script:
 620                 addr = hash_160_to_bc_address(hash_160(redeem_script.decode('hex')), 5)
 621             else:
 622                 import transaction
 623                 _, addr = transaction.get_address_from_output_script(txin["raw_output_script"].decode('hex'))
 624             txin['address'] = addr
 625
 626             # add private keys that are in the wallet
 627             pk = self.get_private_key(addr, password)
 628             for sec in pk:
 629                 pubkey = public_key_from_private_key(sec)
 630                 keypairs[pubkey] = sec
 631                 if not redeem_script:
 632                     txin['redeemPubkey'] = pubkey
 633
 634             print txin
 635
 636         tx.sign( keypairs )
 637
 638     def sign_message(self, address, message, password):
 639         keys = self.get_private_key(address, password)
 640         assert len(keys) == 1
 641         sec = keys[0]
 642         key = regenerate_key(sec)
 643         compressed = is_compressed(sec)
 644         return key.sign_message(message, compressed, address)
 645
 646     def verify_message(self, address, signature, message):
 647         try:
 648             EC_KEY.verify_message(address, signature, message)
 649             return True
 650         except BaseException as e:
 651             print_error("Verification error: {0}".format(e))
 652             return False
 653
 654
 655     def change_gap_limit(self, value):
 656         if value >= self.gap_limit:
 657             self.gap_limit = value
 658             self.storage.put('gap_limit', self.gap_limit, True)
 659             #self.interface.poke('synchronizer')
 660             return True
 661
 662         elif value >= self.min_acceptable_gap():
 663             for key, account in self.accounts.items():
 664                 addresses = account[0]
 665                 k = self.num_unused_trailing_addresses(addresses)
 666                 n = len(addresses) - k + value
 667                 addresses = addresses[0:n]
 668                 self.accounts[key][0] = addresses
 669
 670             self.gap_limit = value
 671             self.storage.put('gap_limit', self.gap_limit, True)
 672             self.save_accounts()
 673             return True
 674         else:
 675             return False
 676
 677     def num_unused_trailing_addresses(self, addresses):
 678         k = 0
 679         for a in addresses[::-1]:
 680             if self.history.get(a):break
 681             k = k + 1
 682         return k
 683
 684     def min_acceptable_gap(self):
 685         # fixme: this assumes wallet is synchronized
 686         n = 0
 687         nmax = 0
 688
 689         for account in self.accounts.values():
 690             addresses = account.get_addresses(0)
 691             k = self.num_unused_trailing_addresses(addresses)
 692             for a in addresses[0:-k]:
 693                 if self.history.get(a):
 694                     n = 0
 695                 else:
 696                     n += 1
 697                     if n > nmax: nmax = n
 698         return nmax + 1
 699
 700
 701     def address_is_old(self, address):
 702         age = -1
 703         h = self.history.get(address, [])
 704         if h == ['*']:
 705             return True
 706         for tx_hash, tx_height in h:
 707             if tx_height == 0:
 708                 tx_age = 0
 709             else:
 710                 tx_age = self.verifier.blockchain.height - tx_height + 1
 711             if tx_age > age:
 712                 age = tx_age
 713         return age > 2
 714
 715
 716     def synchronize_sequence(self, account, for_change):
 717         limit = self.gap_limit_for_change if for_change else self.gap_limit
 718         new_addresses = []
 719         while True:
 720             addresses = account.get_addresses(for_change)
 721             if len(addresses) < limit:
 722                 address = account.create_new_address(for_change)
 723                 self.history[address] = []
 724                 new_addresses.append( address )
 725                 continue
 726
 727             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 728                 break
 729             else:
 730                 address = account.create_new_address(for_change)
 731                 self.history[address] = []
 732                 new_addresses.append( address )
 733
 734         return new_addresses
 735
 736
 737
 738     def create_pending_accounts(self):
 739         for account_type in ['1','2of2','2of3']:
 740             a = self.new_account_address(account_type)
 741             if self.address_is_old(a):
 742                 print_error( "creating account", a )
 743                 self.create_account(account_type)
 744
 745
 746     def synchronize_account(self, account):
 747         new = []
 748         new += self.synchronize_sequence(account, 0)
 749         new += self.synchronize_sequence(account, 1)
 750         return new
 751
 752
 753     def synchronize(self):
 754         if self.master_public_keys:
 755             self.create_pending_accounts()
 756         new = []
 757         for account in self.accounts.values():
 758             new += self.synchronize_account(account)
 759         if new:
 760             self.save_accounts()
 761             self.storage.put('addr_history', self.history, True)
 762         return new
 763
 764
 765     def is_found(self):
 766         return self.history.values() != [[]] * len(self.history)
 767
 768
 769     def add_contact(self, address, label=None):
 770         self.addressbook.append(address)
 771         self.storage.put('contacts', self.addressbook, True)
 772         if label:
 773             self.set_label(address, label)
 774
 775
 776     def delete_contact(self, addr):
 777         if addr in self.addressbook:
 778             self.addressbook.remove(addr)
 779             self.storage.put('addressbook', self.addressbook, True)
 780
 781
 782     def fill_addressbook(self):
 783         for tx_hash, tx in self.transactions.items():
 784             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 785             if is_send:
 786                 for addr, v in tx.outputs:
 787                     if not self.is_mine(addr) and addr not in self.addressbook:
 788                         self.addressbook.append(addr)
 789         # redo labels
 790         # self.update_tx_labels()
 791
 792     def get_num_tx(self, address):
 793         n = 0
 794         for tx in self.transactions.values():
 795             if address in map(lambda x:x[0], tx.outputs): n += 1
 796         return n
 797
 798
 799     def get_address_flags(self, addr):
 800         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 801         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 802         return flags
 803
 804
 805     def get_tx_value(self, tx, account=None):
 806         domain = self.get_account_addresses(account)
 807         return tx.get_value(domain, self.prevout_values)
 808
 809
 810     def update_tx_outputs(self, tx_hash):
 811         tx = self.transactions.get(tx_hash)
 812
 813         for i, (addr, value) in enumerate(tx.outputs):
 814             key = tx_hash+ ':%d'%i
 815             self.prevout_values[key] = value
 816
 817         for item in tx.inputs:
 818             if self.is_mine(item.get('address')):
 819                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 820                 self.spent_outputs.append(key)
 821
 822
 823     def get_addr_balance(self, address):
 824         assert self.is_mine(address)
 825         h = self.history.get(address,[])
 826         if h == ['*']: return 0,0
 827         c = u = 0
 828         received_coins = []   # list of coins received at address
 829
 830         for tx_hash, tx_height in h:
 831             tx = self.transactions.get(tx_hash)
 832             if not tx: continue
 833
 834             for i, (addr, value) in enumerate(tx.outputs):
 835                 if addr == address:
 836                     key = tx_hash + ':%d'%i
 837                     received_coins.append(key)
 838
 839         for tx_hash, tx_height in h:
 840             tx = self.transactions.get(tx_hash)
 841             if not tx: continue
 842             v = 0
 843
 844             for item in tx.inputs:
 845                 addr = item.get('address')
 846                 if addr == address:
 847                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 848                     value = self.prevout_values.get( key )
 849                     if key in received_coins:
 850                         v -= value
 851
 852             for i, (addr, value) in enumerate(tx.outputs):
 853                 key = tx_hash + ':%d'%i
 854                 if addr == address:
 855                     v += value
 856
 857             if tx_height:
 858                 c += v
 859             else:
 860                 u += v
 861         return c, u
 862
 863
 864     def get_account_name(self, k):
 865         if k == 0:
 866             if self.seed_version == 4:
 867                 name = 'Main account'
 868             else:
 869                 name = 'Old account'
 870         else:
 871             name = self.labels.get(k, 'Unnamed account')
 872         return name
 873
 874     def get_account_names(self):
 875         accounts = {}
 876         for k, account in self.accounts.items():
 877             accounts[k] = self.get_account_name(k)
 878         if self.imported_keys:
 879             accounts[-1] = 'Imported keys'
 880         return accounts
 881
 882     def get_account_addresses(self, a, include_change=True):
 883         if a is None:
 884             o = self.addresses(True)
 885         elif a == -1:
 886             o = self.imported_keys.keys()
 887         else:
 888             ac = self.accounts[a]
 889             o = ac.get_addresses(0)
 890             if include_change: o += ac.get_addresses(1)
 891         return o
 892
 893     def get_imported_balance(self):
 894         cc = uu = 0
 895         for addr in self.imported_keys.keys():
 896             c, u = self.get_addr_balance(addr)
 897             cc += c
 898             uu += u
 899         return cc, uu
 900
 901     def get_account_balance(self, account):
 902         if account is None:
 903             return self.get_balance()
 904         elif account == -1:
 905             return self.get_imported_balance()
 906
 907         conf = unconf = 0
 908         for addr in self.get_account_addresses(account):
 909             c, u = self.get_addr_balance(addr)
 910             conf += c
 911             unconf += u
 912         return conf, unconf
 913
 914     def get_frozen_balance(self):
 915         conf = unconf = 0
 916         for addr in self.frozen_addresses:
 917             c, u = self.get_addr_balance(addr)
 918             conf += c
 919             unconf += u
 920         return conf, unconf
 921
 922
 923     def get_balance(self):
 924         cc = uu = 0
 925         for a in self.accounts.keys():
 926             c, u = self.get_account_balance(a)
 927             cc += c
 928             uu += u
 929         c, u = self.get_imported_balance()
 930         cc += c
 931         uu += u
 932         return cc, uu
 933
 934
 935     def get_unspent_coins(self, domain=None):
 936         coins = []
 937         if domain is None: domain = self.addresses(True)
 938         for addr in domain:
 939             h = self.history.get(addr, [])
 940             if h == ['*']: continue
 941             for tx_hash, tx_height in h:
 942                 tx = self.transactions.get(tx_hash)
 943                 if tx is None: raise BaseException("Wallet not synchronized")
 944                 for output in tx.d.get('outputs'):
 945                     if output.get('address') != addr: continue
 946                     key = tx_hash + ":%d" % output.get('index')
 947                     if key in self.spent_outputs: continue
 948                     output['tx_hash'] = tx_hash
 949                     coins.append(output)
 950         return coins
 951
 952
 953
 954     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
 955         """ todo: minimize tx size """
 956         total = 0
 957         fee = self.fee if fixed_fee is None else fixed_fee
 958         if domain is None:
 959             domain.self.addresses()
 960         coins = []
 961         prioritized_coins = []
 962         for i in self.frozen_addresses:
 963             if i in domain: domain.remove(i)
 964
 965         for i in self.prioritized_addresses:
 966             if i in domain: domain.remove(i)
 967
 968         coins = self.get_unspent_coins(domain)
 969         prioritized_coins = self.get_unspent_coins(self.prioritized_addresses)
 970
 971         inputs = []
 972         coins = prioritized_coins + coins
 973
 974         for item in coins:
 975             addr = item.get('address')
 976             v = item.get('value')
 977             total += v
 978             inputs.append( item )
 979             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
 980             if total >= amount + fee: break
 981         else:
 982             inputs = []
 983
 984         return inputs, total, fee
 985
 986
 987     def set_fee(self, fee):
 988         if self.fee != fee:
 989             self.fee = fee
 990             self.storage.put('fee_per_kb', self.fee, True)
 991
 992     def estimated_fee(self, inputs):
 993         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
 994         fee = self.fee * int(round(estimated_size/1024.))
 995         if fee == 0: fee = self.fee
 996         return fee
 997
 998
 999     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1000         "add change to a transaction"
1001         change_amount = total - ( amount + fee )
1002         if change_amount != 0:
1003             if not change_addr:
1004
1005                 # send change to one of the accounts involved in the tx
1006                 address = inputs[0].get('address')
1007                 account, _ = self.get_address_index(address)
1008
1009                 if not self.use_change or account == -1:
1010                     change_addr = inputs[-1]['address']
1011                 else:
1012                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1013
1014             # Insert the change output at a random position in the outputs
1015             posn = random.randint(0, len(outputs))
1016             outputs[posn:posn] = [( change_addr,  change_amount)]
1017         return outputs
1018
1019
1020     def get_history(self, address):
1021         with self.lock:
1022             return self.history.get(address)
1023
1024
1025     def get_status(self, h):
1026         if not h: return None
1027         if h == ['*']: return '*'
1028         status = ''
1029         for tx_hash, height in h:
1030             status += tx_hash + ':%d:' % height
1031         return hashlib.sha256( status ).digest().encode('hex')
1032
1033
1034     def receive_tx_callback(self, tx_hash, tx, tx_height):
1035
1036         with self.transaction_lock:
1037             self.add_extra_addresses(tx)
1038             if not self.check_new_tx(tx_hash, tx):
1039                 # may happen due to pruning
1040                 print_error("received transaction that is no longer referenced in history", tx_hash)
1041                 return
1042             self.transactions[tx_hash] = tx
1043             self.network.interface.pending_transactions_for_notifications.append(tx)
1044             self.save_transactions()
1045             if self.verifier and tx_height>0:
1046                 self.verifier.add(tx_hash, tx_height)
1047             self.update_tx_outputs(tx_hash)
1048
1049
1050     def save_transactions(self):
1051         tx = {}
1052         for k,v in self.transactions.items():
1053             tx[k] = str(v)
1054         self.storage.put('transactions', tx, True)
1055
1056     def receive_history_callback(self, addr, hist):
1057
1058         if not self.check_new_history(addr, hist):
1059             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1060
1061         with self.lock:
1062             self.history[addr] = hist
1063             self.storage.put('addr_history', self.history, True)
1064
1065         if hist != ['*']:
1066             for tx_hash, tx_height in hist:
1067                 if tx_height>0:
1068                     # add it in case it was previously unconfirmed
1069                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1070
1071
1072     def get_tx_history(self, account=None):
1073         with self.transaction_lock:
1074             history = self.transactions.items()
1075             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1076             result = []
1077
1078             balance = 0
1079             for tx_hash, tx in history:
1080                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1081                 if v is not None: balance += v
1082
1083             c, u = self.get_account_balance(account)
1084
1085             if balance != c+u:
1086                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1087
1088             balance = c + u - balance
1089             for tx_hash, tx in history:
1090                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1091                 if not is_relevant:
1092                     continue
1093                 if value is not None:
1094                     balance += value
1095
1096                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1097                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1098
1099         return result
1100
1101
1102     def get_label(self, tx_hash):
1103         label = self.labels.get(tx_hash)
1104         is_default = (label == '') or (label is None)
1105         if is_default: label = self.get_default_label(tx_hash)
1106         return label, is_default
1107
1108
1109     def get_default_label(self, tx_hash):
1110         tx = self.transactions.get(tx_hash)
1111         default_label = ''
1112         if tx:
1113             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1114             if is_mine:
1115                 for o in tx.outputs:
1116                     o_addr, _ = o
1117                     if not self.is_mine(o_addr):
1118                         try:
1119                             default_label = self.labels[o_addr]
1120                         except KeyError:
1121                             default_label = o_addr
1122                         break
1123                 else:
1124                     default_label = '(internal)'
1125             else:
1126                 for o in tx.outputs:
1127                     o_addr, _ = o
1128                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1129                         break
1130                 else:
1131                     for o in tx.outputs:
1132                         o_addr, _ = o
1133                         if self.is_mine(o_addr):
1134                             break
1135                     else:
1136                         o_addr = None
1137
1138                 if o_addr:
1139                     dest_label = self.labels.get(o_addr)
1140                     try:
1141                         default_label = self.labels[o_addr]
1142                     except KeyError:
1143                         default_label = o_addr
1144
1145         return default_label
1146
1147
1148     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1149         for address, x in outputs:
1150             assert is_valid(address)
1151         amount = sum( map(lambda x:x[1], outputs) )
1152         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1153         if not inputs:
1154             raise ValueError("Not enough funds")
1155         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1156         return Transaction.from_io(inputs, outputs)
1157
1158
1159     def mktx_from_account(self, outputs, password, fee=None, change_addr=None, account=None):
1160         if account:
1161             domain = self.get_account_addresses(account)
1162         self.mktx(outputs, password, fee, change_addr, domain)
1163
1164
1165     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1166         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1167         self.sign_transaction(tx, password)
1168         return tx
1169
1170
1171     def sign_transaction(self, tx, password):
1172         keypairs = {}
1173         for i, txin in enumerate(tx.inputs):
1174             address = txin['address']
1175             account, sequence = self.get_address_index(address)
1176             txin['KeyID'] = self.get_keyID(account, sequence)
1177             redeemScript = self.accounts[account].redeem_script(sequence)
1178             if redeemScript:
1179                 txin['redeemScript'] = redeemScript
1180             else:
1181                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1182             private_keys = self.get_private_key(address, password)
1183             for sec in private_keys:
1184                 pubkey = public_key_from_private_key(sec)
1185                 keypairs[ pubkey ] = sec
1186         tx.sign(keypairs)
1187
1188
1189     def sendtx(self, tx):
1190         # synchronous
1191         h = self.send_tx(tx)
1192         self.tx_event.wait()
1193         return self.receive_tx(h)
1194
1195     def send_tx(self, tx):
1196         # asynchronous
1197         self.tx_event.clear()
1198         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1199         return tx.hash()
1200
1201     def on_broadcast(self, i, r):
1202         self.tx_result = r.get('result')
1203         self.tx_event.set()
1204
1205     def receive_tx(self,tx_hash):
1206         out = self.tx_result
1207         if out != tx_hash:
1208             return False, "error: " + out
1209         return True, out
1210
1211
1212
1213     def update_password(self, seed, old_password, new_password):
1214         if new_password == '': new_password = None
1215         # this will throw an exception if unicode cannot be converted
1216         self.seed = pw_encode( seed, new_password)
1217         self.storage.put('seed', self.seed, True)
1218         self.use_encryption = (new_password != None)
1219         self.storage.put('use_encryption', self.use_encryption,True)
1220         for k in self.imported_keys.keys():
1221             a = self.imported_keys[k]
1222             b = pw_decode(a, old_password)
1223             c = pw_encode(b, new_password)
1224             self.imported_keys[k] = c
1225         self.storage.put('imported_keys', self.imported_keys, True)
1226
1227         for k, v in self.master_private_keys.items():
1228             b = pw_decode(v, old_password)
1229             c = pw_encode(b, new_password)
1230             self.master_private_keys[k] = c
1231         self.storage.put('master_private_keys', self.master_private_keys, True)
1232
1233
1234     def freeze(self,addr):
1235         if self.is_mine(addr) and addr not in self.frozen_addresses:
1236             self.unprioritize(addr)
1237             self.frozen_addresses.append(addr)
1238             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1239             return True
1240         else:
1241             return False
1242
1243     def unfreeze(self,addr):
1244         if self.is_mine(addr) and addr in self.frozen_addresses:
1245             self.frozen_addresses.remove(addr)
1246             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1247             return True
1248         else:
1249             return False
1250
1251     def prioritize(self,addr):
1252         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1253             self.unfreeze(addr)
1254             self.prioritized_addresses.append(addr)
1255             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1256             return True
1257         else:
1258             return False
1259
1260     def unprioritize(self,addr):
1261         if self.is_mine(addr) and addr in self.prioritized_addresses:
1262             self.prioritized_addresses.remove(addr)
1263             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1264             return True
1265         else:
1266             return False
1267
1268
1269     def set_verifier(self, verifier):
1270         self.verifier = verifier
1271
1272         # review transactions that are in the history
1273         for addr, hist in self.history.items():
1274             if hist == ['*']: continue
1275             for tx_hash, tx_height in hist:
1276                 if tx_height>0:
1277                     # add it in case it was previously unconfirmed
1278                     self.verifier.add(tx_hash, tx_height)
1279
1280
1281         # if we are on a pruning server, remove unverified transactions
1282         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1283         for tx_hash in self.transactions.keys():
1284             if tx_hash not in vr:
1285                 self.transactions.pop(tx_hash)
1286
1287
1288
1289     def check_new_history(self, addr, hist):
1290
1291         # check that all tx in hist are relevant
1292         if hist != ['*']:
1293             for tx_hash, height in hist:
1294                 tx = self.transactions.get(tx_hash)
1295                 if not tx: continue
1296                 if not tx.has_address(addr):
1297                     return False
1298
1299         # check that we are not "orphaning" a transaction
1300         old_hist = self.history.get(addr,[])
1301         if old_hist == ['*']: return True
1302
1303         for tx_hash, height in old_hist:
1304             if tx_hash in map(lambda x:x[0], hist): continue
1305             found = False
1306             for _addr, _hist in self.history.items():
1307                 if _addr == addr: continue
1308                 if _hist == ['*']: continue
1309                 _tx_hist = map(lambda x:x[0], _hist)
1310                 if tx_hash in _tx_hist:
1311                     found = True
1312                     break
1313
1314             if not found:
1315                 tx = self.transactions.get(tx_hash)
1316                 # tx might not be there
1317                 if not tx: continue
1318
1319                 # already verified?
1320                 if self.verifier.get_height(tx_hash):
1321                     continue
1322                 # unconfirmed tx
1323                 print_error("new history is orphaning transaction:", tx_hash)
1324                 # check that all outputs are not mine, request histories
1325                 ext_requests = []
1326                 for _addr, _v in tx.outputs:
1327                     # assert not self.is_mine(_addr)
1328                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1329
1330                 ext_h = self.network.interface.synchronous_get(ext_requests)
1331                 print_error("sync:", ext_requests, ext_h)
1332                 height = None
1333                 for h in ext_h:
1334                     if h == ['*']: continue
1335                     for item in h:
1336                         if item.get('tx_hash') == tx_hash:
1337                             height = item.get('height')
1338                 if height:
1339                     print_error("found height for", tx_hash, height)
1340                     self.verifier.add(tx_hash, height)
1341                 else:
1342                     print_error("removing orphaned tx from history", tx_hash)
1343                     self.transactions.pop(tx_hash)
1344
1345         return True
1346
1347
1348
1349     def check_new_tx(self, tx_hash, tx):
1350         # 1 check that tx is referenced in addr_history.
1351         addresses = []
1352         for addr, hist in self.history.items():
1353             if hist == ['*']:continue
1354             for txh, height in hist:
1355                 if txh == tx_hash:
1356                     addresses.append(addr)
1357
1358         if not addresses:
1359             return False
1360
1361         # 2 check that referencing addresses are in the tx
1362         for addr in addresses:
1363             if not tx.has_address(addr):
1364                 return False
1365
1366         return True
1367
1368
1369     def start_threads(self, network):
1370         from verifier import TxVerifier
1371         self.network = network
1372         self.verifier = TxVerifier(self.network, self.storage)
1373         self.verifier.start()
1374         self.set_verifier(self.verifier)
1375         self.synchronizer = WalletSynchronizer(self)
1376         self.synchronizer.start()
1377
1378     def stop_threads(self):
1379         self.verifier.stop()
1380         self.synchronizer.stop()
1381
1382
1383
1384
1385
1386 class WalletSynchronizer(threading.Thread):
1387
1388
1389     def __init__(self, wallet):
1390         threading.Thread.__init__(self)
1391         self.daemon = True
1392         self.wallet = wallet
1393         wallet.synchronizer = self
1394         self.network = self.wallet.network
1395         #self.wallet.network.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1396         self.was_updated = True
1397         self.running = False
1398         self.lock = threading.Lock()
1399         self.queue = Queue.Queue()
1400
1401     def stop(self):
1402         with self.lock: self.running = False
1403
1404     def is_running(self):
1405         with self.lock: return self.running
1406
1407
1408     def subscribe_to_addresses(self, addresses):
1409         messages = []
1410         for addr in addresses:
1411             messages.append(('blockchain.address.subscribe', [addr]))
1412         self.network.interface.send( messages, lambda i,r: self.queue.put(r))
1413
1414
1415     def run(self):
1416         with self.lock:
1417             self.running = True
1418
1419         while self.is_running():
1420             interface = self.network.interface
1421             if not interface.is_connected:
1422                 print_error("synchronizer: waiting for interface")
1423                 interface.connect_event.wait()
1424
1425             self.run_interface(interface)
1426
1427
1428     def run_interface(self, interface):
1429
1430         print_error("synchronizer: connected to", interface.server)
1431
1432         requested_tx = []
1433         missing_tx = []
1434         requested_histories = {}
1435
1436         # request any missing transactions
1437         for history in self.wallet.history.values():
1438             if history == ['*']: continue
1439             for tx_hash, tx_height in history:
1440                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1441                     missing_tx.append( (tx_hash, tx_height) )
1442
1443         if missing_tx:
1444             print_error("missing tx", missing_tx)
1445
1446         # subscriptions
1447         self.subscribe_to_addresses(self.wallet.addresses(True, next=True))
1448
1449         while self.is_running():
1450             # 1. create new addresses
1451             new_addresses = self.wallet.synchronize()
1452
1453             # request missing addresses
1454             if new_addresses:
1455                 self.subscribe_to_addresses(new_addresses)
1456
1457             # request missing transactions
1458             for tx_hash, tx_height in missing_tx:
1459                 if (tx_hash, tx_height) not in requested_tx:
1460                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1461                     requested_tx.append( (tx_hash, tx_height) )
1462             missing_tx = []
1463
1464             # detect if situation has changed
1465             if interface.is_up_to_date() and self.queue.empty():
1466                 if not self.wallet.is_up_to_date():
1467                     self.wallet.set_up_to_date(True)
1468                     self.was_updated = True
1469             else:
1470                 if self.wallet.is_up_to_date():
1471                     self.wallet.set_up_to_date(False)
1472                     self.was_updated = True
1473
1474             if self.was_updated:
1475                 self.wallet.network.trigger_callback('updated')
1476                 self.was_updated = False
1477
1478             # 2. get a response
1479             try:
1480                 r = self.queue.get(block=True, timeout=1)
1481             except Queue.Empty:
1482                 continue
1483
1484             if interface != self.network.interface:
1485                 break
1486
1487             if not r:
1488                 continue
1489
1490             # 3. handle response
1491             method = r['method']
1492             params = r['params']
1493             result = r.get('result')
1494             error = r.get('error')
1495             if error:
1496                 print "error", r
1497                 continue
1498
1499             if method == 'blockchain.address.subscribe':
1500                 addr = params[0]
1501                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1502                     if requested_histories.get(addr) is None:
1503                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1504                         requested_histories[addr] = result
1505
1506             elif method == 'blockchain.address.get_history':
1507                 addr = params[0]
1508                 print_error("receiving history", addr, result)
1509                 if result == ['*']:
1510                     assert requested_histories.pop(addr) == '*'
1511                     self.wallet.receive_history_callback(addr, result)
1512                 else:
1513                     hist = []
1514                     # check that txids are unique
1515                     txids = []
1516                     for item in result:
1517                         tx_hash = item['tx_hash']
1518                         if tx_hash not in txids:
1519                             txids.append(tx_hash)
1520                             hist.append( (tx_hash, item['height']) )
1521
1522                     if len(hist) != len(result):
1523                         raise BaseException("error: server sent history with non-unique txid", result)
1524
1525                     # check that the status corresponds to what was announced
1526                     rs = requested_histories.pop(addr)
1527                     if self.wallet.get_status(hist) != rs:
1528                         raise BaseException("error: status mismatch: %s"%addr)
1529
1530                     # store received history
1531                     self.wallet.receive_history_callback(addr, hist)
1532
1533                     # request transactions that we don't have
1534                     for tx_hash, tx_height in hist:
1535                         if self.wallet.transactions.get(tx_hash) is None:
1536                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1537                                 missing_tx.append( (tx_hash, tx_height) )
1538
1539             elif method == 'blockchain.transaction.get':
1540                 tx_hash = params[0]
1541                 tx_height = params[1]
1542                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1543                 tx = Transaction(result)
1544                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1545                 self.was_updated = True
1546                 requested_tx.remove( (tx_hash, tx_height) )
1547                 print_error("received tx:", tx_hash, len(tx.raw))
1548
1549             else:
1550                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1551
1552             if self.was_updated and not requested_tx:
1553                 self.wallet.network.trigger_callback('updated')
1554                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1555
1556                 self.was_updated = False