lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 # AES encryption
  40 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  41 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  42
  43 def pw_encode(s, password):
  44     if password:
  45         secret = Hash(password)
  46         return EncodeAES(secret, s)
  47     else:
  48         return s
  49
  50 def pw_decode(s, password):
  51     if password is not None:
  52         secret = Hash(password)
  53         try:
  54             d = DecodeAES(secret, s)
  55         except:
  56             raise BaseException('Invalid password')
  57         return d
  58     else:
  59         return s
  60
  61
  62
  63
  64
  65 from version import ELECTRUM_VERSION, SEED_VERSION
  66
  67
  68 class WalletStorage:
  69
  70     def __init__(self, config):
  71         self.lock = threading.Lock()
  72         self.data = {}
  73         self.file_exists = False
  74         self.init_path(config)
  75         print_error( "wallet path", self.path )
  76         if self.path:
  77             self.read(self.path)
  78
  79
  80     def init_path(self, config):
  81         """Set the path of the wallet."""
  82
  83         path = config.get('wallet_path')
  84         if not path:
  85             path = config.get('default_wallet_path')
  86         if path is not None:
  87             self.path = path
  88             return
  89
  90         self.path = os.path.join(config.path, "electrum.dat")
  91
  92
  93     def read(self, path):
  94         """Read the contents of the wallet file."""
  95         try:
  96             with open(self.path, "r") as f:
  97                 data = f.read()
  98         except IOError:
  99             return
 100         try:
 101             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 102         except:
 103             raise IOError("Cannot read wallet file.")
 104
 105         self.data = d
 106         self.file_exists = True
 107
 108
 109     def get(self, key, default=None):
 110         return self.data.get(key, default)
 111
 112     def put(self, key, value, save = True):
 113
 114         with self.lock:
 115             if value is not None:
 116                 self.data[key] = value
 117             else:
 118                 self.data.pop[key]
 119             if save:
 120                 self.write()
 121
 122     def write(self):
 123         s = repr(self.data)
 124         f = open(self.path,"w")
 125         f.write( s )
 126         f.close()
 127         if self.get('gui') != 'android':
 128             import stat
 129             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 130
 131
 132 class Wallet:
 133
 134     def __init__(self, storage):
 135
 136         self.storage = storage
 137         self.electrum_version = ELECTRUM_VERSION
 138         self.gap_limit_for_change = 3 # constant
 139
 140         # saved fields
 141         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 142
 143         self.gap_limit             = storage.get('gap_limit', 5)
 144         self.use_change            = storage.get('use_change',True)
 145         self.use_encryption        = storage.get('use_encryption', False)
 146         self.seed                  = storage.get('seed', '')               # encrypted
 147         self.labels                = storage.get('labels', {})
 148         self.frozen_addresses      = storage.get('frozen_addresses',[])
 149         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 150         self.addressbook           = storage.get('contacts', [])
 151
 152         self.imported_keys         = storage.get('imported_keys',{})
 153         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 154
 155         self.fee                   = int(storage.get('fee_per_kb',20000))
 156
 157         self.master_public_keys = storage.get('master_public_keys',{})
 158         self.master_private_keys = storage.get('master_private_keys', {})
 159
 160         self.next_addresses = storage.get('next_addresses',{})
 161
 162         if self.seed_version < 4:
 163             raise ValueError("This wallet seed is deprecated.")
 164
 165         self.load_accounts()
 166
 167         self.transactions = {}
 168         tx_list = self.storage.get('transactions',{})
 169         for k,v in tx_list.items():
 170             try:
 171                 tx = Transaction(v)
 172             except:
 173                 print_msg("Warning: Cannot deserialize transactions. skipping")
 174                 continue
 175
 176             self.add_extra_addresses(tx)
 177             self.transactions[k] = tx
 178
 179         for h,tx in self.transactions.items():
 180             if not self.check_new_tx(h, tx):
 181                 print_error("removing unreferenced tx", h)
 182                 self.transactions.pop(h)
 183
 184
 185         # not saved
 186         self.prevout_values = {}     # my own transaction outputs
 187         self.spent_outputs = []
 188
 189         # spv
 190         self.verifier = None
 191
 192         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 193         # interface.is_up_to_date() returns true when all requests have been answered and processed
 194         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 195
 196         self.up_to_date = False
 197         self.lock = threading.Lock()
 198         self.transaction_lock = threading.Lock()
 199         self.tx_event = threading.Event()
 200
 201         for tx_hash, tx in self.transactions.items():
 202             self.update_tx_outputs(tx_hash)
 203
 204
 205     def add_extra_addresses(self, tx):
 206         h = tx.hash()
 207         # find the address corresponding to pay-to-pubkey inputs
 208         tx.add_extra_addresses(self.transactions)
 209         for o in tx.d.get('outputs'):
 210             if o.get('is_pubkey'):
 211                 for tx2 in self.transactions.values():
 212                     tx2.add_extra_addresses({h:tx})
 213
 214
 215
 216
 217     def set_up_to_date(self,b):
 218         with self.lock: self.up_to_date = b
 219
 220
 221     def is_up_to_date(self):
 222         with self.lock: return self.up_to_date
 223
 224
 225     def update(self):
 226         self.up_to_date = False
 227         while not self.is_up_to_date():
 228             time.sleep(0.1)
 229
 230
 231     def import_key(self, sec, password):
 232         # check password
 233         seed = self.decode_seed(password)
 234         try:
 235             address = address_from_private_key(sec)
 236         except:
 237             raise BaseException('Invalid private key')
 238
 239         if self.is_mine(address):
 240             raise BaseException('Address already in wallet')
 241
 242         # store the originally requested keypair into the imported keys table
 243         self.imported_keys[address] = pw_encode(sec, password )
 244         self.storage.put('imported_keys', self.imported_keys, True)
 245         if self.synchronizer:
 246             self.synchronizer.subscribe_to_addresses([address])
 247         return address
 248
 249     def delete_imported_key(self, addr):
 250         if addr in self.imported_keys:
 251             self.imported_keys.pop(addr)
 252             self.storage.put('imported_keys', self.imported_keys, True)
 253
 254
 255     def init_seed(self, seed):
 256         if self.seed: raise BaseException("a seed exists")
 257         if not seed:
 258             seed = random_seed(128)
 259         self.seed = seed
 260
 261
 262     def save_seed(self):
 263         self.storage.put('seed', self.seed, True)
 264         self.storage.put('seed_version', self.seed_version, True)
 265
 266     def create_watching_only_wallet(self, c0, K0):
 267         cK0 = ""
 268         self.master_public_keys = {
 269             "m/0'/": (c0, K0, cK0),
 270             }
 271         self.storage.put('master_public_keys', self.master_public_keys, True)
 272         self.create_account('1','Main account')
 273
 274     def create_accounts(self):
 275
 276         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 277
 278         # normal accounts
 279         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 280         # p2sh 2of2
 281         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 282         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 283         # p2sh 2of3
 284         k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 285         k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 286         k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 287
 288         self.master_public_keys = {
 289             "m/0'/": (c0, K0, cK0),
 290             "m/1'/": (c1, K1, cK1),
 291             "m/2'/": (c2, K2, cK2),
 292             "m/3'/": (c3, K3, cK3),
 293             "m/4'/": (c4, K4, cK4),
 294             "m/5'/": (c5, K5, cK5)
 295             }
 296
 297         self.master_private_keys = {
 298             "m/0'/": k0,
 299             "m/1'/": k1,
 300             "m/2'/": k2,
 301             "m/3'/": k3,
 302             "m/4'/": k4,
 303             "m/5'/": k5
 304             }
 305
 306         self.storage.put('master_public_keys', self.master_public_keys, True)
 307         self.storage.put('master_private_keys', self.master_private_keys, True)
 308
 309         # create default account
 310         self.create_account('1','Main account')
 311
 312
 313     def find_root_by_master_key(self, c, K):
 314         for key, v in self.master_public_keys.items():
 315             if key == "m/":continue
 316             cc, KK, _ = v
 317             if (c == cc) and (K == KK):
 318                 return key
 319
 320     def deseed_root(self, seed, password):
 321         # for safety, we ask the user to enter their seed
 322         assert seed == self.decode_seed(password)
 323         self.seed = ''
 324         self.storage.put('seed', '', True)
 325
 326
 327     def deseed_branch(self, k):
 328         # check that parent has no seed
 329         assert self.seed == ''
 330         self.master_private_keys.pop(k)
 331         self.storage.put('master_private_keys', self.master_private_keys, True)
 332
 333
 334     def account_id(self, account_type, i):
 335         if account_type == '1':
 336             return "m/0'/%d"%i
 337         elif account_type == '2of2':
 338             return "m/1'/%d & m/2'/%d"%(i,i)
 339         elif account_type == '2of3':
 340             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 341         else:
 342             raise BaseException('unknown account type')
 343
 344
 345     def num_accounts(self, account_type):
 346         keys = self.accounts.keys()
 347         i = 0
 348         while True:
 349             account_id = self.account_id(account_type, i)
 350             if account_id not in keys: break
 351             i += 1
 352         return i
 353
 354
 355     def new_account_address(self, account_type = '1'):
 356         i = self.num_accounts(account_type)
 357         k = self.account_id(account_type,i)
 358
 359         addr = self.next_addresses.get(k)
 360         if not addr:
 361             account_id, account = self.next_account(account_type)
 362             addr = account.first_address()
 363             self.next_addresses[k] = addr
 364             self.storage.put('next_addresses',self.next_addresses)
 365
 366         return addr
 367
 368
 369     def next_account(self, account_type = '1'):
 370
 371         i = self.num_accounts(account_type)
 372         account_id = self.account_id(account_type,i)
 373
 374         if account_type is '1':
 375             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 376             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 377             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 378
 379         elif account_type == '2of2':
 380             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 381             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 382             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 383             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 384             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 385
 386         elif account_type == '2of3':
 387             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 388             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 389             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 390             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 391             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 392             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 393             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 394
 395         return account_id, account
 396
 397
 398     def set_label(self, name, text = None):
 399         changed = False
 400         old_text = self.labels.get(name)
 401         if text:
 402             if old_text != text:
 403                 self.labels[name] = text
 404                 changed = True
 405         else:
 406             if old_text:
 407                 self.labels.pop(name)
 408                 changed = True
 409
 410         if changed:
 411             self.storage.put('labels', self.labels, True)
 412
 413         run_hook('set_label', name, text, changed)
 414         return changed
 415
 416
 417
 418     def create_account(self, account_type = '1', name = None):
 419         account_id, account = self.next_account(account_type)
 420         self.accounts[account_id] = account
 421         self.save_accounts()
 422         if name:
 423             self.set_label(account_id, name)
 424
 425
 426     def create_old_account(self):
 427         mpk = OldAccount.mpk_from_seed(self.seed)
 428         self.storage.put('master_public_key', mpk, True)
 429         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 430         self.save_accounts()
 431
 432
 433     def save_accounts(self):
 434         d = {}
 435         for k, v in self.accounts.items():
 436             d[k] = v.dump()
 437         self.storage.put('accounts', d, True)
 438
 439
 440
 441     def load_accounts(self):
 442         d = self.storage.get('accounts', {})
 443         self.accounts = {}
 444         for k, v in d.items():
 445             if k == 0:
 446                 v['mpk'] = self.storage.get('master_public_key')
 447                 self.accounts[k] = OldAccount(v)
 448             elif '&' in k:
 449                 self.accounts[k] = BIP32_Account_2of2(v)
 450             else:
 451                 self.accounts[k] = BIP32_Account(v)
 452
 453
 454     def addresses(self, include_change = True, next=False):
 455         o = self.get_account_addresses(-1, include_change)
 456         for a in self.accounts.keys():
 457             o += self.get_account_addresses(a, include_change)
 458
 459         if next:
 460             for addr in self.next_addresses.values():
 461                 if addr not in o:
 462                     o += [addr]
 463         return o
 464
 465
 466     def is_mine(self, address):
 467         return address in self.addresses(True)
 468
 469
 470     def is_change(self, address):
 471         if not self.is_mine(address): return False
 472         if address in self.imported_keys.keys(): return False
 473         acct, s = self.get_address_index(address)
 474         if s is None: return False
 475         return s[0] == 1
 476
 477     def get_master_public_key(self):
 478         if self.seed_version == 4:
 479             return self.storage.get("master_public_key")
 480         else:
 481             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 482             return repr((c, K))
 483
 484     def get_master_private_key(self, account, password):
 485         master_k = pw_decode( self.master_private_keys[account], password)
 486         master_c, master_K, master_Kc = self.master_public_keys[account]
 487         try:
 488             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 489             assert K.encode('hex') == master_K
 490         except:
 491             raise BaseException("Invalid password")
 492         return master_k
 493
 494
 495     def get_address_index(self, address):
 496         if address in self.imported_keys.keys():
 497             return -1, None
 498
 499         for account in self.accounts.keys():
 500             for for_change in [0,1]:
 501                 addresses = self.accounts[account].get_addresses(for_change)
 502                 for addr in addresses:
 503                     if address == addr:
 504                         return account, (for_change, addresses.index(addr))
 505
 506         raise BaseException("Address not found", address)
 507
 508
 509     def get_roots(self, account):
 510         roots = []
 511         for a in account.split('&'):
 512             s = a.strip()
 513             m = re.match("(m/\d+'/)(\d+)", s)
 514             roots.append( m.group(1) )
 515         return roots
 516
 517     def is_seeded(self, account):
 518         if type(account) is int:
 519             return self.seed is not None
 520
 521         for root in self.get_roots(account):
 522             if root not in self.master_private_keys.keys():
 523                 return False
 524         return True
 525
 526     def rebase_sequence(self, account, sequence):
 527         c, i = sequence
 528         dd = []
 529         for a in account.split('&'):
 530             s = a.strip()
 531             m = re.match("(m/\d+'/)(\d+)", s)
 532             root = m.group(1)
 533             num = int(m.group(2))
 534             dd.append( (root, [num,c,i] ) )
 535         return dd
 536
 537
 538     def get_keyID(self, account, sequence):
 539         if account == 0:
 540             return 'old'
 541
 542         rs = self.rebase_sequence(account, sequence)
 543         dd = []
 544         for root, public_sequence in rs:
 545             c, K, _ = self.master_public_keys[root]
 546             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 547             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 548         return '&'.join(dd)
 549
 550
 551     def get_public_key(self, address):
 552         account, sequence = self.get_address_index(address)
 553         return self.accounts[account].get_pubkey( *sequence )
 554
 555
 556     def decode_seed(self, password):
 557         seed = pw_decode(self.seed, password)
 558         #todo:  #self.sequences[0].check_seed(seed)
 559         return seed
 560
 561
 562     def get_private_key(self, address, password):
 563         out = []
 564         if address in self.imported_keys.keys():
 565             out.append( pw_decode( self.imported_keys[address], password ) )
 566         else:
 567             account, sequence = self.get_address_index(address)
 568             if account == 0:
 569                 seed = self.decode_seed(password)
 570                 pk = self.accounts[account].get_private_key(seed, sequence)
 571                 out.append(pk)
 572                 return out
 573
 574             # assert address == self.accounts[account].get_address(*sequence)
 575             rs = self.rebase_sequence( account, sequence)
 576             for root, public_sequence in rs:
 577
 578                 if root not in self.master_private_keys.keys(): continue
 579                 master_k = self.get_master_private_key(root, password)
 580                 master_c, _, _ = self.master_public_keys[root]
 581                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 582                 out.append(pk)
 583
 584         return out
 585
 586
 587     def add_keypairs_from_wallet(self, tx, keypairs, password):
 588         for txin in tx.inputs:
 589             address = txin['address']
 590             private_keys = self.get_private_key(address, password)
 591             for sec in private_keys:
 592                 pubkey = public_key_from_private_key(sec)
 593                 keypairs[ pubkey ] = sec
 594
 595
 596     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 597         for txin in tx.inputs:
 598             keyid = txin.get('KeyID')
 599             if keyid:
 600                 roots = []
 601                 for s in keyid.split('&'):
 602                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 603                     if not m: continue
 604                     c = m.group(1)
 605                     K = m.group(2)
 606                     sequence = m.group(3)
 607                     root = self.find_root_by_master_key(c,K)
 608                     if not root: continue
 609                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 610                     root = root + '%d'%sequence[0]
 611                     sequence = sequence[1:]
 612                     roots.append((root,sequence))
 613
 614                 account_id = " & ".join( map(lambda x:x[0], roots) )
 615                 account = self.accounts.get(account_id)
 616                 if not account: continue
 617                 addr = account.get_address(*sequence)
 618                 txin['address'] = addr # fixme: side effect
 619                 pk = self.get_private_key(addr, password)
 620                 for sec in pk:
 621                     pubkey = public_key_from_private_key(sec)
 622                     keypairs[pubkey] = sec
 623
 624
 625
 626     def signrawtransaction(self, tx, input_info, private_keys, password):
 627
 628         # check that the password is correct
 629         seed = self.decode_seed(password)
 630
 631         # add input info
 632         tx.add_input_info(input_info)
 633
 634         # add redeem script for coins that are in the wallet
 635         # FIXME: add redeemPubkey too!
 636         unspent_coins = self.get_unspent_coins()
 637         for txin in tx.inputs:
 638             for item in unspent_coins:
 639                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 640                     print_error( "tx input is in unspent coins" )
 641                     txin['scriptPubKey'] = item['scriptPubKey']
 642                     account, sequence = self.get_address_index(item['address'])
 643                     if account != -1:
 644                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 645                         print_error("added redeemScript", txin['redeemScript'])
 646                     break
 647
 648
 649         # build a list of public/private keys
 650         keypairs = {}
 651
 652         # add private keys from parameter
 653         for sec in private_keys:
 654             pubkey = public_key_from_private_key(sec)
 655             keypairs[ pubkey ] = sec
 656
 657         # add private_keys from KeyID
 658         self.add_keypairs_from_KeyID(tx, keypairs, password)
 659
 660         # add private keys from wallet
 661         self.add_keypairs_from_wallet(tx, keypairs, password)
 662         self.sign_transaction(tx, keypairs)
 663
 664
 665     def sign_message(self, address, message, password):
 666         keys = self.get_private_key(address, password)
 667         assert len(keys) == 1
 668         sec = keys[0]
 669         key = regenerate_key(sec)
 670         compressed = is_compressed(sec)
 671         return key.sign_message(message, compressed, address)
 672
 673     def verify_message(self, address, signature, message):
 674         try:
 675             EC_KEY.verify_message(address, signature, message)
 676             return True
 677         except BaseException as e:
 678             print_error("Verification error: {0}".format(e))
 679             return False
 680
 681
 682     def change_gap_limit(self, value):
 683         if value >= self.gap_limit:
 684             self.gap_limit = value
 685             self.storage.put('gap_limit', self.gap_limit, True)
 686             #self.interface.poke('synchronizer')
 687             return True
 688
 689         elif value >= self.min_acceptable_gap():
 690             for key, account in self.accounts.items():
 691                 addresses = account[0]
 692                 k = self.num_unused_trailing_addresses(addresses)
 693                 n = len(addresses) - k + value
 694                 addresses = addresses[0:n]
 695                 self.accounts[key][0] = addresses
 696
 697             self.gap_limit = value
 698             self.storage.put('gap_limit', self.gap_limit, True)
 699             self.save_accounts()
 700             return True
 701         else:
 702             return False
 703
 704     def num_unused_trailing_addresses(self, addresses):
 705         k = 0
 706         for a in addresses[::-1]:
 707             if self.history.get(a):break
 708             k = k + 1
 709         return k
 710
 711     def min_acceptable_gap(self):
 712         # fixme: this assumes wallet is synchronized
 713         n = 0
 714         nmax = 0
 715
 716         for account in self.accounts.values():
 717             addresses = account.get_addresses(0)
 718             k = self.num_unused_trailing_addresses(addresses)
 719             for a in addresses[0:-k]:
 720                 if self.history.get(a):
 721                     n = 0
 722                 else:
 723                     n += 1
 724                     if n > nmax: nmax = n
 725         return nmax + 1
 726
 727
 728     def address_is_old(self, address):
 729         age = -1
 730         h = self.history.get(address, [])
 731         if h == ['*']:
 732             return True
 733         for tx_hash, tx_height in h:
 734             if tx_height == 0:
 735                 tx_age = 0
 736             else:
 737                 tx_age = self.verifier.blockchain.height - tx_height + 1
 738             if tx_age > age:
 739                 age = tx_age
 740         return age > 2
 741
 742
 743     def synchronize_sequence(self, account, for_change):
 744         limit = self.gap_limit_for_change if for_change else self.gap_limit
 745         new_addresses = []
 746         while True:
 747             addresses = account.get_addresses(for_change)
 748             if len(addresses) < limit:
 749                 address = account.create_new_address(for_change)
 750                 self.history[address] = []
 751                 new_addresses.append( address )
 752                 continue
 753
 754             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 755                 break
 756             else:
 757                 address = account.create_new_address(for_change)
 758                 self.history[address] = []
 759                 new_addresses.append( address )
 760
 761         return new_addresses
 762
 763
 764
 765     def create_pending_accounts(self):
 766         for account_type in ['1','2of2','2of3']:
 767             a = self.new_account_address(account_type)
 768             if self.address_is_old(a):
 769                 print_error( "creating account", a )
 770                 self.create_account(account_type)
 771
 772
 773     def synchronize_account(self, account):
 774         new = []
 775         new += self.synchronize_sequence(account, 0)
 776         new += self.synchronize_sequence(account, 1)
 777         return new
 778
 779
 780     def synchronize(self):
 781         if self.master_public_keys:
 782             self.create_pending_accounts()
 783         new = []
 784         for account in self.accounts.values():
 785             new += self.synchronize_account(account)
 786         if new:
 787             self.save_accounts()
 788             self.storage.put('addr_history', self.history, True)
 789         return new
 790
 791
 792     def is_found(self):
 793         return self.history.values() != [[]] * len(self.history)
 794
 795
 796     def add_contact(self, address, label=None):
 797         self.addressbook.append(address)
 798         self.storage.put('contacts', self.addressbook, True)
 799         if label:
 800             self.set_label(address, label)
 801
 802
 803     def delete_contact(self, addr):
 804         if addr in self.addressbook:
 805             self.addressbook.remove(addr)
 806             self.storage.put('addressbook', self.addressbook, True)
 807
 808
 809     def fill_addressbook(self):
 810         for tx_hash, tx in self.transactions.items():
 811             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 812             if is_send:
 813                 for addr, v in tx.outputs:
 814                     if not self.is_mine(addr) and addr not in self.addressbook:
 815                         self.addressbook.append(addr)
 816         # redo labels
 817         # self.update_tx_labels()
 818
 819     def get_num_tx(self, address):
 820         n = 0
 821         for tx in self.transactions.values():
 822             if address in map(lambda x:x[0], tx.outputs): n += 1
 823         return n
 824
 825
 826     def get_address_flags(self, addr):
 827         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 828         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 829         return flags
 830
 831
 832     def get_tx_value(self, tx, account=None):
 833         domain = self.get_account_addresses(account)
 834         return tx.get_value(domain, self.prevout_values)
 835
 836
 837     def update_tx_outputs(self, tx_hash):
 838         tx = self.transactions.get(tx_hash)
 839
 840         for i, (addr, value) in enumerate(tx.outputs):
 841             key = tx_hash+ ':%d'%i
 842             self.prevout_values[key] = value
 843
 844         for item in tx.inputs:
 845             if self.is_mine(item.get('address')):
 846                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 847                 self.spent_outputs.append(key)
 848
 849
 850     def get_addr_balance(self, address):
 851         assert self.is_mine(address)
 852         h = self.history.get(address,[])
 853         if h == ['*']: return 0,0
 854         c = u = 0
 855         received_coins = []   # list of coins received at address
 856
 857         for tx_hash, tx_height in h:
 858             tx = self.transactions.get(tx_hash)
 859             if not tx: continue
 860
 861             for i, (addr, value) in enumerate(tx.outputs):
 862                 if addr == address:
 863                     key = tx_hash + ':%d'%i
 864                     received_coins.append(key)
 865
 866         for tx_hash, tx_height in h:
 867             tx = self.transactions.get(tx_hash)
 868             if not tx: continue
 869             v = 0
 870
 871             for item in tx.inputs:
 872                 addr = item.get('address')
 873                 if addr == address:
 874                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 875                     value = self.prevout_values.get( key )
 876                     if key in received_coins:
 877                         v -= value
 878
 879             for i, (addr, value) in enumerate(tx.outputs):
 880                 key = tx_hash + ':%d'%i
 881                 if addr == address:
 882                     v += value
 883
 884             if tx_height:
 885                 c += v
 886             else:
 887                 u += v
 888         return c, u
 889
 890
 891     def get_account_name(self, k):
 892         if k == 0:
 893             if self.seed_version == 4:
 894                 name = 'Main account'
 895             else:
 896                 name = 'Old account'
 897         else:
 898             name = self.labels.get(k, 'Unnamed account')
 899         return name
 900
 901     def get_account_names(self):
 902         accounts = {}
 903         for k, account in self.accounts.items():
 904             accounts[k] = self.get_account_name(k)
 905         if self.imported_keys:
 906             accounts[-1] = 'Imported keys'
 907         return accounts
 908
 909     def get_account_addresses(self, a, include_change=True):
 910         if a is None:
 911             o = self.addresses(True)
 912         elif a == -1:
 913             o = self.imported_keys.keys()
 914         else:
 915             ac = self.accounts[a]
 916             o = ac.get_addresses(0)
 917             if include_change: o += ac.get_addresses(1)
 918         return o
 919
 920     def get_imported_balance(self):
 921         cc = uu = 0
 922         for addr in self.imported_keys.keys():
 923             c, u = self.get_addr_balance(addr)
 924             cc += c
 925             uu += u
 926         return cc, uu
 927
 928     def get_account_balance(self, account):
 929         if account is None:
 930             return self.get_balance()
 931         elif account == -1:
 932             return self.get_imported_balance()
 933
 934         conf = unconf = 0
 935         for addr in self.get_account_addresses(account):
 936             c, u = self.get_addr_balance(addr)
 937             conf += c
 938             unconf += u
 939         return conf, unconf
 940
 941     def get_frozen_balance(self):
 942         conf = unconf = 0
 943         for addr in self.frozen_addresses:
 944             c, u = self.get_addr_balance(addr)
 945             conf += c
 946             unconf += u
 947         return conf, unconf
 948
 949
 950     def get_balance(self):
 951         cc = uu = 0
 952         for a in self.accounts.keys():
 953             c, u = self.get_account_balance(a)
 954             cc += c
 955             uu += u
 956         c, u = self.get_imported_balance()
 957         cc += c
 958         uu += u
 959         return cc, uu
 960
 961
 962     def get_unspent_coins(self, domain=None):
 963         coins = []
 964         if domain is None: domain = self.addresses(True)
 965         for addr in domain:
 966             h = self.history.get(addr, [])
 967             if h == ['*']: continue
 968             for tx_hash, tx_height in h:
 969                 tx = self.transactions.get(tx_hash)
 970                 if tx is None: raise BaseException("Wallet not synchronized")
 971                 for output in tx.d.get('outputs'):
 972                     if output.get('address') != addr: continue
 973                     key = tx_hash + ":%d" % output.get('prevout_n')
 974                     if key in self.spent_outputs: continue
 975                     output['prevout_hash'] = tx_hash
 976                     output['height'] = tx_height
 977                     coins.append((tx_height, output))
 978
 979         # sort by age
 980         if coins:
 981             coins = sorted(coins)
 982             if coins[-1][0] != 0:
 983                 while coins[0][0] == 0:
 984                     coins = coins[1:] + [ coins[0] ]
 985         return [x[1] for x in coins]
 986
 987
 988
 989     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
 990         """ todo: minimize tx size """
 991         total = 0
 992         fee = self.fee if fixed_fee is None else fixed_fee
 993         if domain is None:
 994             domain = self.addresses()
 995
 996         for i in self.frozen_addresses:
 997             if i in domain: domain.remove(i)
 998
 999         prioritized = []
1000         for i in self.prioritized_addresses:
1001             if i in domain:
1002                 domain.remove(i)
1003                 prioritized.append(i)
1004
1005         coins = self.get_unspent_coins(domain)
1006         prioritized_coins = self.get_unspent_coins(prioritized)
1007
1008         inputs = []
1009         coins = prioritized_coins + coins
1010
1011         for item in coins:
1012             addr = item.get('address')
1013             v = item.get('value')
1014             total += v
1015             inputs.append(item)
1016             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1017             if total >= amount + fee: break
1018         else:
1019             inputs = []
1020
1021         return inputs, total, fee
1022
1023
1024     def set_fee(self, fee):
1025         if self.fee != fee:
1026             self.fee = fee
1027             self.storage.put('fee_per_kb', self.fee, True)
1028
1029     def estimated_fee(self, inputs):
1030         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1031         fee = self.fee * int(round(estimated_size/1024.))
1032         if fee == 0: fee = self.fee
1033         return fee
1034
1035
1036     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1037         "add change to a transaction"
1038         change_amount = total - ( amount + fee )
1039         if change_amount != 0:
1040             if not change_addr:
1041
1042                 # send change to one of the accounts involved in the tx
1043                 address = inputs[0].get('address')
1044                 account, _ = self.get_address_index(address)
1045
1046                 if not self.use_change or account == -1:
1047                     change_addr = inputs[-1]['address']
1048                 else:
1049                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1050
1051             # Insert the change output at a random position in the outputs
1052             posn = random.randint(0, len(outputs))
1053             outputs[posn:posn] = [( change_addr,  change_amount)]
1054         return outputs
1055
1056
1057     def get_history(self, address):
1058         with self.lock:
1059             return self.history.get(address)
1060
1061
1062     def get_status(self, h):
1063         if not h: return None
1064         if h == ['*']: return '*'
1065         status = ''
1066         for tx_hash, height in h:
1067             status += tx_hash + ':%d:' % height
1068         return hashlib.sha256( status ).digest().encode('hex')
1069
1070
1071     def receive_tx_callback(self, tx_hash, tx, tx_height):
1072
1073         with self.transaction_lock:
1074             self.add_extra_addresses(tx)
1075             if not self.check_new_tx(tx_hash, tx):
1076                 # may happen due to pruning
1077                 print_error("received transaction that is no longer referenced in history", tx_hash)
1078                 return
1079             self.transactions[tx_hash] = tx
1080             self.network.interface.pending_transactions_for_notifications.append(tx)
1081             self.save_transactions()
1082             if self.verifier and tx_height>0:
1083                 self.verifier.add(tx_hash, tx_height)
1084             self.update_tx_outputs(tx_hash)
1085
1086
1087     def save_transactions(self):
1088         tx = {}
1089         for k,v in self.transactions.items():
1090             tx[k] = str(v)
1091         self.storage.put('transactions', tx, True)
1092
1093     def receive_history_callback(self, addr, hist):
1094
1095         if not self.check_new_history(addr, hist):
1096             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1097
1098         with self.lock:
1099             self.history[addr] = hist
1100             self.storage.put('addr_history', self.history, True)
1101
1102         if hist != ['*']:
1103             for tx_hash, tx_height in hist:
1104                 if tx_height>0:
1105                     # add it in case it was previously unconfirmed
1106                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1107
1108
1109     def get_tx_history(self, account=None):
1110         with self.transaction_lock:
1111             history = self.transactions.items()
1112             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1113             result = []
1114
1115             balance = 0
1116             for tx_hash, tx in history:
1117                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1118                 if v is not None: balance += v
1119
1120             c, u = self.get_account_balance(account)
1121
1122             if balance != c+u:
1123                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1124
1125             balance = c + u - balance
1126             for tx_hash, tx in history:
1127                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1128                 if not is_relevant:
1129                     continue
1130                 if value is not None:
1131                     balance += value
1132
1133                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1134                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1135
1136         return result
1137
1138
1139     def get_label(self, tx_hash):
1140         label = self.labels.get(tx_hash)
1141         is_default = (label == '') or (label is None)
1142         if is_default: label = self.get_default_label(tx_hash)
1143         return label, is_default
1144
1145
1146     def get_default_label(self, tx_hash):
1147         tx = self.transactions.get(tx_hash)
1148         default_label = ''
1149         if tx:
1150             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1151             if is_mine:
1152                 for o in tx.outputs:
1153                     o_addr, _ = o
1154                     if not self.is_mine(o_addr):
1155                         try:
1156                             default_label = self.labels[o_addr]
1157                         except KeyError:
1158                             default_label = o_addr
1159                         break
1160                 else:
1161                     default_label = '(internal)'
1162             else:
1163                 for o in tx.outputs:
1164                     o_addr, _ = o
1165                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1166                         break
1167                 else:
1168                     for o in tx.outputs:
1169                         o_addr, _ = o
1170                         if self.is_mine(o_addr):
1171                             break
1172                     else:
1173                         o_addr = None
1174
1175                 if o_addr:
1176                     dest_label = self.labels.get(o_addr)
1177                     try:
1178                         default_label = self.labels[o_addr]
1179                     except KeyError:
1180                         default_label = o_addr
1181
1182         return default_label
1183
1184
1185     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1186         for address, x in outputs:
1187             assert is_valid(address)
1188         amount = sum( map(lambda x:x[1], outputs) )
1189         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1190         if not inputs:
1191             raise ValueError("Not enough funds")
1192         self.add_input_info(inputs)
1193         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1194         return Transaction.from_io(inputs, outputs)
1195
1196
1197     def mktx_from_account(self, outputs, password, fee=None, change_addr=None, account=None):
1198         domain = self.get_account_addresses(account) if account else None
1199         return self.mktx(outputs, password, fee, change_addr, domain)
1200
1201
1202     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1203         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1204         keypairs = {}
1205         self.add_keypairs_from_wallet(tx, keypairs, password)
1206         self.sign_transaction(tx, keypairs)
1207         return tx
1208
1209
1210     def add_input_info(self, inputs):
1211         for txin in inputs:
1212             address = txin['address']
1213             account, sequence = self.get_address_index(address)
1214             txin['KeyID'] = self.get_keyID(account, sequence)
1215             redeemScript = self.accounts[account].redeem_script(sequence)
1216             if redeemScript:
1217                 txin['redeemScript'] = redeemScript
1218             else:
1219                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1220
1221
1222     def sign_transaction(self, tx, keypairs):
1223         tx.sign(keypairs)
1224         run_hook('sign_transaction', tx)
1225
1226
1227     def sendtx(self, tx):
1228         # synchronous
1229         h = self.send_tx(tx)
1230         self.tx_event.wait()
1231         return self.receive_tx(h)
1232
1233     def send_tx(self, tx):
1234         # asynchronous
1235         self.tx_event.clear()
1236         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1237         return tx.hash()
1238
1239     def on_broadcast(self, i, r):
1240         self.tx_result = r.get('result')
1241         self.tx_event.set()
1242
1243     def receive_tx(self,tx_hash):
1244         out = self.tx_result
1245         if out != tx_hash:
1246             return False, "error: " + out
1247         return True, out
1248
1249
1250
1251     def update_password(self, seed, old_password, new_password):
1252         if new_password == '': new_password = None
1253         # this will throw an exception if unicode cannot be converted
1254         self.seed = pw_encode( seed, new_password)
1255         self.storage.put('seed', self.seed, True)
1256         self.use_encryption = (new_password != None)
1257         self.storage.put('use_encryption', self.use_encryption,True)
1258         for k in self.imported_keys.keys():
1259             a = self.imported_keys[k]
1260             b = pw_decode(a, old_password)
1261             c = pw_encode(b, new_password)
1262             self.imported_keys[k] = c
1263         self.storage.put('imported_keys', self.imported_keys, True)
1264
1265         for k, v in self.master_private_keys.items():
1266             b = pw_decode(v, old_password)
1267             c = pw_encode(b, new_password)
1268             self.master_private_keys[k] = c
1269         self.storage.put('master_private_keys', self.master_private_keys, True)
1270
1271
1272     def freeze(self,addr):
1273         if self.is_mine(addr) and addr not in self.frozen_addresses:
1274             self.unprioritize(addr)
1275             self.frozen_addresses.append(addr)
1276             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1277             return True
1278         else:
1279             return False
1280
1281     def unfreeze(self,addr):
1282         if self.is_mine(addr) and addr in self.frozen_addresses:
1283             self.frozen_addresses.remove(addr)
1284             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1285             return True
1286         else:
1287             return False
1288
1289     def prioritize(self,addr):
1290         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1291             self.unfreeze(addr)
1292             self.prioritized_addresses.append(addr)
1293             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1294             return True
1295         else:
1296             return False
1297
1298     def unprioritize(self,addr):
1299         if self.is_mine(addr) and addr in self.prioritized_addresses:
1300             self.prioritized_addresses.remove(addr)
1301             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1302             return True
1303         else:
1304             return False
1305
1306
1307     def set_verifier(self, verifier):
1308         self.verifier = verifier
1309
1310         # review transactions that are in the history
1311         for addr, hist in self.history.items():
1312             if hist == ['*']: continue
1313             for tx_hash, tx_height in hist:
1314                 if tx_height>0:
1315                     # add it in case it was previously unconfirmed
1316                     self.verifier.add(tx_hash, tx_height)
1317
1318
1319         # if we are on a pruning server, remove unverified transactions
1320         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1321         for tx_hash in self.transactions.keys():
1322             if tx_hash not in vr:
1323                 self.transactions.pop(tx_hash)
1324
1325
1326
1327     def check_new_history(self, addr, hist):
1328
1329         # check that all tx in hist are relevant
1330         if hist != ['*']:
1331             for tx_hash, height in hist:
1332                 tx = self.transactions.get(tx_hash)
1333                 if not tx: continue
1334                 if not tx.has_address(addr):
1335                     return False
1336
1337         # check that we are not "orphaning" a transaction
1338         old_hist = self.history.get(addr,[])
1339         if old_hist == ['*']: return True
1340
1341         for tx_hash, height in old_hist:
1342             if tx_hash in map(lambda x:x[0], hist): continue
1343             found = False
1344             for _addr, _hist in self.history.items():
1345                 if _addr == addr: continue
1346                 if _hist == ['*']: continue
1347                 _tx_hist = map(lambda x:x[0], _hist)
1348                 if tx_hash in _tx_hist:
1349                     found = True
1350                     break
1351
1352             if not found:
1353                 tx = self.transactions.get(tx_hash)
1354                 # tx might not be there
1355                 if not tx: continue
1356
1357                 # already verified?
1358                 if self.verifier.get_height(tx_hash):
1359                     continue
1360                 # unconfirmed tx
1361                 print_error("new history is orphaning transaction:", tx_hash)
1362                 # check that all outputs are not mine, request histories
1363                 ext_requests = []
1364                 for _addr, _v in tx.outputs:
1365                     # assert not self.is_mine(_addr)
1366                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1367
1368                 ext_h = self.network.interface.synchronous_get(ext_requests)
1369                 print_error("sync:", ext_requests, ext_h)
1370                 height = None
1371                 for h in ext_h:
1372                     if h == ['*']: continue
1373                     for item in h:
1374                         if item.get('tx_hash') == tx_hash:
1375                             height = item.get('height')
1376                 if height:
1377                     print_error("found height for", tx_hash, height)
1378                     self.verifier.add(tx_hash, height)
1379                 else:
1380                     print_error("removing orphaned tx from history", tx_hash)
1381                     self.transactions.pop(tx_hash)
1382
1383         return True
1384
1385
1386
1387     def check_new_tx(self, tx_hash, tx):
1388         # 1 check that tx is referenced in addr_history.
1389         addresses = []
1390         for addr, hist in self.history.items():
1391             if hist == ['*']:continue
1392             for txh, height in hist:
1393                 if txh == tx_hash:
1394                     addresses.append(addr)
1395
1396         if not addresses:
1397             return False
1398
1399         # 2 check that referencing addresses are in the tx
1400         for addr in addresses:
1401             if not tx.has_address(addr):
1402                 return False
1403
1404         return True
1405
1406
1407     def start_threads(self, network):
1408         from verifier import TxVerifier
1409         self.network = network
1410         self.verifier = TxVerifier(self.network, self.storage)
1411         self.verifier.start()
1412         self.set_verifier(self.verifier)
1413         self.synchronizer = WalletSynchronizer(self)
1414         self.synchronizer.start()
1415
1416     def stop_threads(self):
1417         self.verifier.stop()
1418         self.synchronizer.stop()
1419
1420
1421
1422     def restore(self, callback):
1423         from i18n import _
1424         def wait_for_wallet():
1425             self.set_up_to_date(False)
1426             while not self.is_up_to_date():
1427                 msg = "%s\n%s %d\n%s %.1f"%(
1428                     _("Please wait..."),
1429                     _("Addresses generated:"),
1430                     len(self.addresses(True)),_("Kilobytes received:"),
1431                     self.network.interface.bytes_received/1024.)
1432
1433                 apply(callback, (msg,))
1434                 time.sleep(0.1)
1435
1436         def wait_for_network():
1437             while not self.network.interface.is_connected:
1438                 msg = "%s \n" % (_("Connecting..."))
1439                 apply(callback, (msg,))
1440                 time.sleep(0.1)
1441
1442         # wait until we are connected, because the user might have selected another server
1443         wait_for_network()
1444
1445         # try to restore old account
1446         self.create_old_account()
1447         wait_for_wallet()
1448
1449         if self.is_found():
1450             self.seed_version = 4
1451             self.storage.put('seed_version', wallet.seed_version, True)
1452         else:
1453             self.accounts.pop(0)
1454             self.create_accounts()
1455             wait_for_wallet()
1456
1457
1458
1459
1460 class WalletSynchronizer(threading.Thread):
1461
1462
1463     def __init__(self, wallet):
1464         threading.Thread.__init__(self)
1465         self.daemon = True
1466         self.wallet = wallet
1467         wallet.synchronizer = self
1468         self.network = self.wallet.network
1469         #self.wallet.network.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1470         self.was_updated = True
1471         self.running = False
1472         self.lock = threading.Lock()
1473         self.queue = Queue.Queue()
1474
1475     def stop(self):
1476         with self.lock: self.running = False
1477
1478     def is_running(self):
1479         with self.lock: return self.running
1480
1481
1482     def subscribe_to_addresses(self, addresses):
1483         messages = []
1484         for addr in addresses:
1485             messages.append(('blockchain.address.subscribe', [addr]))
1486         self.network.interface.send( messages, lambda i,r: self.queue.put(r))
1487
1488
1489     def run(self):
1490         with self.lock:
1491             self.running = True
1492
1493         while self.is_running():
1494             interface = self.network.interface
1495             if not interface.is_connected:
1496                 print_error("synchronizer: waiting for interface")
1497                 interface.connect_event.wait()
1498
1499             self.run_interface(interface)
1500
1501
1502     def run_interface(self, interface):
1503
1504         print_error("synchronizer: connected to", interface.server)
1505
1506         requested_tx = []
1507         missing_tx = []
1508         requested_histories = {}
1509
1510         # request any missing transactions
1511         for history in self.wallet.history.values():
1512             if history == ['*']: continue
1513             for tx_hash, tx_height in history:
1514                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1515                     missing_tx.append( (tx_hash, tx_height) )
1516
1517         if missing_tx:
1518             print_error("missing tx", missing_tx)
1519
1520         # subscriptions
1521         self.subscribe_to_addresses(self.wallet.addresses(True, next=True))
1522
1523         while self.is_running():
1524             # 1. create new addresses
1525             new_addresses = self.wallet.synchronize()
1526
1527             # request missing addresses
1528             if new_addresses:
1529                 self.subscribe_to_addresses(new_addresses)
1530
1531             # request missing transactions
1532             for tx_hash, tx_height in missing_tx:
1533                 if (tx_hash, tx_height) not in requested_tx:
1534                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1535                     requested_tx.append( (tx_hash, tx_height) )
1536             missing_tx = []
1537
1538             # detect if situation has changed
1539             if interface.is_up_to_date() and self.queue.empty():
1540                 if not self.wallet.is_up_to_date():
1541                     self.wallet.set_up_to_date(True)
1542                     self.was_updated = True
1543             else:
1544                 if self.wallet.is_up_to_date():
1545                     self.wallet.set_up_to_date(False)
1546                     self.was_updated = True
1547
1548             if self.was_updated:
1549                 self.wallet.network.trigger_callback('updated')
1550                 self.was_updated = False
1551
1552             # 2. get a response
1553             try:
1554                 r = self.queue.get(block=True, timeout=1)
1555             except Queue.Empty:
1556                 continue
1557
1558             if interface != self.network.interface:
1559                 break
1560
1561             if not r:
1562                 continue
1563
1564             # 3. handle response
1565             method = r['method']
1566             params = r['params']
1567             result = r.get('result')
1568             error = r.get('error')
1569             if error:
1570                 print "error", r
1571                 continue
1572
1573             if method == 'blockchain.address.subscribe':
1574                 addr = params[0]
1575                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1576                     if requested_histories.get(addr) is None:
1577                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1578                         requested_histories[addr] = result
1579
1580             elif method == 'blockchain.address.get_history':
1581                 addr = params[0]
1582                 print_error("receiving history", addr, result)
1583                 if result == ['*']:
1584                     assert requested_histories.pop(addr) == '*'
1585                     self.wallet.receive_history_callback(addr, result)
1586                 else:
1587                     hist = []
1588                     # check that txids are unique
1589                     txids = []
1590                     for item in result:
1591                         tx_hash = item['tx_hash']
1592                         if tx_hash not in txids:
1593                             txids.append(tx_hash)
1594                             hist.append( (tx_hash, item['height']) )
1595
1596                     if len(hist) != len(result):
1597                         raise BaseException("error: server sent history with non-unique txid", result)
1598
1599                     # check that the status corresponds to what was announced
1600                     rs = requested_histories.pop(addr)
1601                     if self.wallet.get_status(hist) != rs:
1602                         raise BaseException("error: status mismatch: %s"%addr)
1603
1604                     # store received history
1605                     self.wallet.receive_history_callback(addr, hist)
1606
1607                     # request transactions that we don't have
1608                     for tx_hash, tx_height in hist:
1609                         if self.wallet.transactions.get(tx_hash) is None:
1610                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1611                                 missing_tx.append( (tx_hash, tx_height) )
1612
1613             elif method == 'blockchain.transaction.get':
1614                 tx_hash = params[0]
1615                 tx_height = params[1]
1616                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1617                 tx = Transaction(result)
1618                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1619                 self.was_updated = True
1620                 requested_tx.remove( (tx_hash, tx_height) )
1621                 print_error("received tx:", tx_hash, len(tx.raw))
1622
1623             else:
1624                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1625
1626             if self.was_updated and not requested_tx:
1627                 self.wallet.network.trigger_callback('updated')
1628                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1629
1630                 self.was_updated = False