lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 # AES encryption
  40 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  41 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  42
  43 def pw_encode(s, password):
  44     if password:
  45         secret = Hash(password)
  46         return EncodeAES(secret, s)
  47     else:
  48         return s
  49
  50 def pw_decode(s, password):
  51     if password is not None:
  52         secret = Hash(password)
  53         try:
  54             d = DecodeAES(secret, s)
  55         except:
  56             raise BaseException('Invalid password')
  57         return d
  58     else:
  59         return s
  60
  61
  62
  63
  64
  65 from version import ELECTRUM_VERSION, SEED_VERSION
  66
  67
  68 class WalletStorage:
  69
  70     def __init__(self, config):
  71         self.data = {}
  72         self.file_exists = False
  73         self.init_path(config)
  74         print_error( "wallet path", self.path )
  75         if self.path:
  76             self.read(self.path)
  77
  78
  79     def init_path(self, config):
  80         """Set the path of the wallet."""
  81
  82         path = config.get('wallet_path')
  83         if not path:
  84             path = config.get('default_wallet_path')
  85         if path is not None:
  86             self.path = path
  87             return
  88
  89         self.path = os.path.join(config.path, "electrum.dat")
  90
  91
  92     def read(self, path):
  93         """Read the contents of the wallet file."""
  94         try:
  95             with open(self.path, "r") as f:
  96                 data = f.read()
  97         except IOError:
  98             return
  99         try:
 100             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 101         except:
 102             raise IOError("Cannot read wallet file.")
 103
 104         self.data = d
 105         self.file_exists = True
 106
 107
 108     def get(self, key, default=None):
 109         return self.data.get(key, default)
 110
 111     def put(self, key, value, save = True):
 112
 113         if self.data.get(key) is not None:
 114             self.data[key] = value
 115         else:
 116             # add key to wallet config
 117             self.data[key] = value
 118
 119         if save:
 120             self.write()
 121
 122
 123     def write(self):
 124         s = repr(self.data)
 125         f = open(self.path,"w")
 126         f.write( s )
 127         f.close()
 128         if self.get('gui') != 'android':
 129             import stat
 130             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 131
 132
 133 class Wallet:
 134
 135     def __init__(self, storage):
 136
 137         self.storage = storage
 138         self.electrum_version = ELECTRUM_VERSION
 139         self.gap_limit_for_change = 3 # constant
 140
 141         # saved fields
 142         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 143
 144         self.gap_limit             = storage.get('gap_limit', 5)
 145         self.use_change            = storage.get('use_change',True)
 146         self.use_encryption        = storage.get('use_encryption', False)
 147         self.seed                  = storage.get('seed', '')               # encrypted
 148         self.labels                = storage.get('labels', {})
 149         self.frozen_addresses      = storage.get('frozen_addresses',[])
 150         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 151         self.addressbook           = storage.get('contacts', [])
 152
 153         self.imported_keys         = storage.get('imported_keys',{})
 154         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 155
 156         self.fee                   = int(storage.get('fee_per_kb',20000))
 157
 158         self.master_public_keys = storage.get('master_public_keys',{})
 159         self.master_private_keys = storage.get('master_private_keys', {})
 160
 161         self.next_addresses = storage.get('next_addresses',{})
 162
 163         if self.seed_version < 4:
 164             raise ValueError("This wallet seed is deprecated.")
 165
 166         self.load_accounts()
 167
 168         self.transactions = {}
 169         tx_list = self.storage.get('transactions',{})
 170         for k,v in tx_list.items():
 171             try:
 172                 tx = Transaction(v)
 173             except:
 174                 print_msg("Warning: Cannot deserialize transactions. skipping")
 175                 continue
 176
 177             self.add_extra_addresses(tx)
 178             self.transactions[k] = tx
 179
 180         for h,tx in self.transactions.items():
 181             if not self.check_new_tx(h, tx):
 182                 print_error("removing unreferenced tx", h)
 183                 self.transactions.pop(h)
 184
 185
 186         # not saved
 187         self.prevout_values = {}     # my own transaction outputs
 188         self.spent_outputs = []
 189
 190         # spv
 191         self.verifier = None
 192
 193         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 194         # interface.is_up_to_date() returns true when all requests have been answered and processed
 195         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 196
 197         self.up_to_date = False
 198         self.lock = threading.Lock()
 199         self.transaction_lock = threading.Lock()
 200         self.tx_event = threading.Event()
 201
 202         for tx_hash, tx in self.transactions.items():
 203             self.update_tx_outputs(tx_hash)
 204
 205
 206     def add_extra_addresses(self, tx):
 207         h = tx.hash()
 208         # find the address corresponding to pay-to-pubkey inputs
 209         tx.add_extra_addresses(self.transactions)
 210         for o in tx.d.get('outputs'):
 211             if o.get('is_pubkey'):
 212                 for tx2 in self.transactions.values():
 213                     tx2.add_extra_addresses({h:tx})
 214
 215
 216
 217
 218     def set_up_to_date(self,b):
 219         with self.lock: self.up_to_date = b
 220
 221
 222     def is_up_to_date(self):
 223         with self.lock: return self.up_to_date
 224
 225
 226     def update(self):
 227         self.up_to_date = False
 228         while not self.is_up_to_date():
 229             time.sleep(0.1)
 230
 231
 232     def import_key(self, sec, password):
 233         # check password
 234         seed = self.decode_seed(password)
 235         try:
 236             address = address_from_private_key(sec)
 237         except:
 238             raise BaseException('Invalid private key')
 239
 240         if self.is_mine(address):
 241             raise BaseException('Address already in wallet')
 242
 243         # store the originally requested keypair into the imported keys table
 244         self.imported_keys[address] = pw_encode(sec, password )
 245         self.storage.put('imported_keys', self.imported_keys, True)
 246         if self.synchronizer:
 247             self.synchronizer.subscribe_to_addresses([address])
 248         return address
 249
 250     def delete_imported_key(self, addr):
 251         if addr in self.imported_keys:
 252             self.imported_keys.pop(addr)
 253             self.storage.put('imported_keys', self.imported_keys, True)
 254
 255
 256     def init_seed(self, seed):
 257         if self.seed: raise BaseException("a seed exists")
 258         if not seed:
 259             seed = random_seed(128)
 260         self.seed = seed
 261
 262
 263     def save_seed(self):
 264         self.storage.put('seed', self.seed, True)
 265         self.storage.put('seed_version', self.seed_version, True)
 266
 267     def create_watching_only_wallet(self, c0, K0):
 268         cK0 = ""
 269         self.master_public_keys = {
 270             "m/0'/": (c0, K0, cK0),
 271             }
 272         self.storage.put('master_public_keys', self.master_public_keys, True)
 273         self.create_account('1','Main account')
 274
 275     def create_accounts(self):
 276
 277         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 278
 279         # normal accounts
 280         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 281         # p2sh 2of2
 282         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 283         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 284         # p2sh 2of3
 285         k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 286         k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 287         k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 288
 289         self.master_public_keys = {
 290             "m/0'/": (c0, K0, cK0),
 291             "m/1'/": (c1, K1, cK1),
 292             "m/2'/": (c2, K2, cK2),
 293             "m/3'/": (c3, K3, cK3),
 294             "m/4'/": (c4, K4, cK4),
 295             "m/5'/": (c5, K5, cK5)
 296             }
 297
 298         self.master_private_keys = {
 299             "m/0'/": k0,
 300             "m/1'/": k1,
 301             "m/2'/": k2,
 302             "m/3'/": k3,
 303             "m/4'/": k4,
 304             "m/5'/": k5
 305             }
 306
 307         self.storage.put('master_public_keys', self.master_public_keys, True)
 308         self.storage.put('master_private_keys', self.master_private_keys, True)
 309
 310         # create default account
 311         self.create_account('1','Main account')
 312
 313
 314     def find_root_by_master_key(self, c, K):
 315         for key, v in self.master_public_keys.items():
 316             if key == "m/":continue
 317             cc, KK, _ = v
 318             if (c == cc) and (K == KK):
 319                 return key
 320
 321     def deseed_root(self, seed, password):
 322         # for safety, we ask the user to enter their seed
 323         assert seed == self.decode_seed(password)
 324         self.seed = ''
 325         self.storage.put('seed', '', True)
 326
 327
 328     def deseed_branch(self, k):
 329         # check that parent has no seed
 330         assert self.seed == ''
 331         self.master_private_keys.pop(k)
 332         self.storage.put('master_private_keys', self.master_private_keys, True)
 333
 334
 335     def account_id(self, account_type, i):
 336         if account_type == '1':
 337             return "m/0'/%d"%i
 338         elif account_type == '2of2':
 339             return "m/1'/%d & m/2'/%d"%(i,i)
 340         elif account_type == '2of3':
 341             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 342         else:
 343             raise BaseException('unknown account type')
 344
 345
 346     def num_accounts(self, account_type):
 347         keys = self.accounts.keys()
 348         i = 0
 349         while True:
 350             account_id = self.account_id(account_type, i)
 351             if account_id not in keys: break
 352             i += 1
 353         return i
 354
 355
 356     def new_account_address(self, account_type = '1'):
 357         i = self.num_accounts(account_type)
 358         k = self.account_id(account_type,i)
 359
 360         addr = self.next_addresses.get(k)
 361         if not addr:
 362             account_id, account = self.next_account(account_type)
 363             addr = account.first_address()
 364             self.next_addresses[k] = addr
 365             self.storage.put('next_addresses',self.next_addresses)
 366
 367         return addr
 368
 369
 370     def next_account(self, account_type = '1'):
 371
 372         i = self.num_accounts(account_type)
 373         account_id = self.account_id(account_type,i)
 374
 375         if account_type is '1':
 376             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 377             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 378             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 379
 380         elif account_type == '2of2':
 381             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 382             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 383             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 384             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 385             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 386
 387         elif account_type == '2of3':
 388             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 389             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 390             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 391             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 392             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 393             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 394             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 395
 396         return account_id, account
 397
 398
 399     def set_label(self, name, text = None):
 400         changed = False
 401         old_text = self.labels.get(name)
 402         if text:
 403             if old_text != text:
 404                 self.labels[name] = text
 405                 changed = True
 406         else:
 407             if old_text:
 408                 self.labels.pop(name)
 409                 changed = True
 410
 411         if changed:
 412             self.storage.put('labels', self.labels, True)
 413
 414         run_hook('set_label', name, text, changed)
 415         return changed
 416
 417
 418
 419     def create_account(self, account_type = '1', name = None):
 420         account_id, account = self.next_account(account_type)
 421         self.accounts[account_id] = account
 422         self.save_accounts()
 423         if name:
 424             self.set_label(account_id, name)
 425
 426
 427     def create_old_account(self):
 428         mpk = OldAccount.mpk_from_seed(self.seed)
 429         self.storage.put('master_public_key', mpk, True)
 430         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 431         self.save_accounts()
 432
 433
 434     def save_accounts(self):
 435         d = {}
 436         for k, v in self.accounts.items():
 437             d[k] = v.dump()
 438         self.storage.put('accounts', d, True)
 439
 440
 441
 442     def load_accounts(self):
 443         d = self.storage.get('accounts', {})
 444         self.accounts = {}
 445         for k, v in d.items():
 446             if k == 0:
 447                 v['mpk'] = self.storage.get('master_public_key')
 448                 self.accounts[k] = OldAccount(v)
 449             elif '&' in k:
 450                 self.accounts[k] = BIP32_Account_2of2(v)
 451             else:
 452                 self.accounts[k] = BIP32_Account(v)
 453
 454
 455     def addresses(self, include_change = True, next=False):
 456         o = self.get_account_addresses(-1, include_change)
 457         for a in self.accounts.keys():
 458             o += self.get_account_addresses(a, include_change)
 459
 460         if next:
 461             for addr in self.next_addresses.values():
 462                 if addr not in o:
 463                     o += [addr]
 464         return o
 465
 466
 467     def is_mine(self, address):
 468         return address in self.addresses(True)
 469
 470
 471     def is_change(self, address):
 472         if not self.is_mine(address): return False
 473         if address in self.imported_keys.keys(): return False
 474         acct, s = self.get_address_index(address)
 475         if s is None: return False
 476         return s[0] == 1
 477
 478     def get_master_public_key(self):
 479         if self.seed_version == 4:
 480             return self.storage.get("master_public_key")
 481         else:
 482             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 483             return repr((c, K))
 484
 485     def get_master_private_key(self, account, password):
 486         master_k = pw_decode( self.master_private_keys[account], password)
 487         master_c, master_K, master_Kc = self.master_public_keys[account]
 488         try:
 489             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 490             assert K.encode('hex') == master_K
 491         except:
 492             raise BaseException("Invalid password")
 493         return master_k
 494
 495
 496     def get_address_index(self, address):
 497         if address in self.imported_keys.keys():
 498             return -1, None
 499
 500         for account in self.accounts.keys():
 501             for for_change in [0,1]:
 502                 addresses = self.accounts[account].get_addresses(for_change)
 503                 for addr in addresses:
 504                     if address == addr:
 505                         return account, (for_change, addresses.index(addr))
 506
 507         raise BaseException("Address not found", address)
 508
 509
 510     def get_roots(self, account):
 511         roots = []
 512         for a in account.split('&'):
 513             s = a.strip()
 514             m = re.match("(m/\d+'/)(\d+)", s)
 515             roots.append( m.group(1) )
 516         return roots
 517
 518     def is_seeded(self, account):
 519         if type(account) is int:
 520             return self.seed is not None
 521
 522         for root in self.get_roots(account):
 523             if root not in self.master_private_keys.keys():
 524                 return False
 525         return True
 526
 527     def rebase_sequence(self, account, sequence):
 528         c, i = sequence
 529         dd = []
 530         for a in account.split('&'):
 531             s = a.strip()
 532             m = re.match("(m/\d+'/)(\d+)", s)
 533             root = m.group(1)
 534             num = int(m.group(2))
 535             dd.append( (root, [num,c,i] ) )
 536         return dd
 537
 538
 539     def get_keyID(self, account, sequence):
 540         if account == 0:
 541             return 'old'
 542
 543         rs = self.rebase_sequence(account, sequence)
 544         dd = []
 545         for root, public_sequence in rs:
 546             c, K, _ = self.master_public_keys[root]
 547             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 548             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 549         return '&'.join(dd)
 550
 551
 552     def get_public_key(self, address):
 553         account, sequence = self.get_address_index(address)
 554         return self.accounts[account].get_pubkey( *sequence )
 555
 556
 557     def decode_seed(self, password):
 558         seed = pw_decode(self.seed, password)
 559         #todo:  #self.sequences[0].check_seed(seed)
 560         return seed
 561
 562
 563     def get_private_key(self, address, password):
 564         out = []
 565         if address in self.imported_keys.keys():
 566             out.append( pw_decode( self.imported_keys[address], password ) )
 567         else:
 568             account, sequence = self.get_address_index(address)
 569             if account == 0:
 570                 seed = self.decode_seed(password)
 571                 pk = self.accounts[account].get_private_key(seed, sequence)
 572                 out.append(pk)
 573                 return out
 574
 575             # assert address == self.accounts[account].get_address(*sequence)
 576             rs = self.rebase_sequence( account, sequence)
 577             for root, public_sequence in rs:
 578
 579                 if root not in self.master_private_keys.keys(): continue
 580                 master_k = self.get_master_private_key(root, password)
 581                 master_c, _, _ = self.master_public_keys[root]
 582                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 583                 out.append(pk)
 584
 585         return out
 586
 587
 588     def add_keypairs_from_wallet(self, tx, keypairs, password):
 589         for txin in tx.inputs:
 590             address = txin['address']
 591             private_keys = self.get_private_key(address, password)
 592             for sec in private_keys:
 593                 pubkey = public_key_from_private_key(sec)
 594                 keypairs[ pubkey ] = sec
 595
 596
 597     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 598         for txin in tx.inputs:
 599             keyid = txin.get('KeyID')
 600             if keyid:
 601                 roots = []
 602                 for s in keyid.split('&'):
 603                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 604                     if not m: continue
 605                     c = m.group(1)
 606                     K = m.group(2)
 607                     sequence = m.group(3)
 608                     root = self.find_root_by_master_key(c,K)
 609                     if not root: continue
 610                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 611                     root = root + '%d'%sequence[0]
 612                     sequence = sequence[1:]
 613                     roots.append((root,sequence))
 614
 615                 account_id = " & ".join( map(lambda x:x[0], roots) )
 616                 account = self.accounts.get(account_id)
 617                 if not account: continue
 618                 addr = account.get_address(*sequence)
 619                 txin['address'] = addr # fixme: side effect
 620                 pk = self.get_private_key(addr, password)
 621                 for sec in pk:
 622                     pubkey = public_key_from_private_key(sec)
 623                     keypairs[pubkey] = sec
 624
 625
 626
 627     def signrawtransaction(self, tx, input_info, private_keys, password):
 628
 629         # check that the password is correct
 630         seed = self.decode_seed(password)
 631
 632         # add input info
 633         tx.add_input_info(input_info)
 634
 635         # add redeem script for coins that are in the wallet
 636         # FIXME: add redeemPubkey too!
 637         unspent_coins = self.get_unspent_coins()
 638         for txin in tx.inputs:
 639             for item in unspent_coins:
 640                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 641                     print_error( "tx input is in unspent coins" )
 642                     txin['scriptPubKey'] = item['scriptPubKey']
 643                     account, sequence = self.get_address_index(item['address'])
 644                     if account != -1:
 645                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 646                         print_error("added redeemScript", txin['redeemScript'])
 647                     break
 648
 649
 650         # build a list of public/private keys
 651         keypairs = {}
 652
 653         # add private keys from parameter
 654         for sec in private_keys:
 655             pubkey = public_key_from_private_key(sec)
 656             keypairs[ pubkey ] = sec
 657
 658         # add private_keys from KeyID
 659         self.add_keypairs_from_KeyID(tx, keypairs, password)
 660
 661         # add private keys from wallet
 662         self.add_keypairs_from_wallet(tx, keypairs, password)
 663         self.sign_transaction(tx, keypairs)
 664
 665
 666     def sign_message(self, address, message, password):
 667         keys = self.get_private_key(address, password)
 668         assert len(keys) == 1
 669         sec = keys[0]
 670         key = regenerate_key(sec)
 671         compressed = is_compressed(sec)
 672         return key.sign_message(message, compressed, address)
 673
 674     def verify_message(self, address, signature, message):
 675         try:
 676             EC_KEY.verify_message(address, signature, message)
 677             return True
 678         except BaseException as e:
 679             print_error("Verification error: {0}".format(e))
 680             return False
 681
 682
 683     def change_gap_limit(self, value):
 684         if value >= self.gap_limit:
 685             self.gap_limit = value
 686             self.storage.put('gap_limit', self.gap_limit, True)
 687             #self.interface.poke('synchronizer')
 688             return True
 689
 690         elif value >= self.min_acceptable_gap():
 691             for key, account in self.accounts.items():
 692                 addresses = account[0]
 693                 k = self.num_unused_trailing_addresses(addresses)
 694                 n = len(addresses) - k + value
 695                 addresses = addresses[0:n]
 696                 self.accounts[key][0] = addresses
 697
 698             self.gap_limit = value
 699             self.storage.put('gap_limit', self.gap_limit, True)
 700             self.save_accounts()
 701             return True
 702         else:
 703             return False
 704
 705     def num_unused_trailing_addresses(self, addresses):
 706         k = 0
 707         for a in addresses[::-1]:
 708             if self.history.get(a):break
 709             k = k + 1
 710         return k
 711
 712     def min_acceptable_gap(self):
 713         # fixme: this assumes wallet is synchronized
 714         n = 0
 715         nmax = 0
 716
 717         for account in self.accounts.values():
 718             addresses = account.get_addresses(0)
 719             k = self.num_unused_trailing_addresses(addresses)
 720             for a in addresses[0:-k]:
 721                 if self.history.get(a):
 722                     n = 0
 723                 else:
 724                     n += 1
 725                     if n > nmax: nmax = n
 726         return nmax + 1
 727
 728
 729     def address_is_old(self, address):
 730         age = -1
 731         h = self.history.get(address, [])
 732         if h == ['*']:
 733             return True
 734         for tx_hash, tx_height in h:
 735             if tx_height == 0:
 736                 tx_age = 0
 737             else:
 738                 tx_age = self.verifier.blockchain.height - tx_height + 1
 739             if tx_age > age:
 740                 age = tx_age
 741         return age > 2
 742
 743
 744     def synchronize_sequence(self, account, for_change):
 745         limit = self.gap_limit_for_change if for_change else self.gap_limit
 746         new_addresses = []
 747         while True:
 748             addresses = account.get_addresses(for_change)
 749             if len(addresses) < limit:
 750                 address = account.create_new_address(for_change)
 751                 self.history[address] = []
 752                 new_addresses.append( address )
 753                 continue
 754
 755             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 756                 break
 757             else:
 758                 address = account.create_new_address(for_change)
 759                 self.history[address] = []
 760                 new_addresses.append( address )
 761
 762         return new_addresses
 763
 764
 765
 766     def create_pending_accounts(self):
 767         for account_type in ['1','2of2','2of3']:
 768             a = self.new_account_address(account_type)
 769             if self.address_is_old(a):
 770                 print_error( "creating account", a )
 771                 self.create_account(account_type)
 772
 773
 774     def synchronize_account(self, account):
 775         new = []
 776         new += self.synchronize_sequence(account, 0)
 777         new += self.synchronize_sequence(account, 1)
 778         return new
 779
 780
 781     def synchronize(self):
 782         if self.master_public_keys:
 783             self.create_pending_accounts()
 784         new = []
 785         for account in self.accounts.values():
 786             new += self.synchronize_account(account)
 787         if new:
 788             self.save_accounts()
 789             self.storage.put('addr_history', self.history, True)
 790         return new
 791
 792
 793     def is_found(self):
 794         return self.history.values() != [[]] * len(self.history)
 795
 796
 797     def add_contact(self, address, label=None):
 798         self.addressbook.append(address)
 799         self.storage.put('contacts', self.addressbook, True)
 800         if label:
 801             self.set_label(address, label)
 802
 803
 804     def delete_contact(self, addr):
 805         if addr in self.addressbook:
 806             self.addressbook.remove(addr)
 807             self.storage.put('addressbook', self.addressbook, True)
 808
 809
 810     def fill_addressbook(self):
 811         for tx_hash, tx in self.transactions.items():
 812             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 813             if is_send:
 814                 for addr, v in tx.outputs:
 815                     if not self.is_mine(addr) and addr not in self.addressbook:
 816                         self.addressbook.append(addr)
 817         # redo labels
 818         # self.update_tx_labels()
 819
 820     def get_num_tx(self, address):
 821         n = 0
 822         for tx in self.transactions.values():
 823             if address in map(lambda x:x[0], tx.outputs): n += 1
 824         return n
 825
 826
 827     def get_address_flags(self, addr):
 828         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 829         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 830         return flags
 831
 832
 833     def get_tx_value(self, tx, account=None):
 834         domain = self.get_account_addresses(account)
 835         return tx.get_value(domain, self.prevout_values)
 836
 837
 838     def update_tx_outputs(self, tx_hash):
 839         tx = self.transactions.get(tx_hash)
 840
 841         for i, (addr, value) in enumerate(tx.outputs):
 842             key = tx_hash+ ':%d'%i
 843             self.prevout_values[key] = value
 844
 845         for item in tx.inputs:
 846             if self.is_mine(item.get('address')):
 847                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 848                 self.spent_outputs.append(key)
 849
 850
 851     def get_addr_balance(self, address):
 852         assert self.is_mine(address)
 853         h = self.history.get(address,[])
 854         if h == ['*']: return 0,0
 855         c = u = 0
 856         received_coins = []   # list of coins received at address
 857
 858         for tx_hash, tx_height in h:
 859             tx = self.transactions.get(tx_hash)
 860             if not tx: continue
 861
 862             for i, (addr, value) in enumerate(tx.outputs):
 863                 if addr == address:
 864                     key = tx_hash + ':%d'%i
 865                     received_coins.append(key)
 866
 867         for tx_hash, tx_height in h:
 868             tx = self.transactions.get(tx_hash)
 869             if not tx: continue
 870             v = 0
 871
 872             for item in tx.inputs:
 873                 addr = item.get('address')
 874                 if addr == address:
 875                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 876                     value = self.prevout_values.get( key )
 877                     if key in received_coins:
 878                         v -= value
 879
 880             for i, (addr, value) in enumerate(tx.outputs):
 881                 key = tx_hash + ':%d'%i
 882                 if addr == address:
 883                     v += value
 884
 885             if tx_height:
 886                 c += v
 887             else:
 888                 u += v
 889         return c, u
 890
 891
 892     def get_account_name(self, k):
 893         if k == 0:
 894             if self.seed_version == 4:
 895                 name = 'Main account'
 896             else:
 897                 name = 'Old account'
 898         else:
 899             name = self.labels.get(k, 'Unnamed account')
 900         return name
 901
 902     def get_account_names(self):
 903         accounts = {}
 904         for k, account in self.accounts.items():
 905             accounts[k] = self.get_account_name(k)
 906         if self.imported_keys:
 907             accounts[-1] = 'Imported keys'
 908         return accounts
 909
 910     def get_account_addresses(self, a, include_change=True):
 911         if a is None:
 912             o = self.addresses(True)
 913         elif a == -1:
 914             o = self.imported_keys.keys()
 915         else:
 916             ac = self.accounts[a]
 917             o = ac.get_addresses(0)
 918             if include_change: o += ac.get_addresses(1)
 919         return o
 920
 921     def get_imported_balance(self):
 922         cc = uu = 0
 923         for addr in self.imported_keys.keys():
 924             c, u = self.get_addr_balance(addr)
 925             cc += c
 926             uu += u
 927         return cc, uu
 928
 929     def get_account_balance(self, account):
 930         if account is None:
 931             return self.get_balance()
 932         elif account == -1:
 933             return self.get_imported_balance()
 934
 935         conf = unconf = 0
 936         for addr in self.get_account_addresses(account):
 937             c, u = self.get_addr_balance(addr)
 938             conf += c
 939             unconf += u
 940         return conf, unconf
 941
 942     def get_frozen_balance(self):
 943         conf = unconf = 0
 944         for addr in self.frozen_addresses:
 945             c, u = self.get_addr_balance(addr)
 946             conf += c
 947             unconf += u
 948         return conf, unconf
 949
 950
 951     def get_balance(self):
 952         cc = uu = 0
 953         for a in self.accounts.keys():
 954             c, u = self.get_account_balance(a)
 955             cc += c
 956             uu += u
 957         c, u = self.get_imported_balance()
 958         cc += c
 959         uu += u
 960         return cc, uu
 961
 962
 963     def get_unspent_coins(self, domain=None):
 964         coins = []
 965         if domain is None: domain = self.addresses(True)
 966         for addr in domain:
 967             h = self.history.get(addr, [])
 968             if h == ['*']: continue
 969             for tx_hash, tx_height in h:
 970                 tx = self.transactions.get(tx_hash)
 971                 if tx is None: raise BaseException("Wallet not synchronized")
 972                 for output in tx.d.get('outputs'):
 973                     if output.get('address') != addr: continue
 974                     key = tx_hash + ":%d" % output.get('prevout_n')
 975                     if key in self.spent_outputs: continue
 976                     output['prevout_hash'] = tx_hash
 977                     output['height'] = tx_height
 978                     coins.append((tx_height, output))
 979
 980         # sort by age
 981         if coins:
 982             coins = sorted(coins)
 983             if coins[-1][0] != 0:
 984                 while coins[0][0] == 0:
 985                     coins = coins[1:] + [ coins[0] ]
 986         return [x[1] for x in coins]
 987
 988
 989
 990     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
 991         """ todo: minimize tx size """
 992         total = 0
 993         fee = self.fee if fixed_fee is None else fixed_fee
 994         if domain is None:
 995             domain = self.addresses()
 996
 997         for i in self.frozen_addresses:
 998             if i in domain: domain.remove(i)
 999
1000         prioritized = []
1001         for i in self.prioritized_addresses:
1002             if i in domain:
1003                 domain.remove(i)
1004                 prioritized.append(i)
1005
1006         coins = self.get_unspent_coins(domain)
1007         prioritized_coins = self.get_unspent_coins(prioritized)
1008
1009         inputs = []
1010         coins = prioritized_coins + coins
1011
1012         for item in coins:
1013             addr = item.get('address')
1014             v = item.get('value')
1015             total += v
1016             inputs.append(item)
1017             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1018             if total >= amount + fee: break
1019         else:
1020             inputs = []
1021
1022         return inputs, total, fee
1023
1024
1025     def set_fee(self, fee):
1026         if self.fee != fee:
1027             self.fee = fee
1028             self.storage.put('fee_per_kb', self.fee, True)
1029
1030     def estimated_fee(self, inputs):
1031         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1032         fee = self.fee * int(round(estimated_size/1024.))
1033         if fee == 0: fee = self.fee
1034         return fee
1035
1036
1037     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1038         "add change to a transaction"
1039         change_amount = total - ( amount + fee )
1040         if change_amount != 0:
1041             if not change_addr:
1042
1043                 # send change to one of the accounts involved in the tx
1044                 address = inputs[0].get('address')
1045                 account, _ = self.get_address_index(address)
1046
1047                 if not self.use_change or account == -1:
1048                     change_addr = inputs[-1]['address']
1049                 else:
1050                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1051
1052             # Insert the change output at a random position in the outputs
1053             posn = random.randint(0, len(outputs))
1054             outputs[posn:posn] = [( change_addr,  change_amount)]
1055         return outputs
1056
1057
1058     def get_history(self, address):
1059         with self.lock:
1060             return self.history.get(address)
1061
1062
1063     def get_status(self, h):
1064         if not h: return None
1065         if h == ['*']: return '*'
1066         status = ''
1067         for tx_hash, height in h:
1068             status += tx_hash + ':%d:' % height
1069         return hashlib.sha256( status ).digest().encode('hex')
1070
1071
1072     def receive_tx_callback(self, tx_hash, tx, tx_height):
1073
1074         with self.transaction_lock:
1075             self.add_extra_addresses(tx)
1076             if not self.check_new_tx(tx_hash, tx):
1077                 # may happen due to pruning
1078                 print_error("received transaction that is no longer referenced in history", tx_hash)
1079                 return
1080             self.transactions[tx_hash] = tx
1081             self.network.interface.pending_transactions_for_notifications.append(tx)
1082             self.save_transactions()
1083             if self.verifier and tx_height>0:
1084                 self.verifier.add(tx_hash, tx_height)
1085             self.update_tx_outputs(tx_hash)
1086
1087
1088     def save_transactions(self):
1089         tx = {}
1090         for k,v in self.transactions.items():
1091             tx[k] = str(v)
1092         self.storage.put('transactions', tx, True)
1093
1094     def receive_history_callback(self, addr, hist):
1095
1096         if not self.check_new_history(addr, hist):
1097             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1098
1099         with self.lock:
1100             self.history[addr] = hist
1101             self.storage.put('addr_history', self.history, True)
1102
1103         if hist != ['*']:
1104             for tx_hash, tx_height in hist:
1105                 if tx_height>0:
1106                     # add it in case it was previously unconfirmed
1107                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1108
1109
1110     def get_tx_history(self, account=None):
1111         with self.transaction_lock:
1112             history = self.transactions.items()
1113             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1114             result = []
1115
1116             balance = 0
1117             for tx_hash, tx in history:
1118                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1119                 if v is not None: balance += v
1120
1121             c, u = self.get_account_balance(account)
1122
1123             if balance != c+u:
1124                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1125
1126             balance = c + u - balance
1127             for tx_hash, tx in history:
1128                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1129                 if not is_relevant:
1130                     continue
1131                 if value is not None:
1132                     balance += value
1133
1134                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1135                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1136
1137         return result
1138
1139
1140     def get_label(self, tx_hash):
1141         label = self.labels.get(tx_hash)
1142         is_default = (label == '') or (label is None)
1143         if is_default: label = self.get_default_label(tx_hash)
1144         return label, is_default
1145
1146
1147     def get_default_label(self, tx_hash):
1148         tx = self.transactions.get(tx_hash)
1149         default_label = ''
1150         if tx:
1151             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1152             if is_mine:
1153                 for o in tx.outputs:
1154                     o_addr, _ = o
1155                     if not self.is_mine(o_addr):
1156                         try:
1157                             default_label = self.labels[o_addr]
1158                         except KeyError:
1159                             default_label = o_addr
1160                         break
1161                 else:
1162                     default_label = '(internal)'
1163             else:
1164                 for o in tx.outputs:
1165                     o_addr, _ = o
1166                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1167                         break
1168                 else:
1169                     for o in tx.outputs:
1170                         o_addr, _ = o
1171                         if self.is_mine(o_addr):
1172                             break
1173                     else:
1174                         o_addr = None
1175
1176                 if o_addr:
1177                     dest_label = self.labels.get(o_addr)
1178                     try:
1179                         default_label = self.labels[o_addr]
1180                     except KeyError:
1181                         default_label = o_addr
1182
1183         return default_label
1184
1185
1186     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1187         for address, x in outputs:
1188             assert is_valid(address)
1189         amount = sum( map(lambda x:x[1], outputs) )
1190         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1191         if not inputs:
1192             raise ValueError("Not enough funds")
1193         self.add_input_info(inputs)
1194         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1195         return Transaction.from_io(inputs, outputs)
1196
1197
1198     def mktx_from_account(self, outputs, password, fee=None, change_addr=None, account=None):
1199         domain = self.get_account_addresses(account) if account else None
1200         return self.mktx(outputs, password, fee, change_addr, domain)
1201
1202
1203     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1204         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1205         keypairs = {}
1206         self.add_keypairs_from_wallet(tx, keypairs, password)
1207         self.sign_transaction(tx, keypairs)
1208         return tx
1209
1210
1211     def add_input_info(self, inputs):
1212         for txin in inputs:
1213             address = txin['address']
1214             account, sequence = self.get_address_index(address)
1215             txin['KeyID'] = self.get_keyID(account, sequence)
1216             redeemScript = self.accounts[account].redeem_script(sequence)
1217             if redeemScript:
1218                 txin['redeemScript'] = redeemScript
1219             else:
1220                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1221
1222
1223     def sign_transaction(self, tx, keypairs):
1224         tx.sign(keypairs)
1225         run_hook('sign_transaction', tx)
1226
1227
1228     def sendtx(self, tx):
1229         # synchronous
1230         h = self.send_tx(tx)
1231         self.tx_event.wait()
1232         return self.receive_tx(h)
1233
1234     def send_tx(self, tx):
1235         # asynchronous
1236         self.tx_event.clear()
1237         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1238         return tx.hash()
1239
1240     def on_broadcast(self, i, r):
1241         self.tx_result = r.get('result')
1242         self.tx_event.set()
1243
1244     def receive_tx(self,tx_hash):
1245         out = self.tx_result
1246         if out != tx_hash:
1247             return False, "error: " + out
1248         return True, out
1249
1250
1251
1252     def update_password(self, seed, old_password, new_password):
1253         if new_password == '': new_password = None
1254         # this will throw an exception if unicode cannot be converted
1255         self.seed = pw_encode( seed, new_password)
1256         self.storage.put('seed', self.seed, True)
1257         self.use_encryption = (new_password != None)
1258         self.storage.put('use_encryption', self.use_encryption,True)
1259         for k in self.imported_keys.keys():
1260             a = self.imported_keys[k]
1261             b = pw_decode(a, old_password)
1262             c = pw_encode(b, new_password)
1263             self.imported_keys[k] = c
1264         self.storage.put('imported_keys', self.imported_keys, True)
1265
1266         for k, v in self.master_private_keys.items():
1267             b = pw_decode(v, old_password)
1268             c = pw_encode(b, new_password)
1269             self.master_private_keys[k] = c
1270         self.storage.put('master_private_keys', self.master_private_keys, True)
1271
1272
1273     def freeze(self,addr):
1274         if self.is_mine(addr) and addr not in self.frozen_addresses:
1275             self.unprioritize(addr)
1276             self.frozen_addresses.append(addr)
1277             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1278             return True
1279         else:
1280             return False
1281
1282     def unfreeze(self,addr):
1283         if self.is_mine(addr) and addr in self.frozen_addresses:
1284             self.frozen_addresses.remove(addr)
1285             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1286             return True
1287         else:
1288             return False
1289
1290     def prioritize(self,addr):
1291         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1292             self.unfreeze(addr)
1293             self.prioritized_addresses.append(addr)
1294             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1295             return True
1296         else:
1297             return False
1298
1299     def unprioritize(self,addr):
1300         if self.is_mine(addr) and addr in self.prioritized_addresses:
1301             self.prioritized_addresses.remove(addr)
1302             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1303             return True
1304         else:
1305             return False
1306
1307
1308     def set_verifier(self, verifier):
1309         self.verifier = verifier
1310
1311         # review transactions that are in the history
1312         for addr, hist in self.history.items():
1313             if hist == ['*']: continue
1314             for tx_hash, tx_height in hist:
1315                 if tx_height>0:
1316                     # add it in case it was previously unconfirmed
1317                     self.verifier.add(tx_hash, tx_height)
1318
1319
1320         # if we are on a pruning server, remove unverified transactions
1321         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1322         for tx_hash in self.transactions.keys():
1323             if tx_hash not in vr:
1324                 self.transactions.pop(tx_hash)
1325
1326
1327
1328     def check_new_history(self, addr, hist):
1329
1330         # check that all tx in hist are relevant
1331         if hist != ['*']:
1332             for tx_hash, height in hist:
1333                 tx = self.transactions.get(tx_hash)
1334                 if not tx: continue
1335                 if not tx.has_address(addr):
1336                     return False
1337
1338         # check that we are not "orphaning" a transaction
1339         old_hist = self.history.get(addr,[])
1340         if old_hist == ['*']: return True
1341
1342         for tx_hash, height in old_hist:
1343             if tx_hash in map(lambda x:x[0], hist): continue
1344             found = False
1345             for _addr, _hist in self.history.items():
1346                 if _addr == addr: continue
1347                 if _hist == ['*']: continue
1348                 _tx_hist = map(lambda x:x[0], _hist)
1349                 if tx_hash in _tx_hist:
1350                     found = True
1351                     break
1352
1353             if not found:
1354                 tx = self.transactions.get(tx_hash)
1355                 # tx might not be there
1356                 if not tx: continue
1357
1358                 # already verified?
1359                 if self.verifier.get_height(tx_hash):
1360                     continue
1361                 # unconfirmed tx
1362                 print_error("new history is orphaning transaction:", tx_hash)
1363                 # check that all outputs are not mine, request histories
1364                 ext_requests = []
1365                 for _addr, _v in tx.outputs:
1366                     # assert not self.is_mine(_addr)
1367                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1368
1369                 ext_h = self.network.interface.synchronous_get(ext_requests)
1370                 print_error("sync:", ext_requests, ext_h)
1371                 height = None
1372                 for h in ext_h:
1373                     if h == ['*']: continue
1374                     for item in h:
1375                         if item.get('tx_hash') == tx_hash:
1376                             height = item.get('height')
1377                 if height:
1378                     print_error("found height for", tx_hash, height)
1379                     self.verifier.add(tx_hash, height)
1380                 else:
1381                     print_error("removing orphaned tx from history", tx_hash)
1382                     self.transactions.pop(tx_hash)
1383
1384         return True
1385
1386
1387
1388     def check_new_tx(self, tx_hash, tx):
1389         # 1 check that tx is referenced in addr_history.
1390         addresses = []
1391         for addr, hist in self.history.items():
1392             if hist == ['*']:continue
1393             for txh, height in hist:
1394                 if txh == tx_hash:
1395                     addresses.append(addr)
1396
1397         if not addresses:
1398             return False
1399
1400         # 2 check that referencing addresses are in the tx
1401         for addr in addresses:
1402             if not tx.has_address(addr):
1403                 return False
1404
1405         return True
1406
1407
1408     def start_threads(self, network):
1409         from verifier import TxVerifier
1410         self.network = network
1411         self.verifier = TxVerifier(self.network, self.storage)
1412         self.verifier.start()
1413         self.set_verifier(self.verifier)
1414         self.synchronizer = WalletSynchronizer(self)
1415         self.synchronizer.start()
1416
1417     def stop_threads(self):
1418         self.verifier.stop()
1419         self.synchronizer.stop()
1420
1421
1422
1423     def restore(self, callback):
1424         from i18n import _
1425         def wait_for_wallet():
1426             self.set_up_to_date(False)
1427             while not self.is_up_to_date():
1428                 msg = "%s\n%s %d\n%s %.1f"%(
1429                     _("Please wait..."),
1430                     _("Addresses generated:"),
1431                     len(self.addresses(True)),_("Kilobytes received:"),
1432                     self.network.interface.bytes_received/1024.)
1433
1434                 apply(callback, (msg,))
1435                 time.sleep(0.1)
1436
1437         def wait_for_network():
1438             while not self.network.interface.is_connected:
1439                 msg = "%s \n" % (_("Connecting..."))
1440                 apply(callback, (msg,))
1441                 time.sleep(0.1)
1442
1443         # wait until we are connected, because the user might have selected another server
1444         wait_for_network()
1445
1446         # try to restore old account
1447         self.create_old_account()
1448         wait_for_wallet()
1449
1450         if self.is_found():
1451             self.seed_version = 4
1452             self.storage.put('seed_version', wallet.seed_version, True)
1453         else:
1454             self.accounts.pop(0)
1455             self.create_accounts()
1456             wait_for_wallet()
1457
1458
1459
1460
1461 class WalletSynchronizer(threading.Thread):
1462
1463
1464     def __init__(self, wallet):
1465         threading.Thread.__init__(self)
1466         self.daemon = True
1467         self.wallet = wallet
1468         wallet.synchronizer = self
1469         self.network = self.wallet.network
1470         #self.wallet.network.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1471         self.was_updated = True
1472         self.running = False
1473         self.lock = threading.Lock()
1474         self.queue = Queue.Queue()
1475
1476     def stop(self):
1477         with self.lock: self.running = False
1478
1479     def is_running(self):
1480         with self.lock: return self.running
1481
1482
1483     def subscribe_to_addresses(self, addresses):
1484         messages = []
1485         for addr in addresses:
1486             messages.append(('blockchain.address.subscribe', [addr]))
1487         self.network.interface.send( messages, lambda i,r: self.queue.put(r))
1488
1489
1490     def run(self):
1491         with self.lock:
1492             self.running = True
1493
1494         while self.is_running():
1495             interface = self.network.interface
1496             if not interface.is_connected:
1497                 print_error("synchronizer: waiting for interface")
1498                 interface.connect_event.wait()
1499
1500             self.run_interface(interface)
1501
1502
1503     def run_interface(self, interface):
1504
1505         print_error("synchronizer: connected to", interface.server)
1506
1507         requested_tx = []
1508         missing_tx = []
1509         requested_histories = {}
1510
1511         # request any missing transactions
1512         for history in self.wallet.history.values():
1513             if history == ['*']: continue
1514             for tx_hash, tx_height in history:
1515                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1516                     missing_tx.append( (tx_hash, tx_height) )
1517
1518         if missing_tx:
1519             print_error("missing tx", missing_tx)
1520
1521         # subscriptions
1522         self.subscribe_to_addresses(self.wallet.addresses(True, next=True))
1523
1524         while self.is_running():
1525             # 1. create new addresses
1526             new_addresses = self.wallet.synchronize()
1527
1528             # request missing addresses
1529             if new_addresses:
1530                 self.subscribe_to_addresses(new_addresses)
1531
1532             # request missing transactions
1533             for tx_hash, tx_height in missing_tx:
1534                 if (tx_hash, tx_height) not in requested_tx:
1535                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1536                     requested_tx.append( (tx_hash, tx_height) )
1537             missing_tx = []
1538
1539             # detect if situation has changed
1540             if interface.is_up_to_date() and self.queue.empty():
1541                 if not self.wallet.is_up_to_date():
1542                     self.wallet.set_up_to_date(True)
1543                     self.was_updated = True
1544             else:
1545                 if self.wallet.is_up_to_date():
1546                     self.wallet.set_up_to_date(False)
1547                     self.was_updated = True
1548
1549             if self.was_updated:
1550                 self.wallet.network.trigger_callback('updated')
1551                 self.was_updated = False
1552
1553             # 2. get a response
1554             try:
1555                 r = self.queue.get(block=True, timeout=1)
1556             except Queue.Empty:
1557                 continue
1558
1559             if interface != self.network.interface:
1560                 break
1561
1562             if not r:
1563                 continue
1564
1565             # 3. handle response
1566             method = r['method']
1567             params = r['params']
1568             result = r.get('result')
1569             error = r.get('error')
1570             if error:
1571                 print "error", r
1572                 continue
1573
1574             if method == 'blockchain.address.subscribe':
1575                 addr = params[0]
1576                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1577                     if requested_histories.get(addr) is None:
1578                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1579                         requested_histories[addr] = result
1580
1581             elif method == 'blockchain.address.get_history':
1582                 addr = params[0]
1583                 print_error("receiving history", addr, result)
1584                 if result == ['*']:
1585                     assert requested_histories.pop(addr) == '*'
1586                     self.wallet.receive_history_callback(addr, result)
1587                 else:
1588                     hist = []
1589                     # check that txids are unique
1590                     txids = []
1591                     for item in result:
1592                         tx_hash = item['tx_hash']
1593                         if tx_hash not in txids:
1594                             txids.append(tx_hash)
1595                             hist.append( (tx_hash, item['height']) )
1596
1597                     if len(hist) != len(result):
1598                         raise BaseException("error: server sent history with non-unique txid", result)
1599
1600                     # check that the status corresponds to what was announced
1601                     rs = requested_histories.pop(addr)
1602                     if self.wallet.get_status(hist) != rs:
1603                         raise BaseException("error: status mismatch: %s"%addr)
1604
1605                     # store received history
1606                     self.wallet.receive_history_callback(addr, hist)
1607
1608                     # request transactions that we don't have
1609                     for tx_hash, tx_height in hist:
1610                         if self.wallet.transactions.get(tx_hash) is None:
1611                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1612                                 missing_tx.append( (tx_hash, tx_height) )
1613
1614             elif method == 'blockchain.transaction.get':
1615                 tx_hash = params[0]
1616                 tx_height = params[1]
1617                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1618                 tx = Transaction(result)
1619                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1620                 self.was_updated = True
1621                 requested_tx.remove( (tx_hash, tx_height) )
1622                 print_error("received tx:", tx_hash, len(tx.raw))
1623
1624             else:
1625                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1626
1627             if self.was_updated and not requested_tx:
1628                 self.wallet.network.trigger_callback('updated')
1629                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1630
1631                 self.was_updated = False