lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         v = self.data.get(key)
 129         if v is None:
 130             v = default
 131         return v
 132
 133     def put(self, key, value, save = True):
 134
 135         with self.lock:
 136             if value is not None:
 137                 self.data[key] = value
 138             else:
 139                 self.data.pop(key)
 140             if save:
 141                 self.write()
 142
 143     def write(self):
 144         s = repr(self.data)
 145         f = open(self.path,"w")
 146         f.write( s )
 147         f.close()
 148         if 'ANDROID_DATA' not in os.environ:
 149             import stat
 150             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 151
 152
 153 class Wallet:
 154
 155     def __init__(self, storage):
 156
 157         self.storage = storage
 158         self.electrum_version = ELECTRUM_VERSION
 159         self.gap_limit_for_change = 3 # constant
 160
 161         # saved fields
 162         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 163
 164         self.gap_limit             = storage.get('gap_limit', 5)
 165         self.use_change            = storage.get('use_change',True)
 166         self.use_encryption        = storage.get('use_encryption', False)
 167         self.seed                  = storage.get('seed', '')               # encrypted
 168         self.labels                = storage.get('labels', {})
 169         self.frozen_addresses      = storage.get('frozen_addresses',[])
 170         self.addressbook           = storage.get('contacts', [])
 171
 172         self.imported_keys         = storage.get('imported_keys',{})
 173         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 174
 175         self.fee                   = int(storage.get('fee_per_kb',20000))
 176
 177         self.master_public_keys = storage.get('master_public_keys',{})
 178         self.master_private_keys = storage.get('master_private_keys', {})
 179
 180         self.next_addresses = storage.get('next_addresses',{})
 181
 182         if self.seed_version not in [4, 6]:
 183             msg = "This wallet seed is not supported."
 184             if self.seed_version in [5]:
 185                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 186                 print msg
 187                 sys.exit(1)
 188
 189         # This attribute is set when wallet.start_threads is called.
 190         self.synchronizer = None
 191
 192         self.load_accounts()
 193
 194         self.transactions = {}
 195         tx_list = self.storage.get('transactions',{})
 196         for k,v in tx_list.items():
 197             try:
 198                 tx = Transaction(v)
 199             except Exception:
 200                 print_msg("Warning: Cannot deserialize transactions. skipping")
 201                 continue
 202
 203             self.add_extra_addresses(tx)
 204             self.transactions[k] = tx
 205
 206         for h,tx in self.transactions.items():
 207             if not self.check_new_tx(h, tx):
 208                 print_error("removing unreferenced tx", h)
 209                 self.transactions.pop(h)
 210
 211
 212         # not saved
 213         self.prevout_values = {}     # my own transaction outputs
 214         self.spent_outputs = []
 215
 216         # spv
 217         self.verifier = None
 218
 219         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 220         # interface.is_up_to_date() returns true when all requests have been answered and processed
 221         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 222
 223         self.up_to_date = False
 224         self.lock = threading.Lock()
 225         self.transaction_lock = threading.Lock()
 226         self.tx_event = threading.Event()
 227
 228         for tx_hash, tx in self.transactions.items():
 229             self.update_tx_outputs(tx_hash)
 230
 231
 232     def add_extra_addresses(self, tx):
 233         h = tx.hash()
 234         # find the address corresponding to pay-to-pubkey inputs
 235         tx.add_extra_addresses(self.transactions)
 236         for o in tx.d.get('outputs'):
 237             if o.get('is_pubkey'):
 238                 for tx2 in self.transactions.values():
 239                     tx2.add_extra_addresses({h:tx})
 240
 241
 242
 243
 244     def set_up_to_date(self,b):
 245         with self.lock: self.up_to_date = b
 246
 247
 248     def is_up_to_date(self):
 249         with self.lock: return self.up_to_date
 250
 251
 252     def update(self):
 253         self.up_to_date = False
 254         while not self.is_up_to_date():
 255             time.sleep(0.1)
 256
 257
 258     def import_key(self, sec, password):
 259         # check password
 260         seed = self.get_seed(password)
 261         try:
 262             address = address_from_private_key(sec)
 263         except Exception:
 264             raise Exception('Invalid private key')
 265
 266         if self.is_mine(address):
 267             raise Exception('Address already in wallet')
 268
 269         # store the originally requested keypair into the imported keys table
 270         self.imported_keys[address] = pw_encode(sec, password )
 271         self.storage.put('imported_keys', self.imported_keys, True)
 272         if self.synchronizer:
 273             self.synchronizer.subscribe_to_addresses([address])
 274         return address
 275
 276     def delete_imported_key(self, addr):
 277         if addr in self.imported_keys:
 278             self.imported_keys.pop(addr)
 279             self.storage.put('imported_keys', self.imported_keys, True)
 280
 281
 282     def make_seed(self):
 283         import mnemonic, ecdsa
 284         entropy = ecdsa.util.randrange( pow(2,160) )
 285         nonce = 0
 286         while True:
 287             ss = "%040x"%(entropy+nonce)
 288             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 289             # we keep only 13 words, that's approximately 139 bits of entropy
 290             words = mnemonic.mn_encode(s)[0:13]
 291             seed = ' '.join(words)
 292             if is_seed(seed):
 293                 break  # this will remove 8 bits of entropy
 294             nonce += 1
 295
 296         return seed
 297
 298
 299     def init_seed(self, seed):
 300         import mnemonic
 301
 302         if self.seed:
 303             raise Exception("a seed exists")
 304
 305         if not seed:
 306             self.seed = self.make_seed()
 307             self.seed_version = SEED_VERSION
 308             return
 309
 310         # find out what kind of wallet we are
 311         try:
 312             seed.strip().decode('hex')
 313             self.seed_version = 4
 314             self.seed = str(seed)
 315             return
 316         except Exception:
 317             pass
 318
 319         words = seed.split()
 320         try:
 321             mnemonic.mn_decode(words)
 322             uses_electrum_words = True
 323         except Exception:
 324             uses_electrum_words = False
 325
 326         if uses_electrum_words and len(words) != 13:
 327             self.seed_version = 4
 328             self.seed = mnemonic.mn_decode(words)
 329         else:
 330             #assert is_seed(seed)
 331             self.seed_version = SEED_VERSION
 332             self.seed = seed
 333
 334
 335     def save_seed(self, password):
 336         if password:
 337             self.seed = pw_encode( self.seed, password)
 338             self.use_encryption = True
 339         self.storage.put('seed', self.seed, True)
 340         self.storage.put('seed_version', self.seed_version, True)
 341         self.storage.put('use_encryption', self.use_encryption,True)
 342         self.create_accounts(password)
 343
 344
 345     def create_watching_only_wallet(self, params):
 346         K0, c0 = params
 347         if not K0:
 348             return
 349
 350         if not c0:
 351             self.seed_version = 4
 352             self.storage.put('seed_version', self.seed_version, True)
 353             self.create_old_account(K0)
 354             return
 355
 356         cK0 = ""
 357         self.master_public_keys = {
 358             "m/0'/": (c0, K0, cK0),
 359             }
 360         self.storage.put('master_public_keys', self.master_public_keys, True)
 361         self.storage.put('seed_version', self.seed_version, True)
 362         self.create_account('1','Main account')
 363
 364
 365     def create_accounts(self, password):
 366         seed = pw_decode(self.seed, password)
 367
 368         if self.seed_version == 4:
 369             mpk = OldAccount.mpk_from_seed(seed)
 370             self.create_old_account(mpk)
 371         else:
 372             # create default account
 373             self.create_master_keys('1', password)
 374             self.create_account('1','Main account')
 375
 376
 377     def create_master_keys(self, account_type, password):
 378         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(password))
 379         if account_type == '1':
 380             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 381             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 382             self.master_private_keys["m/0'/"] = pw_encode(k0, password)
 383         elif account_type == '2of2':
 384             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 385             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 386             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 387             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 388             self.master_private_keys["m/1'/"] = pw_encode(k1, password)
 389             self.master_private_keys["m/2'/"] = pw_encode(k2, password)
 390         elif account_type == '2of3':
 391             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 392             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 393             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 394             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 395             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 396             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 397             self.master_private_keys["m/3'/"] = pw_encode(k3, password)
 398             self.master_private_keys["m/4'/"] = pw_encode(k4, password)
 399             self.master_private_keys["m/5'/"] = pw_encode(k5, password)
 400
 401         self.storage.put('master_public_keys', self.master_public_keys, True)
 402         self.storage.put('master_private_keys', self.master_private_keys, True)
 403
 404     def has_master_public_keys(self, account_type):
 405         if account_type == '1':
 406             return "m/0'/" in self.master_public_keys
 407         elif account_type == '2of2':
 408             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 409         elif account_type == '2of3':
 410             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 411
 412     def find_root_by_master_key(self, c, K):
 413         for key, v in self.master_public_keys.items():
 414             if key == "m/":continue
 415             cc, KK, _ = v
 416             if (c == cc) and (K == KK):
 417                 return key
 418
 419     def deseed_root(self, seed, password):
 420         # for safety, we ask the user to enter their seed
 421         assert seed == self.get_seed(password)
 422         self.seed = ''
 423         self.storage.put('seed', '', True)
 424
 425
 426     def deseed_branch(self, k):
 427         # check that parent has no seed
 428         # assert self.seed == ''
 429         self.master_private_keys.pop(k)
 430         self.storage.put('master_private_keys', self.master_private_keys, True)
 431
 432
 433     def is_watching_only(self):
 434         return (self.seed == '') and (self.master_private_keys == {})
 435
 436
 437
 438     def account_id(self, account_type, i):
 439         if account_type == '1':
 440             return "m/0'/%d"%i
 441         elif account_type == '2of2':
 442             return "m/1'/%d & m/2'/%d"%(i,i)
 443         elif account_type == '2of3':
 444             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 445         else:
 446             raise Exception('unknown account type')
 447
 448
 449     def num_accounts(self, account_type):
 450         keys = self.accounts.keys()
 451         i = 0
 452         while True:
 453             account_id = self.account_id(account_type, i)
 454             if account_id not in keys: break
 455             i += 1
 456         return i
 457
 458
 459     def new_account_address(self, account_type = '1'):
 460         i = self.num_accounts(account_type)
 461         k = self.account_id(account_type,i)
 462
 463         addr = self.next_addresses.get(k)
 464         if not addr:
 465             account_id, account = self.next_account(account_type)
 466             addr = account.first_address()
 467             self.next_addresses[k] = addr
 468             self.storage.put('next_addresses',self.next_addresses)
 469
 470         return k, addr
 471
 472
 473     def next_account(self, account_type = '1'):
 474
 475         i = self.num_accounts(account_type)
 476         account_id = self.account_id(account_type,i)
 477
 478         if account_type is '1':
 479             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 480             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 481             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 482
 483         elif account_type == '2of2':
 484             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 485             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 486             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 487             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 488             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 489
 490         elif account_type == '2of3':
 491             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 492             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 493             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 494             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 495             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 496             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 497             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 498
 499         return account_id, account
 500
 501
 502     def set_label(self, name, text = None):
 503         changed = False
 504         old_text = self.labels.get(name)
 505         if text:
 506             if old_text != text:
 507                 self.labels[name] = text
 508                 changed = True
 509         else:
 510             if old_text:
 511                 self.labels.pop(name)
 512                 changed = True
 513
 514         if changed:
 515             self.storage.put('labels', self.labels, True)
 516
 517         run_hook('set_label', name, text, changed)
 518         return changed
 519
 520
 521
 522     def create_account(self, account_type = '1', name = None):
 523         k, account = self.next_account(account_type)
 524         if k in self.pending_accounts:
 525             self.pending_accounts.pop(k)
 526             self.storage.put('pending_accounts', self.pending_accounts)
 527
 528         self.accounts[k] = account
 529         self.save_accounts()
 530         if name:
 531             self.set_label(k, name)
 532
 533
 534     def create_old_account(self, mpk):
 535         self.storage.put('master_public_key', mpk, True)
 536         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 537         self.save_accounts()
 538
 539
 540     def save_accounts(self):
 541         d = {}
 542         for k, v in self.accounts.items():
 543             d[k] = v.dump()
 544         self.storage.put('accounts', d, True)
 545
 546
 547
 548     def load_accounts(self):
 549         d = self.storage.get('accounts', {})
 550         self.accounts = {}
 551         for k, v in d.items():
 552             if k == 0:
 553                 v['mpk'] = self.storage.get('master_public_key')
 554                 self.accounts[k] = OldAccount(v)
 555             elif '&' in k:
 556                 self.accounts[k] = BIP32_Account_2of2(v)
 557             else:
 558                 self.accounts[k] = BIP32_Account(v)
 559
 560         self.pending_accounts = self.storage.get('pending_accounts',{})
 561
 562
 563     def delete_pending_account(self, k):
 564         self.pending_accounts.pop(k)
 565         self.storage.put('pending_accounts', self.pending_accounts)
 566
 567     def account_is_pending(self, k):
 568         return k in self.pending_accounts
 569
 570     def create_pending_account(self, acct_type, name):
 571         k, addr = self.new_account_address(acct_type)
 572         self.set_label(k, name)
 573         self.pending_accounts[k] = addr
 574         self.storage.put('pending_accounts', self.pending_accounts)
 575
 576     def get_pending_accounts(self):
 577         return self.pending_accounts.items()
 578
 579
 580     def addresses(self, include_change = True, _next=True):
 581         o = self.get_account_addresses(-1, include_change)
 582         for a in self.accounts.keys():
 583             o += self.get_account_addresses(a, include_change)
 584
 585         if _next:
 586             for addr in self.next_addresses.values():
 587                 if addr not in o:
 588                     o += [addr]
 589         return o
 590
 591
 592     def is_mine(self, address):
 593         return address in self.addresses(True)
 594
 595
 596     def is_change(self, address):
 597         if not self.is_mine(address): return False
 598         if address in self.imported_keys.keys(): return False
 599         acct, s = self.get_address_index(address)
 600         if s is None: return False
 601         return s[0] == 1
 602
 603     def get_master_public_key(self):
 604         if self.seed_version == 4:
 605             return self.storage.get("master_public_key")
 606         else:
 607             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 608             return repr((c, K))
 609
 610     def get_master_private_key(self, account, password):
 611         k = self.master_private_keys.get(account)
 612         if not k: return
 613         master_k = pw_decode( k, password)
 614         master_c, master_K, master_Kc = self.master_public_keys[account]
 615         try:
 616             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 617             assert K.encode('hex') == master_K
 618         except Exception:
 619             raise Exception("Invalid password")
 620         return master_k
 621
 622
 623     def get_address_index(self, address):
 624         if address in self.imported_keys.keys():
 625             return -1, None
 626
 627         for account in self.accounts.keys():
 628             for for_change in [0,1]:
 629                 addresses = self.accounts[account].get_addresses(for_change)
 630                 for addr in addresses:
 631                     if address == addr:
 632                         return account, (for_change, addresses.index(addr))
 633
 634         for k,v in self.next_addresses.items():
 635             if v == address:
 636                 return k, (0,0)
 637
 638         raise Exception("Address not found", address)
 639
 640
 641     def get_roots(self, account):
 642         roots = []
 643         for a in account.split('&'):
 644             s = a.strip()
 645             m = re.match("(m/\d+'/)(\d+)", s)
 646             roots.append( m.group(1) )
 647         return roots
 648
 649     def is_seeded(self, account):
 650         if type(account) is int:
 651             return self.seed is not None
 652
 653         for root in self.get_roots(account):
 654             if root not in self.master_private_keys.keys():
 655                 return False
 656         return True
 657
 658     def rebase_sequence(self, account, sequence):
 659         c, i = sequence
 660         dd = []
 661         for a in account.split('&'):
 662             s = a.strip()
 663             m = re.match("(m/\d+'/)(\d+)", s)
 664             root = m.group(1)
 665             num = int(m.group(2))
 666             dd.append( (root, [num,c,i] ) )
 667         return dd
 668
 669
 670     def get_keyID(self, account, sequence):
 671         if account == 0:
 672             a, b = sequence
 673             mpk = self.storage.get('master_public_key')
 674             return 'old(%s,%d,%d)'%(mpk,a,b)
 675
 676         rs = self.rebase_sequence(account, sequence)
 677         dd = []
 678         for root, public_sequence in rs:
 679             c, K, _ = self.master_public_keys[root]
 680             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 681             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 682         return '&'.join(dd)
 683
 684
 685
 686     def get_seed(self, password):
 687         s = pw_decode(self.seed, password)
 688         if self.seed_version == 4:
 689             seed = s
 690             self.accounts[0].check_seed(seed)
 691         else:
 692             seed = mnemonic_hash(s)
 693         return seed
 694
 695
 696     def get_mnemonic(self, password):
 697         import mnemonic
 698         s = pw_decode(self.seed, password)
 699         if self.seed_version == 4:
 700             return ' '.join(mnemonic.mn_encode(s))
 701         else:
 702             return s
 703
 704
 705
 706     def get_private_key(self, address, password):
 707         if self.is_watching_only():
 708             return []
 709
 710         # first check the provided password
 711         seed = self.get_seed(password)
 712
 713         out = []
 714         if address in self.imported_keys.keys():
 715             out.append( pw_decode( self.imported_keys[address], password ) )
 716         else:
 717             account, sequence = self.get_address_index(address)
 718             if account == 0:
 719                 pk = self.accounts[account].get_private_key(seed, sequence)
 720                 out.append(pk)
 721                 return out
 722
 723             # assert address == self.accounts[account].get_address(*sequence)
 724             rs = self.rebase_sequence( account, sequence)
 725             for root, public_sequence in rs:
 726
 727                 if root not in self.master_private_keys.keys(): continue
 728                 master_k = self.get_master_private_key(root, password)
 729                 master_c, _, _ = self.master_public_keys[root]
 730                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 731                 out.append(pk)
 732
 733         return out
 734
 735
 736     def add_keypairs_from_wallet(self, tx, keypairs, password):
 737         for txin in tx.inputs:
 738             address = txin['address']
 739             if not self.is_mine(address):
 740                 continue
 741             private_keys = self.get_private_key(address, password)
 742             for sec in private_keys:
 743                 pubkey = public_key_from_private_key(sec)
 744                 keypairs[ pubkey ] = sec
 745                 if address in self.imported_keys.keys():
 746                     txin['redeemPubkey'] = pubkey
 747
 748
 749     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 750         # first check the provided password
 751         seed = self.get_seed(password)
 752
 753         for txin in tx.inputs:
 754             keyid = txin.get('KeyID')
 755             if keyid:
 756
 757                 if self.seed_version == 4:
 758                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 759                     if not m: continue
 760                     mpk = m.group(1)
 761                     if mpk != self.storage.get('master_public_key'): continue
 762                     for_change = int(m.group(2))
 763                     num = int(m.group(3))
 764                     account = self.accounts[0]
 765                     addr = account.get_address(for_change, num)
 766                     txin['address'] = addr # fixme: side effect
 767                     pk = account.get_private_key(seed, (for_change, num))
 768                     pubkey = public_key_from_private_key(pk)
 769                     keypairs[pubkey] = pk
 770                     continue
 771
 772
 773                 roots = []
 774                 for s in keyid.split('&'):
 775                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 776                     if not m: continue
 777                     c = m.group(1)
 778                     K = m.group(2)
 779                     sequence = m.group(3)
 780                     root = self.find_root_by_master_key(c,K)
 781                     if not root: continue
 782                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 783                     root = root + '%d'%sequence[0]
 784                     sequence = sequence[1:]
 785                     roots.append((root,sequence))
 786
 787                 account_id = " & ".join( map(lambda x:x[0], roots) )
 788                 account = self.accounts.get(account_id)
 789                 if not account: continue
 790                 addr = account.get_address(*sequence)
 791                 txin['address'] = addr # fixme: side effect
 792                 pk = self.get_private_key(addr, password)
 793                 for sec in pk:
 794                     pubkey = public_key_from_private_key(sec)
 795                     keypairs[pubkey] = sec
 796
 797
 798
 799     def signrawtransaction(self, tx, input_info, private_keys, password):
 800
 801         # check that the password is correct
 802         seed = self.get_seed(password)
 803
 804         # add input info
 805         tx.add_input_info(input_info)
 806
 807         # add redeem script for coins that are in the wallet
 808         # FIXME: add redeemPubkey too!
 809         unspent_coins = self.get_unspent_coins()
 810         for txin in tx.inputs:
 811             for item in unspent_coins:
 812                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 813                     print_error( "tx input is in unspent coins" )
 814                     txin['scriptPubKey'] = item['scriptPubKey']
 815                     account, sequence = self.get_address_index(item['address'])
 816                     if account != -1:
 817                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 818                         print_error("added redeemScript", txin['redeemScript'])
 819                     break
 820
 821
 822         # build a list of public/private keys
 823         keypairs = {}
 824
 825         # add private keys from parameter
 826         for sec in private_keys:
 827             pubkey = public_key_from_private_key(sec)
 828             keypairs[ pubkey ] = sec
 829
 830         # add private_keys from KeyID
 831         self.add_keypairs_from_KeyID(tx, keypairs, password)
 832
 833         # add private keys from wallet
 834         self.add_keypairs_from_wallet(tx, keypairs, password)
 835         self.sign_transaction(tx, keypairs)
 836
 837
 838     def sign_message(self, address, message, password):
 839         keys = self.get_private_key(address, password)
 840         assert len(keys) == 1
 841         sec = keys[0]
 842         key = regenerate_key(sec)
 843         compressed = is_compressed(sec)
 844         return key.sign_message(message, compressed, address)
 845
 846
 847     def change_gap_limit(self, value):
 848         if value >= self.gap_limit:
 849             self.gap_limit = value
 850             self.storage.put('gap_limit', self.gap_limit, True)
 851             #self.interface.poke('synchronizer')
 852             return True
 853
 854         elif value >= self.min_acceptable_gap():
 855             for key, account in self.accounts.items():
 856                 addresses = account[0]
 857                 k = self.num_unused_trailing_addresses(addresses)
 858                 n = len(addresses) - k + value
 859                 addresses = addresses[0:n]
 860                 self.accounts[key][0] = addresses
 861
 862             self.gap_limit = value
 863             self.storage.put('gap_limit', self.gap_limit, True)
 864             self.save_accounts()
 865             return True
 866         else:
 867             return False
 868
 869     def num_unused_trailing_addresses(self, addresses):
 870         k = 0
 871         for a in addresses[::-1]:
 872             if self.history.get(a):break
 873             k = k + 1
 874         return k
 875
 876     def min_acceptable_gap(self):
 877         # fixme: this assumes wallet is synchronized
 878         n = 0
 879         nmax = 0
 880
 881         for account in self.accounts.values():
 882             addresses = account.get_addresses(0)
 883             k = self.num_unused_trailing_addresses(addresses)
 884             for a in addresses[0:-k]:
 885                 if self.history.get(a):
 886                     n = 0
 887                 else:
 888                     n += 1
 889                     if n > nmax: nmax = n
 890         return nmax + 1
 891
 892
 893     def address_is_old(self, address):
 894         age = -1
 895         h = self.history.get(address, [])
 896         if h == ['*']:
 897             return True
 898         for tx_hash, tx_height in h:
 899             if tx_height == 0:
 900                 tx_age = 0
 901             else:
 902                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 903             if tx_age > age:
 904                 age = tx_age
 905         return age > 2
 906
 907
 908     def synchronize_sequence(self, account, for_change):
 909         limit = self.gap_limit_for_change if for_change else self.gap_limit
 910         new_addresses = []
 911         while True:
 912             addresses = account.get_addresses(for_change)
 913             if len(addresses) < limit:
 914                 address = account.create_new_address(for_change)
 915                 self.history[address] = []
 916                 new_addresses.append( address )
 917                 continue
 918
 919             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 920                 break
 921             else:
 922                 address = account.create_new_address(for_change)
 923                 self.history[address] = []
 924                 new_addresses.append( address )
 925
 926         return new_addresses
 927
 928
 929
 930     def create_pending_accounts(self):
 931         for account_type in ['1','2of2','2of3']:
 932             if not self.has_master_public_keys(account_type):
 933                 continue
 934             k, a = self.new_account_address(account_type)
 935             if self.address_is_old(a):
 936                 print_error( "creating account", a )
 937                 self.create_account(account_type)
 938                 self.next_addresses.pop(k)
 939
 940
 941     def synchronize_account(self, account):
 942         new = []
 943         new += self.synchronize_sequence(account, 0)
 944         new += self.synchronize_sequence(account, 1)
 945         return new
 946
 947
 948     def synchronize(self):
 949         if self.master_public_keys:
 950             self.create_pending_accounts()
 951         new = []
 952         for account in self.accounts.values():
 953             new += self.synchronize_account(account)
 954         if new:
 955             self.save_accounts()
 956             self.storage.put('addr_history', self.history, True)
 957         return new
 958
 959
 960     def is_found(self):
 961         return self.history.values() != [[]] * len(self.history)
 962
 963
 964     def add_contact(self, address, label=None):
 965         self.addressbook.append(address)
 966         self.storage.put('contacts', self.addressbook, True)
 967         if label:
 968             self.set_label(address, label)
 969
 970
 971     def delete_contact(self, addr):
 972         if addr in self.addressbook:
 973             self.addressbook.remove(addr)
 974             self.storage.put('addressbook', self.addressbook, True)
 975
 976
 977     def fill_addressbook(self):
 978         for tx_hash, tx in self.transactions.items():
 979             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 980             if is_send:
 981                 for addr, v in tx.outputs:
 982                     if not self.is_mine(addr) and addr not in self.addressbook:
 983                         self.addressbook.append(addr)
 984         # redo labels
 985         # self.update_tx_labels()
 986
 987     def get_num_tx(self, address):
 988         n = 0
 989         for tx in self.transactions.values():
 990             if address in map(lambda x:x[0], tx.outputs): n += 1
 991         return n
 992
 993
 994     def get_address_flags(self, addr):
 995         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 996         flags += "F" if addr in self.frozen_addresses else "-"
 997         return flags
 998
 999
1000     def get_tx_value(self, tx, account=None):
1001         domain = self.get_account_addresses(account)
1002         return tx.get_value(domain, self.prevout_values)
1003
1004
1005     def update_tx_outputs(self, tx_hash):
1006         tx = self.transactions.get(tx_hash)
1007
1008         for i, (addr, value) in enumerate(tx.outputs):
1009             key = tx_hash+ ':%d'%i
1010             self.prevout_values[key] = value
1011
1012         for item in tx.inputs:
1013             if self.is_mine(item.get('address')):
1014                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1015                 self.spent_outputs.append(key)
1016
1017
1018     def get_addr_balance(self, address):
1019         assert self.is_mine(address)
1020         h = self.history.get(address,[])
1021         if h == ['*']: return 0,0
1022         c = u = 0
1023         received_coins = []   # list of coins received at address
1024
1025         for tx_hash, tx_height in h:
1026             tx = self.transactions.get(tx_hash)
1027             if not tx: continue
1028
1029             for i, (addr, value) in enumerate(tx.outputs):
1030                 if addr == address:
1031                     key = tx_hash + ':%d'%i
1032                     received_coins.append(key)
1033
1034         for tx_hash, tx_height in h:
1035             tx = self.transactions.get(tx_hash)
1036             if not tx: continue
1037             v = 0
1038
1039             for item in tx.inputs:
1040                 addr = item.get('address')
1041                 if addr == address:
1042                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1043                     value = self.prevout_values.get( key )
1044                     if key in received_coins:
1045                         v -= value
1046
1047             for i, (addr, value) in enumerate(tx.outputs):
1048                 key = tx_hash + ':%d'%i
1049                 if addr == address:
1050                     v += value
1051
1052             if tx_height:
1053                 c += v
1054             else:
1055                 u += v
1056         return c, u
1057
1058
1059     def get_account_name(self, k):
1060         if k == 0:
1061             if self.seed_version == 4:
1062                 name = 'Main account'
1063             else:
1064                 name = 'Old account'
1065         else:
1066             default = "Unnamed account"
1067             m = re.match("m/0'/(\d+)", k)
1068             if m:
1069                 num = m.group(1)
1070                 if num == '0':
1071                     default = "Main account"
1072                 else:
1073                     default = "Account %s"%num
1074
1075             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1076             if m:
1077                 num = m.group(1)
1078                 default = "2of2 account %s"%num
1079             name = self.labels.get(k, default)
1080
1081         return name
1082
1083     def get_account_names(self):
1084         accounts = {}
1085         for k, account in self.accounts.items():
1086             accounts[k] = self.get_account_name(k)
1087         if self.imported_keys:
1088             accounts[-1] = 'Imported keys'
1089         return accounts
1090
1091     def get_account_addresses(self, a, include_change=True):
1092         if a is None:
1093             o = self.addresses(True)
1094         elif a == -1:
1095             o = self.imported_keys.keys()
1096         else:
1097             ac = self.accounts[a]
1098             o = ac.get_addresses(0)
1099             if include_change: o += ac.get_addresses(1)
1100         return o
1101
1102     def get_imported_balance(self):
1103         return self.get_balance(self.imported_keys.keys())
1104
1105     def get_account_balance(self, account):
1106         return self.get_balance(self.get_account_addresses(account))
1107
1108     def get_frozen_balance(self):
1109         return self.get_balance(self.frozen_addresses)
1110
1111     def get_balance(self, domain=None):
1112         if domain is None: domain = self.addresses(True)
1113         cc = uu = 0
1114         for addr in domain:
1115             c, u = self.get_addr_balance(addr)
1116             cc += c
1117             uu += u
1118         return cc, uu
1119
1120
1121     def get_unspent_coins(self, domain=None):
1122         coins = []
1123         if domain is None: domain = self.addresses(True)
1124         for addr in domain:
1125             h = self.history.get(addr, [])
1126             if h == ['*']: continue
1127             for tx_hash, tx_height in h:
1128                 tx = self.transactions.get(tx_hash)
1129                 if tx is None: raise Exception("Wallet not synchronized")
1130                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1131                 for o in tx.d.get('outputs'):
1132                     output = o.copy()
1133                     if output.get('address') != addr: continue
1134                     key = tx_hash + ":%d" % output.get('prevout_n')
1135                     if key in self.spent_outputs: continue
1136                     output['prevout_hash'] = tx_hash
1137                     output['height'] = tx_height
1138                     output['coinbase'] = is_coinbase
1139                     coins.append((tx_height, output))
1140
1141         # sort by age
1142         if coins:
1143             coins = sorted(coins)
1144             if coins[-1][0] != 0:
1145                 while coins[0][0] == 0:
1146                     coins = coins[1:] + [ coins[0] ]
1147         return [x[1] for x in coins]
1148
1149
1150     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1151         """ todo: minimize tx size """
1152         total = 0
1153         fee = self.fee if fixed_fee is None else fixed_fee
1154         if domain is None:
1155             domain = self.addresses(True)
1156
1157         for i in self.frozen_addresses:
1158             if i in domain: domain.remove(i)
1159
1160         coins = self.get_unspent_coins(domain)
1161         inputs = []
1162
1163         for item in coins:
1164             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1165                 continue
1166             addr = item.get('address')
1167             v = item.get('value')
1168             total += v
1169             inputs.append(item)
1170             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1171             if total >= amount + fee: break
1172         else:
1173             inputs = []
1174
1175         return inputs, total, fee
1176
1177
1178     def set_fee(self, fee):
1179         if self.fee != fee:
1180             self.fee = fee
1181             self.storage.put('fee_per_kb', self.fee, True)
1182
1183     def estimated_fee(self, inputs):
1184         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1185         fee = self.fee * int(round(estimated_size/1024.))
1186         if fee == 0: fee = self.fee
1187         return fee
1188
1189
1190     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1191         "add change to a transaction"
1192         change_amount = total - ( amount + fee )
1193         if change_amount > DUST_THRESHOLD:
1194             if not change_addr:
1195
1196                 # send change to one of the accounts involved in the tx
1197                 address = inputs[0].get('address')
1198                 account, _ = self.get_address_index(address)
1199
1200                 if not self.use_change or account == -1:
1201                     change_addr = inputs[-1]['address']
1202                 else:
1203                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1204
1205             # Insert the change output at a random position in the outputs
1206             posn = random.randint(0, len(outputs))
1207             outputs[posn:posn] = [( change_addr,  change_amount)]
1208         return outputs
1209
1210
1211     def get_history(self, address):
1212         with self.lock:
1213             return self.history.get(address)
1214
1215
1216     def get_status(self, h):
1217         if not h: return None
1218         if h == ['*']: return '*'
1219         status = ''
1220         for tx_hash, height in h:
1221             status += tx_hash + ':%d:' % height
1222         return hashlib.sha256( status ).digest().encode('hex')
1223
1224
1225     def receive_tx_callback(self, tx_hash, tx, tx_height):
1226
1227         with self.transaction_lock:
1228             self.add_extra_addresses(tx)
1229             if not self.check_new_tx(tx_hash, tx):
1230                 # may happen due to pruning
1231                 print_error("received transaction that is no longer referenced in history", tx_hash)
1232                 return
1233             self.transactions[tx_hash] = tx
1234             self.network.interface.pending_transactions_for_notifications.append(tx)
1235             self.save_transactions()
1236             if self.verifier and tx_height>0:
1237                 self.verifier.add(tx_hash, tx_height)
1238             self.update_tx_outputs(tx_hash)
1239
1240
1241     def save_transactions(self):
1242         tx = {}
1243         for k,v in self.transactions.items():
1244             tx[k] = str(v)
1245         self.storage.put('transactions', tx, True)
1246
1247     def receive_history_callback(self, addr, hist):
1248
1249         if not self.check_new_history(addr, hist):
1250             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1251
1252         with self.lock:
1253             self.history[addr] = hist
1254             self.storage.put('addr_history', self.history, True)
1255
1256         if hist != ['*']:
1257             for tx_hash, tx_height in hist:
1258                 if tx_height>0:
1259                     # add it in case it was previously unconfirmed
1260                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1261
1262
1263     def get_tx_history(self, account=None):
1264         if not self.verifier:
1265             return []
1266
1267         with self.transaction_lock:
1268             history = self.transactions.items()
1269             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1270             result = []
1271
1272             balance = 0
1273             for tx_hash, tx in history:
1274                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1275                 if v is not None: balance += v
1276
1277             c, u = self.get_account_balance(account)
1278
1279             if balance != c+u:
1280                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1281
1282             balance = c + u - balance
1283             for tx_hash, tx in history:
1284                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1285                 if not is_relevant:
1286                     continue
1287                 if value is not None:
1288                     balance += value
1289
1290                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1291                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1292
1293         return result
1294
1295
1296     def get_label(self, tx_hash):
1297         label = self.labels.get(tx_hash)
1298         is_default = (label == '') or (label is None)
1299         if is_default: label = self.get_default_label(tx_hash)
1300         return label, is_default
1301
1302
1303     def get_default_label(self, tx_hash):
1304         tx = self.transactions.get(tx_hash)
1305         default_label = ''
1306         if tx:
1307             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1308             if is_mine:
1309                 for o in tx.outputs:
1310                     o_addr, _ = o
1311                     if not self.is_mine(o_addr):
1312                         try:
1313                             default_label = self.labels[o_addr]
1314                         except KeyError:
1315                             default_label = o_addr
1316                         break
1317                 else:
1318                     default_label = '(internal)'
1319             else:
1320                 for o in tx.outputs:
1321                     o_addr, _ = o
1322                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1323                         break
1324                 else:
1325                     for o in tx.outputs:
1326                         o_addr, _ = o
1327                         if self.is_mine(o_addr):
1328                             break
1329                     else:
1330                         o_addr = None
1331
1332                 if o_addr:
1333                     dest_label = self.labels.get(o_addr)
1334                     try:
1335                         default_label = self.labels[o_addr]
1336                     except KeyError:
1337                         default_label = o_addr
1338
1339         return default_label
1340
1341
1342     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1343         for address, x in outputs:
1344             assert is_valid(address)
1345         amount = sum( map(lambda x:x[1], outputs) )
1346         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1347         if not inputs:
1348             raise ValueError("Not enough funds")
1349         self.add_input_info(inputs)
1350         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1351         return Transaction.from_io(inputs, outputs)
1352
1353
1354     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1355         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1356         keypairs = {}
1357         self.add_keypairs_from_wallet(tx, keypairs, password)
1358         if keypairs:
1359             self.sign_transaction(tx, keypairs)
1360         return tx
1361
1362
1363     def add_input_info(self, inputs):
1364         for txin in inputs:
1365             address = txin['address']
1366             if address in self.imported_keys.keys():
1367                 continue
1368             account, sequence = self.get_address_index(address)
1369             txin['KeyID'] = self.get_keyID(account, sequence)
1370             redeemScript = self.accounts[account].redeem_script(sequence)
1371             if redeemScript:
1372                 txin['redeemScript'] = redeemScript
1373             else:
1374                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1375
1376
1377     def sign_transaction(self, tx, keypairs):
1378         tx.sign(keypairs)
1379         run_hook('sign_transaction', tx)
1380
1381
1382     def sendtx(self, tx):
1383         # synchronous
1384         h = self.send_tx(tx)
1385         self.tx_event.wait()
1386         return self.receive_tx(h, tx)
1387
1388     def send_tx(self, tx):
1389         # asynchronous
1390         self.tx_event.clear()
1391         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1392         return tx.hash()
1393
1394     def on_broadcast(self, i, r):
1395         self.tx_result = r.get('result')
1396         self.tx_event.set()
1397
1398     def receive_tx(self, tx_hash, tx):
1399         out = self.tx_result
1400         if out != tx_hash:
1401             return False, "error: " + out
1402         run_hook('receive_tx', tx, self)
1403         return True, out
1404
1405
1406
1407     def update_password(self, old_password, new_password):
1408         if new_password == '': new_password = None
1409         decoded = self.get_seed(old_password)
1410         self.seed = pw_encode( decoded, new_password)
1411         self.storage.put('seed', self.seed, True)
1412         self.use_encryption = (new_password != None)
1413         self.storage.put('use_encryption', self.use_encryption,True)
1414         for k in self.imported_keys.keys():
1415             a = self.imported_keys[k]
1416             b = pw_decode(a, old_password)
1417             c = pw_encode(b, new_password)
1418             self.imported_keys[k] = c
1419         self.storage.put('imported_keys', self.imported_keys, True)
1420
1421         for k, v in self.master_private_keys.items():
1422             b = pw_decode(v, old_password)
1423             c = pw_encode(b, new_password)
1424             self.master_private_keys[k] = c
1425         self.storage.put('master_private_keys', self.master_private_keys, True)
1426
1427
1428     def freeze(self,addr):
1429         if self.is_mine(addr) and addr not in self.frozen_addresses:
1430             self.frozen_addresses.append(addr)
1431             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1432             return True
1433         else:
1434             return False
1435
1436     def unfreeze(self,addr):
1437         if self.is_mine(addr) and addr in self.frozen_addresses:
1438             self.frozen_addresses.remove(addr)
1439             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1440             return True
1441         else:
1442             return False
1443
1444
1445     def set_verifier(self, verifier):
1446         self.verifier = verifier
1447
1448         # review transactions that are in the history
1449         for addr, hist in self.history.items():
1450             if hist == ['*']: continue
1451             for tx_hash, tx_height in hist:
1452                 if tx_height>0:
1453                     # add it in case it was previously unconfirmed
1454                     self.verifier.add(tx_hash, tx_height)
1455
1456
1457         # if we are on a pruning server, remove unverified transactions
1458         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1459         for tx_hash in self.transactions.keys():
1460             if tx_hash not in vr:
1461                 self.transactions.pop(tx_hash)
1462
1463
1464
1465     def check_new_history(self, addr, hist):
1466
1467         # check that all tx in hist are relevant
1468         if hist != ['*']:
1469             for tx_hash, height in hist:
1470                 tx = self.transactions.get(tx_hash)
1471                 if not tx: continue
1472                 if not tx.has_address(addr):
1473                     return False
1474
1475         # check that we are not "orphaning" a transaction
1476         old_hist = self.history.get(addr,[])
1477         if old_hist == ['*']: return True
1478
1479         for tx_hash, height in old_hist:
1480             if tx_hash in map(lambda x:x[0], hist): continue
1481             found = False
1482             for _addr, _hist in self.history.items():
1483                 if _addr == addr: continue
1484                 if _hist == ['*']: continue
1485                 _tx_hist = map(lambda x:x[0], _hist)
1486                 if tx_hash in _tx_hist:
1487                     found = True
1488                     break
1489
1490             if not found:
1491                 tx = self.transactions.get(tx_hash)
1492                 # tx might not be there
1493                 if not tx: continue
1494
1495                 # already verified?
1496                 if self.verifier.get_height(tx_hash):
1497                     continue
1498                 # unconfirmed tx
1499                 print_error("new history is orphaning transaction:", tx_hash)
1500                 # check that all outputs are not mine, request histories
1501                 ext_requests = []
1502                 for _addr, _v in tx.outputs:
1503                     # assert not self.is_mine(_addr)
1504                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1505
1506                 ext_h = self.network.synchronous_get(ext_requests)
1507                 print_error("sync:", ext_requests, ext_h)
1508                 height = None
1509                 for h in ext_h:
1510                     if h == ['*']: continue
1511                     for item in h:
1512                         if item.get('tx_hash') == tx_hash:
1513                             height = item.get('height')
1514                 if height:
1515                     print_error("found height for", tx_hash, height)
1516                     self.verifier.add(tx_hash, height)
1517                 else:
1518                     print_error("removing orphaned tx from history", tx_hash)
1519                     self.transactions.pop(tx_hash)
1520
1521         return True
1522
1523
1524
1525     def check_new_tx(self, tx_hash, tx):
1526         # 1 check that tx is referenced in addr_history.
1527         addresses = []
1528         for addr, hist in self.history.items():
1529             if hist == ['*']:continue
1530             for txh, height in hist:
1531                 if txh == tx_hash:
1532                     addresses.append(addr)
1533
1534         if not addresses:
1535             return False
1536
1537         # 2 check that referencing addresses are in the tx
1538         for addr in addresses:
1539             if not tx.has_address(addr):
1540                 return False
1541
1542         return True
1543
1544
1545     def start_threads(self, network):
1546         from verifier import TxVerifier
1547         self.network = network
1548         if self.network is not None:
1549             self.verifier = TxVerifier(self.network, self.storage)
1550             self.verifier.start()
1551             self.set_verifier(self.verifier)
1552             self.synchronizer = WalletSynchronizer(self, network)
1553             self.synchronizer.start()
1554         else:
1555             self.verifier = None
1556             self.synchronizer =None
1557
1558     def stop_threads(self):
1559         if self.network:
1560             self.verifier.stop()
1561             self.synchronizer.stop()
1562
1563
1564     def restore(self, callback):
1565         from i18n import _
1566         def wait_for_wallet():
1567             self.set_up_to_date(False)
1568             while not self.is_up_to_date():
1569                 msg = "%s\n%s %d\n%s %.1f"%(
1570                     _("Please wait..."),
1571                     _("Addresses generated:"),
1572                     len(self.addresses(True)),
1573                     _("Kilobytes received:"),
1574                     self.network.interface.bytes_received/1024.)
1575
1576                 apply(callback, (msg,))
1577                 time.sleep(0.1)
1578
1579         def wait_for_network():
1580             while not self.network.is_connected():
1581                 msg = "%s \n" % (_("Connecting..."))
1582                 apply(callback, (msg,))
1583                 time.sleep(0.1)
1584
1585         # wait until we are connected, because the user might have selected another server
1586         if self.network:
1587             wait_for_network()
1588             wait_for_wallet()
1589         else:
1590             self.synchronize()
1591
1592         self.fill_addressbook()
1593
1594
1595
1596
1597 class WalletSynchronizer(threading.Thread):
1598
1599
1600     def __init__(self, wallet, network):
1601         threading.Thread.__init__(self)
1602         self.daemon = True
1603         self.wallet = wallet
1604         self.network = network
1605         self.was_updated = True
1606         self.running = False
1607         self.lock = threading.Lock()
1608         self.queue = Queue.Queue()
1609
1610     def stop(self):
1611         with self.lock: self.running = False
1612
1613     def is_running(self):
1614         with self.lock: return self.running
1615
1616
1617     def subscribe_to_addresses(self, addresses):
1618         messages = []
1619         for addr in addresses:
1620             messages.append(('blockchain.address.subscribe', [addr]))
1621         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1622
1623
1624     def run(self):
1625         with self.lock:
1626             self.running = True
1627
1628         while self.is_running():
1629
1630             if not self.network.is_connected():
1631                 self.network.wait_until_connected()
1632
1633             self.run_interface()
1634
1635
1636     def run_interface(self):
1637
1638         print_error("synchronizer: connected to", self.network.main_server())
1639
1640         requested_tx = []
1641         missing_tx = []
1642         requested_histories = {}
1643
1644         # request any missing transactions
1645         for history in self.wallet.history.values():
1646             if history == ['*']: continue
1647             for tx_hash, tx_height in history:
1648                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1649                     missing_tx.append( (tx_hash, tx_height) )
1650
1651         if missing_tx:
1652             print_error("missing tx", missing_tx)
1653
1654         # subscriptions
1655         self.subscribe_to_addresses(self.wallet.addresses(True))
1656
1657         while self.is_running():
1658             # 1. create new addresses
1659             new_addresses = self.wallet.synchronize()
1660
1661             # request missing addresses
1662             if new_addresses:
1663                 self.subscribe_to_addresses(new_addresses)
1664
1665             # request missing transactions
1666             for tx_hash, tx_height in missing_tx:
1667                 if (tx_hash, tx_height) not in requested_tx:
1668                     self.network.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1669                     requested_tx.append( (tx_hash, tx_height) )
1670             missing_tx = []
1671
1672             # detect if situation has changed
1673             if self.network.is_up_to_date() and self.queue.empty():
1674                 if not self.wallet.is_up_to_date():
1675                     self.wallet.set_up_to_date(True)
1676                     self.was_updated = True
1677             else:
1678                 if self.wallet.is_up_to_date():
1679                     self.wallet.set_up_to_date(False)
1680                     self.was_updated = True
1681
1682             if self.was_updated:
1683                 self.network.trigger_callback('updated')
1684                 self.was_updated = False
1685
1686             # 2. get a response
1687             try:
1688                 r = self.queue.get(block=True, timeout=1)
1689             except Queue.Empty:
1690                 continue
1691
1692             # see if it changed
1693             #if interface != self.network.interface:
1694             #    break
1695
1696             if not r:
1697                 continue
1698
1699             # 3. handle response
1700             method = r['method']
1701             params = r['params']
1702             result = r.get('result')
1703             error = r.get('error')
1704             if error:
1705                 print "error", r
1706                 continue
1707
1708             if method == 'blockchain.address.subscribe':
1709                 addr = params[0]
1710                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1711                     if requested_histories.get(addr) is None:
1712                         self.network.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1713                         requested_histories[addr] = result
1714
1715             elif method == 'blockchain.address.get_history':
1716                 addr = params[0]
1717                 print_error("receiving history", addr, result)
1718                 if result == ['*']:
1719                     assert requested_histories.pop(addr) == '*'
1720                     self.wallet.receive_history_callback(addr, result)
1721                 else:
1722                     hist = []
1723                     # check that txids are unique
1724                     txids = []
1725                     for item in result:
1726                         tx_hash = item['tx_hash']
1727                         if tx_hash not in txids:
1728                             txids.append(tx_hash)
1729                             hist.append( (tx_hash, item['height']) )
1730
1731                     if len(hist) != len(result):
1732                         raise Exception("error: server sent history with non-unique txid", result)
1733
1734                     # check that the status corresponds to what was announced
1735                     rs = requested_histories.pop(addr)
1736                     if self.wallet.get_status(hist) != rs:
1737                         raise Exception("error: status mismatch: %s"%addr)
1738
1739                     # store received history
1740                     self.wallet.receive_history_callback(addr, hist)
1741
1742                     # request transactions that we don't have
1743                     for tx_hash, tx_height in hist:
1744                         if self.wallet.transactions.get(tx_hash) is None:
1745                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1746                                 missing_tx.append( (tx_hash, tx_height) )
1747
1748             elif method == 'blockchain.transaction.get':
1749                 tx_hash = params[0]
1750                 tx_height = params[1]
1751                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1752                 tx = Transaction(result)
1753                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1754                 self.was_updated = True
1755                 requested_tx.remove( (tx_hash, tx_height) )
1756                 print_error("received tx:", tx_hash, len(tx.raw))
1757
1758             else:
1759                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1760
1761             if self.was_updated and not requested_tx:
1762                 self.network.trigger_callback('updated')
1763                 # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1764                 self.network.trigger_callback("new_transaction")
1765                 self.was_updated = False