lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32 import math
  33
  34 from util import print_msg, print_error, format_satoshis
  35 from bitcoin import *
  36 from account import *
  37 from transaction import Transaction
  38 from plugins import run_hook
  39
  40 COINBASE_MATURITY = 100
  41 DUST_THRESHOLD = 5430
  42
  43 # AES encryption
  44 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  45 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  46
  47 def pw_encode(s, password):
  48     if password:
  49         secret = Hash(password)
  50         return EncodeAES(secret, s)
  51     else:
  52         return s
  53
  54 def pw_decode(s, password):
  55     if password is not None:
  56         secret = Hash(password)
  57         try:
  58             d = DecodeAES(secret, s)
  59         except Exception:
  60             raise Exception('Invalid password')
  61         return d
  62     else:
  63         return s
  64
  65
  66
  67
  68
  69 from version import *
  70
  71
  72 class WalletStorage:
  73
  74     def __init__(self, config):
  75         self.lock = threading.Lock()
  76         self.data = {}
  77         self.file_exists = False
  78         self.path = self.init_path(config)
  79         print_error( "wallet path", self.path )
  80         if self.path:
  81             self.read(self.path)
  82
  83
  84     def init_path(self, config):
  85         """Set the path of the wallet."""
  86
  87         # command line -w option
  88         path = config.get('wallet_path')
  89         if path:
  90             return path
  91
  92         # path in config file
  93         path = config.get('default_wallet_path')
  94         if path:
  95             return path
  96
  97         # default path
  98         dirpath = os.path.join(config.path, "wallets")
  99         if not os.path.exists(dirpath):
 100             os.mkdir(dirpath)
 101
 102         new_path = os.path.join(config.path, "wallets", "default_wallet")
 103
 104         # default path in pre 1.9 versions
 105         old_path = os.path.join(config.path, "electrum.dat")
 106         if os.path.exists(old_path) and not os.path.exists(new_path):
 107             os.rename(old_path, new_path)
 108
 109         return new_path
 110
 111
 112     def read(self, path):
 113         """Read the contents of the wallet file."""
 114         try:
 115             with open(self.path, "r") as f:
 116                 data = f.read()
 117         except IOError:
 118             return
 119         try:
 120             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 121         except Exception:
 122             raise IOError("Cannot read wallet file.")
 123
 124         self.data = d
 125         self.file_exists = True
 126
 127
 128     def get(self, key, default=None):
 129         v = self.data.get(key)
 130         if v is None:
 131             v = default
 132         return v
 133
 134     def put(self, key, value, save = True):
 135
 136         with self.lock:
 137             if value is not None:
 138                 self.data[key] = value
 139             else:
 140                 self.data.pop(key)
 141             if save:
 142                 self.write()
 143
 144     def write(self):
 145         s = repr(self.data)
 146         f = open(self.path,"w")
 147         f.write( s )
 148         f.close()
 149         if 'ANDROID_DATA' not in os.environ:
 150             import stat
 151             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 152
 153
 154 class Wallet:
 155
 156     def __init__(self, storage):
 157
 158         self.storage = storage
 159         self.electrum_version = ELECTRUM_VERSION
 160         self.gap_limit_for_change = 3 # constant
 161
 162         # saved fields
 163         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 164
 165         self.gap_limit             = storage.get('gap_limit', 5)
 166         self.use_change            = storage.get('use_change',True)
 167         self.use_encryption        = storage.get('use_encryption', False)
 168         self.seed                  = storage.get('seed', '')               # encrypted
 169         self.labels                = storage.get('labels', {})
 170         self.frozen_addresses      = storage.get('frozen_addresses',[])
 171         self.addressbook           = storage.get('contacts', [])
 172
 173         self.imported_keys         = storage.get('imported_keys',{})
 174         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 175
 176         self.fee                   = int(storage.get('fee_per_kb',20000))
 177
 178         self.master_public_keys = storage.get('master_public_keys',{})
 179         self.master_private_keys = storage.get('master_private_keys', {})
 180
 181         self.next_addresses = storage.get('next_addresses',{})
 182
 183         if self.seed_version not in [4, 6]:
 184             msg = "This wallet seed is not supported."
 185             if self.seed_version in [5]:
 186                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 187                 print msg
 188                 sys.exit(1)
 189
 190         # This attribute is set when wallet.start_threads is called.
 191         self.synchronizer = None
 192
 193         self.load_accounts()
 194
 195         self.transactions = {}
 196         tx_list = self.storage.get('transactions',{})
 197         for k,v in tx_list.items():
 198             try:
 199                 tx = Transaction(v)
 200             except Exception:
 201                 print_msg("Warning: Cannot deserialize transactions. skipping")
 202                 continue
 203
 204             self.add_extra_addresses(tx)
 205             self.transactions[k] = tx
 206
 207         for h,tx in self.transactions.items():
 208             if not self.check_new_tx(h, tx):
 209                 print_error("removing unreferenced tx", h)
 210                 self.transactions.pop(h)
 211
 212
 213         # not saved
 214         self.prevout_values = {}     # my own transaction outputs
 215         self.spent_outputs = []
 216
 217         # spv
 218         self.verifier = None
 219
 220         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 221         # interface.is_up_to_date() returns true when all requests have been answered and processed
 222         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 223
 224         self.up_to_date = False
 225         self.lock = threading.Lock()
 226         self.transaction_lock = threading.Lock()
 227         self.tx_event = threading.Event()
 228
 229         for tx_hash, tx in self.transactions.items():
 230             self.update_tx_outputs(tx_hash)
 231
 232
 233     def add_extra_addresses(self, tx):
 234         h = tx.hash()
 235         # find the address corresponding to pay-to-pubkey inputs
 236         tx.add_extra_addresses(self.transactions)
 237         for o in tx.d.get('outputs'):
 238             if o.get('is_pubkey'):
 239                 for tx2 in self.transactions.values():
 240                     tx2.add_extra_addresses({h:tx})
 241
 242
 243
 244
 245     def set_up_to_date(self,b):
 246         with self.lock: self.up_to_date = b
 247
 248
 249     def is_up_to_date(self):
 250         with self.lock: return self.up_to_date
 251
 252
 253     def update(self):
 254         self.up_to_date = False
 255         while not self.is_up_to_date():
 256             time.sleep(0.1)
 257
 258
 259     def import_key(self, sec, password):
 260         # check password
 261         seed = self.get_seed(password)
 262         try:
 263             address = address_from_private_key(sec)
 264         except Exception:
 265             raise Exception('Invalid private key')
 266
 267         if self.is_mine(address):
 268             raise Exception('Address already in wallet')
 269
 270         # store the originally requested keypair into the imported keys table
 271         self.imported_keys[address] = pw_encode(sec, password )
 272         self.storage.put('imported_keys', self.imported_keys, True)
 273         if self.synchronizer:
 274             self.synchronizer.subscribe_to_addresses([address])
 275         return address
 276
 277     def delete_imported_key(self, addr):
 278         if addr in self.imported_keys:
 279             self.imported_keys.pop(addr)
 280             self.storage.put('imported_keys', self.imported_keys, True)
 281
 282
 283     def make_seed(self):
 284         import mnemonic, ecdsa
 285         entropy = ecdsa.util.randrange( pow(2,160) )
 286         nonce = 0
 287         while True:
 288             ss = "%040x"%(entropy+nonce)
 289             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 290             # we keep only 13 words, that's approximately 139 bits of entropy
 291             words = mnemonic.mn_encode(s)[0:13]
 292             seed = ' '.join(words)
 293             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 294                 break  # this removes 12 bits of entropy
 295             nonce += 1
 296
 297         return seed
 298
 299
 300     def init_seed(self, seed):
 301         import mnemonic
 302
 303         if self.seed:
 304             raise Exception("a seed exists")
 305
 306         if not seed:
 307             self.seed = random_seed(128)
 308             self.seed_version = 4
 309             return
 310
 311         #if not seed:
 312         #    self.seed = self.make_seed()
 313         #    self.seed_version = SEED_VERSION
 314         #    return
 315
 316         # find out what kind of wallet we are
 317         try:
 318             seed.strip().decode('hex')
 319             self.seed_version = 4
 320             self.seed = str(seed)
 321             return
 322         except Exception:
 323             pass
 324
 325         words = seed.split()
 326         self.seed_version = 4
 327         self.seed = mnemonic.mn_decode(words)
 328
 329         #try:
 330         #    mnemonic.mn_decode(words)
 331         #    uses_electrum_words = True
 332         #except Exception:
 333         #    uses_electrum_words = False
 334         #
 335         #if uses_electrum_words and len(words) != 13:
 336         #    self.seed_version = 4
 337         #    self.seed = mnemonic.mn_decode(words)
 338         #else:
 339         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 340         #    self.seed_version = SEED_VERSION
 341         #    self.seed = seed
 342
 343
 344     def save_seed(self, password):
 345         if password:
 346             self.seed = pw_encode( self.seed, password)
 347             self.use_encryption = True
 348         self.storage.put('seed', self.seed, True)
 349         self.storage.put('seed_version', self.seed_version, True)
 350         self.storage.put('use_encryption', self.use_encryption,True)
 351         self.create_accounts(password)
 352
 353
 354     def create_watching_only_wallet(self, params):
 355         K0, c0 = params
 356         if not K0:
 357             return
 358
 359         if not c0:
 360             self.seed_version = 4
 361             self.storage.put('seed_version', self.seed_version, True)
 362             self.create_old_account(K0)
 363             return
 364
 365         cK0 = ""
 366         self.master_public_keys = {
 367             "m/0'/": (c0, K0, cK0),
 368             }
 369         self.storage.put('master_public_keys', self.master_public_keys, True)
 370         self.storage.put('seed_version', self.seed_version, True)
 371         self.create_account('1','Main account')
 372
 373
 374     def create_accounts(self, password):
 375         seed = pw_decode(self.seed, password)
 376
 377         if self.seed_version == 4:
 378             mpk = OldAccount.mpk_from_seed(seed)
 379             self.create_old_account(mpk)
 380         else:
 381             # create default account
 382             self.create_master_keys('1', password)
 383             self.create_account('1','Main account')
 384
 385
 386     def create_master_keys(self, account_type, password):
 387         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 388         if account_type == '1':
 389             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 390             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 391             self.master_private_keys["m/0'/"] = pw_encode(k0, password)
 392         elif account_type == '2of2':
 393             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 394             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 395             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 396             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 397             self.master_private_keys["m/1'/"] = pw_encode(k1, password)
 398             self.master_private_keys["m/2'/"] = pw_encode(k2, password)
 399         elif account_type == '2of3':
 400             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 401             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 402             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 403             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 404             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 405             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 406             self.master_private_keys["m/3'/"] = pw_encode(k3, password)
 407             self.master_private_keys["m/4'/"] = pw_encode(k4, password)
 408             self.master_private_keys["m/5'/"] = pw_encode(k5, password)
 409
 410         self.storage.put('master_public_keys', self.master_public_keys, True)
 411         self.storage.put('master_private_keys', self.master_private_keys, True)
 412
 413     def has_master_public_keys(self, account_type):
 414         if account_type == '1':
 415             return "m/0'/" in self.master_public_keys
 416         elif account_type == '2of2':
 417             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 418         elif account_type == '2of3':
 419             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 420
 421     def find_root_by_master_key(self, c, K):
 422         for key, v in self.master_public_keys.items():
 423             if key == "m/":continue
 424             cc, KK, _ = v
 425             if (c == cc) and (K == KK):
 426                 return key
 427
 428     def deseed_root(self, seed, password):
 429         # for safety, we ask the user to enter their seed
 430         assert seed == self.get_seed(password)
 431         self.seed = ''
 432         self.storage.put('seed', '', True)
 433
 434
 435     def deseed_branch(self, k):
 436         # check that parent has no seed
 437         assert self.seed == ''
 438         self.master_private_keys.pop(k)
 439         self.storage.put('master_private_keys', self.master_private_keys, True)
 440
 441     def is_watching_only(self):
 442         return (self.seed == '') and (self.master_private_keys == {})
 443
 444
 445
 446     def account_id(self, account_type, i):
 447         if account_type == '1':
 448             return "m/0'/%d"%i
 449         elif account_type == '2of2':
 450             return "m/1'/%d & m/2'/%d"%(i,i)
 451         elif account_type == '2of3':
 452             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 453         else:
 454             raise Exception('unknown account type')
 455
 456
 457     def num_accounts(self, account_type):
 458         keys = self.accounts.keys()
 459         i = 0
 460         while True:
 461             account_id = self.account_id(account_type, i)
 462             if account_id not in keys: break
 463             i += 1
 464         return i
 465
 466
 467     def new_account_address(self, account_type = '1'):
 468         i = self.num_accounts(account_type)
 469         k = self.account_id(account_type,i)
 470
 471         addr = self.next_addresses.get(k)
 472         if not addr:
 473             account_id, account = self.next_account(account_type)
 474             addr = account.first_address()
 475             self.next_addresses[k] = addr
 476             self.storage.put('next_addresses',self.next_addresses)
 477
 478         return k, addr
 479
 480
 481     def next_account(self, account_type = '1'):
 482
 483         i = self.num_accounts(account_type)
 484         account_id = self.account_id(account_type,i)
 485
 486         if account_type is '1':
 487             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 488             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 489             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 490
 491         elif account_type == '2of2':
 492             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 493             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 494             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 495             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 496             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 497
 498         elif account_type == '2of3':
 499             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 500             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 501             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 502             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 503             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 504             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 505             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 506
 507         return account_id, account
 508
 509
 510     def set_label(self, name, text = None):
 511         changed = False
 512         old_text = self.labels.get(name)
 513         if text:
 514             if old_text != text:
 515                 self.labels[name] = text
 516                 changed = True
 517         else:
 518             if old_text:
 519                 self.labels.pop(name)
 520                 changed = True
 521
 522         if changed:
 523             self.storage.put('labels', self.labels, True)
 524
 525         run_hook('set_label', name, text, changed)
 526         return changed
 527
 528
 529
 530     def create_account(self, account_type = '1', name = None):
 531         k, account = self.next_account(account_type)
 532         if k in self.pending_accounts:
 533             self.pending_accounts.pop(k)
 534             self.storage.put('pending_accounts', self.pending_accounts)
 535
 536         self.accounts[k] = account
 537         self.save_accounts()
 538         if name:
 539             self.set_label(k, name)
 540
 541
 542     def create_old_account(self, mpk):
 543         self.storage.put('master_public_key', mpk, True)
 544         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 545         self.save_accounts()
 546
 547
 548     def save_accounts(self):
 549         d = {}
 550         for k, v in self.accounts.items():
 551             d[k] = v.dump()
 552         self.storage.put('accounts', d, True)
 553
 554
 555
 556     def load_accounts(self):
 557         d = self.storage.get('accounts', {})
 558         self.accounts = {}
 559         for k, v in d.items():
 560             if k == 0:
 561                 v['mpk'] = self.storage.get('master_public_key')
 562                 self.accounts[k] = OldAccount(v)
 563             elif '&' in k:
 564                 self.accounts[k] = BIP32_Account_2of2(v)
 565             else:
 566                 self.accounts[k] = BIP32_Account(v)
 567
 568         self.pending_accounts = self.storage.get('pending_accounts',{})
 569
 570
 571     def delete_pending_account(self, k):
 572         self.pending_accounts.pop(k)
 573         self.storage.put('pending_accounts', self.pending_accounts)
 574
 575     def account_is_pending(self, k):
 576         return k in self.pending_accounts
 577
 578     def create_pending_account(self, acct_type, name):
 579         k, addr = self.new_account_address(acct_type)
 580         self.set_label(k, name)
 581         self.pending_accounts[k] = addr
 582         self.storage.put('pending_accounts', self.pending_accounts)
 583
 584     def get_pending_accounts(self):
 585         return self.pending_accounts.items()
 586
 587
 588     def addresses(self, include_change = True, _next=True):
 589         o = self.get_account_addresses(-1, include_change)
 590         for a in self.accounts.keys():
 591             o += self.get_account_addresses(a, include_change)
 592
 593         if _next:
 594             for addr in self.next_addresses.values():
 595                 if addr not in o:
 596                     o += [addr]
 597         return o
 598
 599
 600     def is_mine(self, address):
 601         return address in self.addresses(True)
 602
 603
 604     def is_change(self, address):
 605         if not self.is_mine(address): return False
 606         if address in self.imported_keys.keys(): return False
 607         acct, s = self.get_address_index(address)
 608         if s is None: return False
 609         return s[0] == 1
 610
 611     def get_master_public_key(self):
 612         if self.seed_version == 4:
 613             return self.storage.get("master_public_key")
 614         else:
 615             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 616             return repr((c, K))
 617
 618     def get_master_private_key(self, account, password):
 619         master_k = pw_decode( self.master_private_keys[account], password)
 620         master_c, master_K, master_Kc = self.master_public_keys[account]
 621         try:
 622             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 623             assert K.encode('hex') == master_K
 624         except Exception:
 625             raise Exception("Invalid password")
 626         return master_k
 627
 628
 629     def get_address_index(self, address):
 630         if address in self.imported_keys.keys():
 631             return -1, None
 632
 633         for account in self.accounts.keys():
 634             for for_change in [0,1]:
 635                 addresses = self.accounts[account].get_addresses(for_change)
 636                 for addr in addresses:
 637                     if address == addr:
 638                         return account, (for_change, addresses.index(addr))
 639
 640         for k,v in self.next_addresses.items():
 641             if v == address:
 642                 return k, (0,0)
 643
 644         raise Exception("Address not found", address)
 645
 646
 647     def get_roots(self, account):
 648         roots = []
 649         for a in account.split('&'):
 650             s = a.strip()
 651             m = re.match("(m/\d+'/)(\d+)", s)
 652             roots.append( m.group(1) )
 653         return roots
 654
 655     def is_seeded(self, account):
 656         if type(account) is int:
 657             return self.seed is not None
 658
 659         for root in self.get_roots(account):
 660             if root not in self.master_private_keys.keys():
 661                 return False
 662         return True
 663
 664     def rebase_sequence(self, account, sequence):
 665         c, i = sequence
 666         dd = []
 667         for a in account.split('&'):
 668             s = a.strip()
 669             m = re.match("(m/\d+'/)(\d+)", s)
 670             root = m.group(1)
 671             num = int(m.group(2))
 672             dd.append( (root, [num,c,i] ) )
 673         return dd
 674
 675
 676     def get_keyID(self, account, sequence):
 677         if account == 0:
 678             a, b = sequence
 679             mpk = self.storage.get('master_public_key')
 680             return 'old(%s,%d,%d)'%(mpk,a,b)
 681
 682         rs = self.rebase_sequence(account, sequence)
 683         dd = []
 684         for root, public_sequence in rs:
 685             c, K, _ = self.master_public_keys[root]
 686             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 687             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 688         return '&'.join(dd)
 689
 690
 691
 692     def get_seed(self, password):
 693         s = pw_decode(self.seed, password)
 694         if self.seed_version == 4:
 695             seed = s
 696             self.accounts[0].check_seed(seed)
 697         else:
 698             seed = mnemonic_hash(s)
 699         return seed
 700
 701
 702     def get_mnemonic(self, password):
 703         import mnemonic
 704         s = pw_decode(self.seed, password)
 705         if self.seed_version == 4:
 706             return ' '.join(mnemonic.mn_encode(s))
 707         else:
 708             return s
 709
 710
 711
 712     def get_private_key(self, address, password):
 713         if self.is_watching_only():
 714             return []
 715
 716         # first check the provided password
 717         seed = self.get_seed(password)
 718
 719         out = []
 720         if address in self.imported_keys.keys():
 721             out.append( pw_decode( self.imported_keys[address], password ) )
 722         else:
 723             account, sequence = self.get_address_index(address)
 724             if account == 0:
 725                 pk = self.accounts[account].get_private_key(seed, sequence)
 726                 out.append(pk)
 727                 return out
 728
 729             # assert address == self.accounts[account].get_address(*sequence)
 730             rs = self.rebase_sequence( account, sequence)
 731             for root, public_sequence in rs:
 732
 733                 if root not in self.master_private_keys.keys(): continue
 734                 master_k = self.get_master_private_key(root, password)
 735                 master_c, _, _ = self.master_public_keys[root]
 736                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 737                 out.append(pk)
 738
 739         return out
 740
 741
 742     def add_keypairs_from_wallet(self, tx, keypairs, password):
 743         for txin in tx.inputs:
 744             address = txin['address']
 745             if not self.is_mine(address):
 746                 continue
 747             private_keys = self.get_private_key(address, password)
 748             for sec in private_keys:
 749                 pubkey = public_key_from_private_key(sec)
 750                 keypairs[ pubkey ] = sec
 751                 if address in self.imported_keys.keys():
 752                     txin['redeemPubkey'] = pubkey
 753
 754
 755     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 756         # first check the provided password
 757         seed = self.get_seed(password)
 758
 759         for txin in tx.inputs:
 760             keyid = txin.get('KeyID')
 761             if keyid:
 762
 763                 if self.seed_version == 4:
 764                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 765                     if not m: continue
 766                     mpk = m.group(1)
 767                     if mpk != self.storage.get('master_public_key'): continue
 768                     for_change = int(m.group(2))
 769                     num = int(m.group(3))
 770                     account = self.accounts[0]
 771                     addr = account.get_address(for_change, num)
 772                     txin['address'] = addr # fixme: side effect
 773                     pk = account.get_private_key(seed, (for_change, num))
 774                     pubkey = public_key_from_private_key(pk)
 775                     keypairs[pubkey] = pk
 776                     continue
 777
 778
 779                 roots = []
 780                 for s in keyid.split('&'):
 781                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 782                     if not m: continue
 783                     c = m.group(1)
 784                     K = m.group(2)
 785                     sequence = m.group(3)
 786                     root = self.find_root_by_master_key(c,K)
 787                     if not root: continue
 788                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 789                     root = root + '%d'%sequence[0]
 790                     sequence = sequence[1:]
 791                     roots.append((root,sequence))
 792
 793                 account_id = " & ".join( map(lambda x:x[0], roots) )
 794                 account = self.accounts.get(account_id)
 795                 if not account: continue
 796                 addr = account.get_address(*sequence)
 797                 txin['address'] = addr # fixme: side effect
 798                 pk = self.get_private_key(addr, password)
 799                 for sec in pk:
 800                     pubkey = public_key_from_private_key(sec)
 801                     keypairs[pubkey] = sec
 802
 803
 804
 805     def signrawtransaction(self, tx, input_info, private_keys, password):
 806
 807         # check that the password is correct
 808         seed = self.get_seed(password)
 809
 810         # add input info
 811         tx.add_input_info(input_info)
 812
 813         # add redeem script for coins that are in the wallet
 814         # FIXME: add redeemPubkey too!
 815         unspent_coins = self.get_unspent_coins()
 816         for txin in tx.inputs:
 817             for item in unspent_coins:
 818                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 819                     print_error( "tx input is in unspent coins" )
 820                     txin['scriptPubKey'] = item['scriptPubKey']
 821                     account, sequence = self.get_address_index(item['address'])
 822                     if account != -1:
 823                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 824                         print_error("added redeemScript", txin['redeemScript'])
 825                     break
 826
 827
 828         # build a list of public/private keys
 829         keypairs = {}
 830
 831         # add private keys from parameter
 832         for sec in private_keys:
 833             pubkey = public_key_from_private_key(sec)
 834             keypairs[ pubkey ] = sec
 835
 836         # add private_keys from KeyID
 837         self.add_keypairs_from_KeyID(tx, keypairs, password)
 838
 839         # add private keys from wallet
 840         self.add_keypairs_from_wallet(tx, keypairs, password)
 841         self.sign_transaction(tx, keypairs)
 842
 843
 844     def sign_message(self, address, message, password):
 845         keys = self.get_private_key(address, password)
 846         assert len(keys) == 1
 847         sec = keys[0]
 848         key = regenerate_key(sec)
 849         compressed = is_compressed(sec)
 850         return key.sign_message(message, compressed, address)
 851
 852
 853     def change_gap_limit(self, value):
 854         if value >= self.gap_limit:
 855             self.gap_limit = value
 856             self.storage.put('gap_limit', self.gap_limit, True)
 857             #self.interface.poke('synchronizer')
 858             return True
 859
 860         elif value >= self.min_acceptable_gap():
 861             for key, account in self.accounts.items():
 862                 addresses = account[0]
 863                 k = self.num_unused_trailing_addresses(addresses)
 864                 n = len(addresses) - k + value
 865                 addresses = addresses[0:n]
 866                 self.accounts[key][0] = addresses
 867
 868             self.gap_limit = value
 869             self.storage.put('gap_limit', self.gap_limit, True)
 870             self.save_accounts()
 871             return True
 872         else:
 873             return False
 874
 875     def num_unused_trailing_addresses(self, addresses):
 876         k = 0
 877         for a in addresses[::-1]:
 878             if self.history.get(a):break
 879             k = k + 1
 880         return k
 881
 882     def min_acceptable_gap(self):
 883         # fixme: this assumes wallet is synchronized
 884         n = 0
 885         nmax = 0
 886
 887         for account in self.accounts.values():
 888             addresses = account.get_addresses(0)
 889             k = self.num_unused_trailing_addresses(addresses)
 890             for a in addresses[0:-k]:
 891                 if self.history.get(a):
 892                     n = 0
 893                 else:
 894                     n += 1
 895                     if n > nmax: nmax = n
 896         return nmax + 1
 897
 898
 899     def address_is_old(self, address):
 900         age = -1
 901         h = self.history.get(address, [])
 902         if h == ['*']:
 903             return True
 904         for tx_hash, tx_height in h:
 905             if tx_height == 0:
 906                 tx_age = 0
 907             else:
 908                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 909             if tx_age > age:
 910                 age = tx_age
 911         return age > 2
 912
 913
 914     def synchronize_sequence(self, account, for_change):
 915         limit = self.gap_limit_for_change if for_change else self.gap_limit
 916         new_addresses = []
 917         while True:
 918             addresses = account.get_addresses(for_change)
 919             if len(addresses) < limit:
 920                 address = account.create_new_address(for_change)
 921                 self.history[address] = []
 922                 new_addresses.append( address )
 923                 continue
 924
 925             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 926                 break
 927             else:
 928                 address = account.create_new_address(for_change)
 929                 self.history[address] = []
 930                 new_addresses.append( address )
 931
 932         return new_addresses
 933
 934
 935
 936     def create_pending_accounts(self):
 937         for account_type in ['1','2of2','2of3']:
 938             if not self.has_master_public_keys(account_type):
 939                 continue
 940             k, a = self.new_account_address(account_type)
 941             if self.address_is_old(a):
 942                 print_error( "creating account", a )
 943                 self.create_account(account_type)
 944                 self.next_addresses.pop(k)
 945
 946
 947     def synchronize_account(self, account):
 948         new = []
 949         new += self.synchronize_sequence(account, 0)
 950         new += self.synchronize_sequence(account, 1)
 951         return new
 952
 953
 954     def synchronize(self):
 955         if self.master_public_keys:
 956             self.create_pending_accounts()
 957         new = []
 958         for account in self.accounts.values():
 959             new += self.synchronize_account(account)
 960         if new:
 961             self.save_accounts()
 962             self.storage.put('addr_history', self.history, True)
 963         return new
 964
 965
 966     def is_found(self):
 967         return self.history.values() != [[]] * len(self.history)
 968
 969
 970     def add_contact(self, address, label=None):
 971         self.addressbook.append(address)
 972         self.storage.put('contacts', self.addressbook, True)
 973         if label:
 974             self.set_label(address, label)
 975
 976
 977     def delete_contact(self, addr):
 978         if addr in self.addressbook:
 979             self.addressbook.remove(addr)
 980             self.storage.put('addressbook', self.addressbook, True)
 981
 982
 983     def fill_addressbook(self):
 984         for tx_hash, tx in self.transactions.items():
 985             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 986             if is_send:
 987                 for addr, v in tx.outputs:
 988                     if not self.is_mine(addr) and addr not in self.addressbook:
 989                         self.addressbook.append(addr)
 990         # redo labels
 991         # self.update_tx_labels()
 992
 993     def get_num_tx(self, address):
 994         n = 0
 995         for tx in self.transactions.values():
 996             if address in map(lambda x:x[0], tx.outputs): n += 1
 997         return n
 998
 999
1000     def get_address_flags(self, addr):
1001         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
1002         flags += "F" if addr in self.frozen_addresses else "-"
1003         return flags
1004
1005
1006     def get_tx_value(self, tx, account=None):
1007         domain = self.get_account_addresses(account)
1008         return tx.get_value(domain, self.prevout_values)
1009
1010
1011     def update_tx_outputs(self, tx_hash):
1012         tx = self.transactions.get(tx_hash)
1013
1014         for i, (addr, value) in enumerate(tx.outputs):
1015             key = tx_hash+ ':%d'%i
1016             self.prevout_values[key] = value
1017
1018         for item in tx.inputs:
1019             if self.is_mine(item.get('address')):
1020                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1021                 self.spent_outputs.append(key)
1022
1023
1024     def get_addr_balance(self, address):
1025         assert self.is_mine(address)
1026         h = self.history.get(address,[])
1027         if h == ['*']: return 0,0
1028         c = u = 0
1029         received_coins = []   # list of coins received at address
1030
1031         for tx_hash, tx_height in h:
1032             tx = self.transactions.get(tx_hash)
1033             if not tx: continue
1034
1035             for i, (addr, value) in enumerate(tx.outputs):
1036                 if addr == address:
1037                     key = tx_hash + ':%d'%i
1038                     received_coins.append(key)
1039
1040         for tx_hash, tx_height in h:
1041             tx = self.transactions.get(tx_hash)
1042             if not tx: continue
1043             v = 0
1044
1045             for item in tx.inputs:
1046                 addr = item.get('address')
1047                 if addr == address:
1048                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1049                     value = self.prevout_values.get( key )
1050                     if key in received_coins:
1051                         v -= value
1052
1053             for i, (addr, value) in enumerate(tx.outputs):
1054                 key = tx_hash + ':%d'%i
1055                 if addr == address:
1056                     v += value
1057
1058             if tx_height:
1059                 c += v
1060             else:
1061                 u += v
1062         return c, u
1063
1064
1065     def get_account_name(self, k):
1066         if k == 0:
1067             if self.seed_version == 4:
1068                 name = 'Main account'
1069             else:
1070                 name = 'Old account'
1071         else:
1072             default = "Unnamed account"
1073             m = re.match("m/0'/(\d+)", k)
1074             if m:
1075                 num = m.group(1)
1076                 if num == '0':
1077                     default = "Main account"
1078                 else:
1079                     default = "Account %s"%num
1080
1081             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1082             if m:
1083                 num = m.group(1)
1084                 default = "2of2 account %s"%num
1085             name = self.labels.get(k, default)
1086
1087         return name
1088
1089     def get_account_names(self):
1090         accounts = {}
1091         for k, account in self.accounts.items():
1092             accounts[k] = self.get_account_name(k)
1093         if self.imported_keys:
1094             accounts[-1] = 'Imported keys'
1095         return accounts
1096
1097     def get_account_addresses(self, a, include_change=True):
1098         if a is None:
1099             o = self.addresses(True)
1100         elif a == -1:
1101             o = self.imported_keys.keys()
1102         else:
1103             ac = self.accounts[a]
1104             o = ac.get_addresses(0)
1105             if include_change: o += ac.get_addresses(1)
1106         return o
1107
1108     def get_imported_balance(self):
1109         return self.get_balance(self.imported_keys.keys())
1110
1111     def get_account_balance(self, account):
1112         return self.get_balance(self.get_account_addresses(account))
1113
1114     def get_frozen_balance(self):
1115         return self.get_balance(self.frozen_addresses)
1116
1117     def get_balance(self, domain=None):
1118         if domain is None: domain = self.addresses(True)
1119         cc = uu = 0
1120         for addr in domain:
1121             c, u = self.get_addr_balance(addr)
1122             cc += c
1123             uu += u
1124         return cc, uu
1125
1126
1127     def get_unspent_coins(self, domain=None):
1128         coins = []
1129         if domain is None: domain = self.addresses(True)
1130         for addr in domain:
1131             h = self.history.get(addr, [])
1132             if h == ['*']: continue
1133             for tx_hash, tx_height in h:
1134                 tx = self.transactions.get(tx_hash)
1135                 if tx is None: raise Exception("Wallet not synchronized")
1136                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1137                 for o in tx.d.get('outputs'):
1138                     output = o.copy()
1139                     if output.get('address') != addr: continue
1140                     key = tx_hash + ":%d" % output.get('prevout_n')
1141                     if key in self.spent_outputs: continue
1142                     output['prevout_hash'] = tx_hash
1143                     output['height'] = tx_height
1144                     output['coinbase'] = is_coinbase
1145                     coins.append((tx_height, output))
1146
1147         # sort by age
1148         if coins:
1149             coins = sorted(coins)
1150             if coins[-1][0] != 0:
1151                 while coins[0][0] == 0:
1152                     coins = coins[1:] + [ coins[0] ]
1153         return [x[1] for x in coins]
1154
1155
1156     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1157         """ todo: minimize tx size """
1158         total = 0
1159         fee = self.fee if fixed_fee is None else fixed_fee
1160         if domain is None:
1161             domain = self.addresses(True)
1162
1163         for i in self.frozen_addresses:
1164             if i in domain: domain.remove(i)
1165
1166         coins = self.get_unspent_coins(domain)
1167         inputs = []
1168
1169         for item in coins:
1170             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1171                 continue
1172             addr = item.get('address')
1173             v = item.get('value')
1174             total += v
1175             inputs.append(item)
1176             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1177             if total >= amount + fee: break
1178         else:
1179             inputs = []
1180
1181         return inputs, total, fee
1182
1183
1184     def set_fee(self, fee):
1185         if self.fee != fee:
1186             self.fee = fee
1187             self.storage.put('fee_per_kb', self.fee, True)
1188
1189     def estimated_fee(self, inputs):
1190         estimated_size =  len(inputs) * 180 + 80    # this assumes non-compressed keys
1191         fee = self.fee * int(math.ceil(estimated_size/1000.))
1192         return fee
1193
1194
1195     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1196         "add change to a transaction"
1197         change_amount = total - ( amount + fee )
1198         if change_amount > DUST_THRESHOLD:
1199             if not change_addr:
1200
1201                 # send change to one of the accounts involved in the tx
1202                 address = inputs[0].get('address')
1203                 account, _ = self.get_address_index(address)
1204
1205                 if not self.use_change or account == -1:
1206                     change_addr = inputs[-1]['address']
1207                 else:
1208                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1209
1210             # Insert the change output at a random position in the outputs
1211             posn = random.randint(0, len(outputs))
1212             outputs[posn:posn] = [( change_addr,  change_amount)]
1213         return outputs
1214
1215
1216     def get_history(self, address):
1217         with self.lock:
1218             return self.history.get(address)
1219
1220
1221     def get_status(self, h):
1222         if not h: return None
1223         if h == ['*']: return '*'
1224         status = ''
1225         for tx_hash, height in h:
1226             status += tx_hash + ':%d:' % height
1227         return hashlib.sha256( status ).digest().encode('hex')
1228
1229
1230     def receive_tx_callback(self, tx_hash, tx, tx_height):
1231
1232         with self.transaction_lock:
1233             self.add_extra_addresses(tx)
1234             if not self.check_new_tx(tx_hash, tx):
1235                 # may happen due to pruning
1236                 print_error("received transaction that is no longer referenced in history", tx_hash)
1237                 return
1238             self.transactions[tx_hash] = tx
1239             self.network.interface.pending_transactions_for_notifications.append(tx)
1240             self.save_transactions()
1241             if self.verifier and tx_height>0:
1242                 self.verifier.add(tx_hash, tx_height)
1243             self.update_tx_outputs(tx_hash)
1244
1245
1246     def save_transactions(self):
1247         tx = {}
1248         for k,v in self.transactions.items():
1249             tx[k] = str(v)
1250         self.storage.put('transactions', tx, True)
1251
1252     def receive_history_callback(self, addr, hist):
1253
1254         if not self.check_new_history(addr, hist):
1255             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1256
1257         with self.lock:
1258             self.history[addr] = hist
1259             self.storage.put('addr_history', self.history, True)
1260
1261         if hist != ['*']:
1262             for tx_hash, tx_height in hist:
1263                 if tx_height>0:
1264                     # add it in case it was previously unconfirmed
1265                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1266
1267
1268     def get_tx_history(self, account=None):
1269         if not self.verifier:
1270             return []
1271
1272         with self.transaction_lock:
1273             history = self.transactions.items()
1274             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1275             result = []
1276
1277             balance = 0
1278             for tx_hash, tx in history:
1279                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1280                 if v is not None: balance += v
1281
1282             c, u = self.get_account_balance(account)
1283
1284             if balance != c+u:
1285                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1286
1287             balance = c + u - balance
1288             for tx_hash, tx in history:
1289                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1290                 if not is_relevant:
1291                     continue
1292                 if value is not None:
1293                     balance += value
1294
1295                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1296                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1297
1298         return result
1299
1300
1301     def get_label(self, tx_hash):
1302         label = self.labels.get(tx_hash)
1303         is_default = (label == '') or (label is None)
1304         if is_default: label = self.get_default_label(tx_hash)
1305         return label, is_default
1306
1307
1308     def get_default_label(self, tx_hash):
1309         tx = self.transactions.get(tx_hash)
1310         default_label = ''
1311         if tx:
1312             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1313             if is_mine:
1314                 for o in tx.outputs:
1315                     o_addr, _ = o
1316                     if not self.is_mine(o_addr):
1317                         try:
1318                             default_label = self.labels[o_addr]
1319                         except KeyError:
1320                             default_label = o_addr
1321                         break
1322                 else:
1323                     default_label = '(internal)'
1324             else:
1325                 for o in tx.outputs:
1326                     o_addr, _ = o
1327                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1328                         break
1329                 else:
1330                     for o in tx.outputs:
1331                         o_addr, _ = o
1332                         if self.is_mine(o_addr):
1333                             break
1334                     else:
1335                         o_addr = None
1336
1337                 if o_addr:
1338                     dest_label = self.labels.get(o_addr)
1339                     try:
1340                         default_label = self.labels[o_addr]
1341                     except KeyError:
1342                         default_label = o_addr
1343
1344         return default_label
1345
1346
1347     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1348         for address, x in outputs:
1349             assert is_valid(address)
1350         amount = sum( map(lambda x:x[1], outputs) )
1351         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1352         if not inputs:
1353             raise ValueError("Not enough funds")
1354         self.add_input_info(inputs)
1355         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1356         return Transaction.from_io(inputs, outputs)
1357
1358
1359     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1360         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1361         keypairs = {}
1362         self.add_keypairs_from_wallet(tx, keypairs, password)
1363         if keypairs:
1364             self.sign_transaction(tx, keypairs)
1365         return tx
1366
1367
1368     def add_input_info(self, inputs):
1369         for txin in inputs:
1370             address = txin['address']
1371             if address in self.imported_keys.keys():
1372                 continue
1373             account, sequence = self.get_address_index(address)
1374             txin['KeyID'] = self.get_keyID(account, sequence)
1375             redeemScript = self.accounts[account].redeem_script(sequence)
1376             if redeemScript:
1377                 txin['redeemScript'] = redeemScript
1378             else:
1379                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1380
1381
1382     def sign_transaction(self, tx, keypairs):
1383         tx.sign(keypairs)
1384         run_hook('sign_transaction', tx)
1385
1386
1387     def sendtx(self, tx):
1388         # synchronous
1389         h = self.send_tx(tx)
1390         self.tx_event.wait()
1391         return self.receive_tx(h, tx)
1392
1393     def send_tx(self, tx):
1394         # asynchronous
1395         self.tx_event.clear()
1396         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1397         return tx.hash()
1398
1399     def on_broadcast(self, i, r):
1400         self.tx_result = r.get('result')
1401         self.tx_event.set()
1402
1403     def receive_tx(self, tx_hash, tx):
1404         out = self.tx_result
1405         if out != tx_hash:
1406             return False, "error: " + out
1407         run_hook('receive_tx', tx, self)
1408         return True, out
1409
1410
1411
1412     def update_password(self, old_password, new_password):
1413         if new_password == '': new_password = None
1414         decoded = self.get_seed(old_password)
1415         self.seed = pw_encode( decoded, new_password)
1416         self.storage.put('seed', self.seed, True)
1417         self.use_encryption = (new_password != None)
1418         self.storage.put('use_encryption', self.use_encryption,True)
1419         for k in self.imported_keys.keys():
1420             a = self.imported_keys[k]
1421             b = pw_decode(a, old_password)
1422             c = pw_encode(b, new_password)
1423             self.imported_keys[k] = c
1424         self.storage.put('imported_keys', self.imported_keys, True)
1425
1426         for k, v in self.master_private_keys.items():
1427             b = pw_decode(v, old_password)
1428             c = pw_encode(b, new_password)
1429             self.master_private_keys[k] = c
1430         self.storage.put('master_private_keys', self.master_private_keys, True)
1431
1432
1433     def freeze(self,addr):
1434         if self.is_mine(addr) and addr not in self.frozen_addresses:
1435             self.frozen_addresses.append(addr)
1436             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1437             return True
1438         else:
1439             return False
1440
1441     def unfreeze(self,addr):
1442         if self.is_mine(addr) and addr in self.frozen_addresses:
1443             self.frozen_addresses.remove(addr)
1444             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1445             return True
1446         else:
1447             return False
1448
1449
1450     def set_verifier(self, verifier):
1451         self.verifier = verifier
1452
1453         # review transactions that are in the history
1454         for addr, hist in self.history.items():
1455             if hist == ['*']: continue
1456             for tx_hash, tx_height in hist:
1457                 if tx_height>0:
1458                     # add it in case it was previously unconfirmed
1459                     self.verifier.add(tx_hash, tx_height)
1460
1461
1462         # if we are on a pruning server, remove unverified transactions
1463         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1464         for tx_hash in self.transactions.keys():
1465             if tx_hash not in vr:
1466                 self.transactions.pop(tx_hash)
1467
1468
1469
1470     def check_new_history(self, addr, hist):
1471
1472         # check that all tx in hist are relevant
1473         if hist != ['*']:
1474             for tx_hash, height in hist:
1475                 tx = self.transactions.get(tx_hash)
1476                 if not tx: continue
1477                 if not tx.has_address(addr):
1478                     return False
1479
1480         # check that we are not "orphaning" a transaction
1481         old_hist = self.history.get(addr,[])
1482         if old_hist == ['*']: return True
1483
1484         for tx_hash, height in old_hist:
1485             if tx_hash in map(lambda x:x[0], hist): continue
1486             found = False
1487             for _addr, _hist in self.history.items():
1488                 if _addr == addr: continue
1489                 if _hist == ['*']: continue
1490                 _tx_hist = map(lambda x:x[0], _hist)
1491                 if tx_hash in _tx_hist:
1492                     found = True
1493                     break
1494
1495             if not found:
1496                 tx = self.transactions.get(tx_hash)
1497                 # tx might not be there
1498                 if not tx: continue
1499
1500                 # already verified?
1501                 if self.verifier.get_height(tx_hash):
1502                     continue
1503                 # unconfirmed tx
1504                 print_error("new history is orphaning transaction:", tx_hash)
1505                 # check that all outputs are not mine, request histories
1506                 ext_requests = []
1507                 for _addr, _v in tx.outputs:
1508                     # assert not self.is_mine(_addr)
1509                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1510
1511                 ext_h = self.network.synchronous_get(ext_requests)
1512                 print_error("sync:", ext_requests, ext_h)
1513                 height = None
1514                 for h in ext_h:
1515                     if h == ['*']: continue
1516                     for item in h:
1517                         if item.get('tx_hash') == tx_hash:
1518                             height = item.get('height')
1519                 if height:
1520                     print_error("found height for", tx_hash, height)
1521                     self.verifier.add(tx_hash, height)
1522                 else:
1523                     print_error("removing orphaned tx from history", tx_hash)
1524                     self.transactions.pop(tx_hash)
1525
1526         return True
1527
1528
1529
1530     def check_new_tx(self, tx_hash, tx):
1531         # 1 check that tx is referenced in addr_history.
1532         addresses = []
1533         for addr, hist in self.history.items():
1534             if hist == ['*']:continue
1535             for txh, height in hist:
1536                 if txh == tx_hash:
1537                     addresses.append(addr)
1538
1539         if not addresses:
1540             return False
1541
1542         # 2 check that referencing addresses are in the tx
1543         for addr in addresses:
1544             if not tx.has_address(addr):
1545                 return False
1546
1547         return True
1548
1549
1550     def start_threads(self, network):
1551         from verifier import TxVerifier
1552         self.network = network
1553         if self.network:
1554             self.verifier = TxVerifier(self.network, self.storage)
1555             self.verifier.start()
1556             self.set_verifier(self.verifier)
1557             self.synchronizer = WalletSynchronizer(self, network)
1558             self.synchronizer.start()
1559         else:
1560             self.verifier = None
1561             self.synchronizer =None
1562
1563     def stop_threads(self):
1564         if self.network:
1565             self.verifier.stop()
1566             self.synchronizer.stop()
1567
1568
1569     def restore(self, callback):
1570         from i18n import _
1571         def wait_for_wallet():
1572             self.set_up_to_date(False)
1573             while not self.is_up_to_date():
1574                 msg = "%s\n%s %d\n%s %.1f"%(
1575                     _("Please wait..."),
1576                     _("Addresses generated:"),
1577                     len(self.addresses(True)),
1578                     _("Kilobytes received:"),
1579                     self.network.interface.bytes_received/1024.)
1580
1581                 apply(callback, (msg,))
1582                 time.sleep(0.1)
1583
1584         def wait_for_network():
1585             while not self.network.is_connected():
1586                 msg = "%s \n" % (_("Connecting..."))
1587                 apply(callback, (msg,))
1588                 time.sleep(0.1)
1589
1590         # wait until we are connected, because the user might have selected another server
1591         if self.network:
1592             wait_for_network()
1593             wait_for_wallet()
1594         else:
1595             self.synchronize()
1596
1597         self.fill_addressbook()
1598
1599
1600
1601
1602 class WalletSynchronizer(threading.Thread):
1603
1604
1605     def __init__(self, wallet, network):
1606         threading.Thread.__init__(self)
1607         self.daemon = True
1608         self.wallet = wallet
1609         self.network = network
1610         self.was_updated = True
1611         self.running = False
1612         self.lock = threading.Lock()
1613         self.queue = Queue.Queue()
1614
1615     def stop(self):
1616         with self.lock: self.running = False
1617
1618     def is_running(self):
1619         with self.lock: return self.running
1620
1621
1622     def subscribe_to_addresses(self, addresses):
1623         messages = []
1624         for addr in addresses:
1625             messages.append(('blockchain.address.subscribe', [addr]))
1626         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1627
1628
1629     def run(self):
1630         with self.lock:
1631             self.running = True
1632
1633         while self.is_running():
1634
1635             if not self.network.is_connected():
1636                 self.network.wait_until_connected()
1637
1638             self.run_interface(self.network.interface)
1639
1640
1641     def run_interface(self, interface):
1642
1643         print_error("synchronizer: connected to", interface.server)
1644
1645         requested_tx = []
1646         missing_tx = []
1647         requested_histories = {}
1648
1649         # request any missing transactions
1650         for history in self.wallet.history.values():
1651             if history == ['*']: continue
1652             for tx_hash, tx_height in history:
1653                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1654                     missing_tx.append( (tx_hash, tx_height) )
1655
1656         if missing_tx:
1657             print_error("missing tx", missing_tx)
1658
1659         # subscriptions
1660         self.subscribe_to_addresses(self.wallet.addresses(True))
1661
1662         while self.is_running():
1663             # 1. create new addresses
1664             new_addresses = self.wallet.synchronize()
1665
1666             # request missing addresses
1667             if new_addresses:
1668                 self.subscribe_to_addresses(new_addresses)
1669
1670             # request missing transactions
1671             for tx_hash, tx_height in missing_tx:
1672                 if (tx_hash, tx_height) not in requested_tx:
1673                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1674                     requested_tx.append( (tx_hash, tx_height) )
1675             missing_tx = []
1676
1677             # detect if situation has changed
1678             if interface.is_up_to_date() and self.queue.empty():
1679                 if not self.wallet.is_up_to_date():
1680                     self.wallet.set_up_to_date(True)
1681                     self.was_updated = True
1682             else:
1683                 if self.wallet.is_up_to_date():
1684                     self.wallet.set_up_to_date(False)
1685                     self.was_updated = True
1686
1687             if self.was_updated:
1688                 self.wallet.network.trigger_callback('updated')
1689                 self.was_updated = False
1690
1691             # 2. get a response
1692             try:
1693                 r = self.queue.get(block=True, timeout=1)
1694             except Queue.Empty:
1695                 continue
1696
1697             if interface != self.network.interface:
1698                 break
1699
1700             if not r:
1701                 continue
1702
1703             # 3. handle response
1704             method = r['method']
1705             params = r['params']
1706             result = r.get('result')
1707             error = r.get('error')
1708             if error:
1709                 print "error", r
1710                 continue
1711
1712             if method == 'blockchain.address.subscribe':
1713                 addr = params[0]
1714                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1715                     if requested_histories.get(addr) is None:
1716                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1717                         requested_histories[addr] = result
1718
1719             elif method == 'blockchain.address.get_history':
1720                 addr = params[0]
1721                 print_error("receiving history", addr, result)
1722                 if result == ['*']:
1723                     assert requested_histories.pop(addr) == '*'
1724                     self.wallet.receive_history_callback(addr, result)
1725                 else:
1726                     hist = []
1727                     # check that txids are unique
1728                     txids = []
1729                     for item in result:
1730                         tx_hash = item['tx_hash']
1731                         if tx_hash not in txids:
1732                             txids.append(tx_hash)
1733                             hist.append( (tx_hash, item['height']) )
1734
1735                     if len(hist) != len(result):
1736                         raise Exception("error: server sent history with non-unique txid", result)
1737
1738                     # check that the status corresponds to what was announced
1739                     rs = requested_histories.pop(addr)
1740                     if self.wallet.get_status(hist) != rs:
1741                         raise Exception("error: status mismatch: %s"%addr)
1742
1743                     # store received history
1744                     self.wallet.receive_history_callback(addr, hist)
1745
1746                     # request transactions that we don't have
1747                     for tx_hash, tx_height in hist:
1748                         if self.wallet.transactions.get(tx_hash) is None:
1749                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1750                                 missing_tx.append( (tx_hash, tx_height) )
1751
1752             elif method == 'blockchain.transaction.get':
1753                 tx_hash = params[0]
1754                 tx_height = params[1]
1755                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1756                 tx = Transaction(result)
1757                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1758                 self.was_updated = True
1759                 requested_tx.remove( (tx_hash, tx_height) )
1760                 print_error("received tx:", tx_hash, len(tx.raw))
1761
1762             else:
1763                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1764
1765             if self.was_updated and not requested_tx:
1766                 self.wallet.network.trigger_callback('updated')
1767                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1768
1769                 self.was_updated = False