lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32 import math
  33
  34 from util import print_msg, print_error, format_satoshis
  35 from bitcoin import *
  36 from account import *
  37 from transaction import Transaction
  38 from plugins import run_hook
  39
  40 COINBASE_MATURITY = 100
  41 DUST_THRESHOLD = 5430
  42
  43 # AES encryption
  44 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  45 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  46
  47 def pw_encode(s, password):
  48     if password:
  49         secret = Hash(password)
  50         return EncodeAES(secret, s)
  51     else:
  52         return s
  53
  54 def pw_decode(s, password):
  55     if password is not None:
  56         secret = Hash(password)
  57         try:
  58             d = DecodeAES(secret, s)
  59         except Exception:
  60             raise Exception('Invalid password')
  61         return d
  62     else:
  63         return s
  64
  65
  66
  67
  68
  69 from version import *
  70
  71
  72 class WalletStorage:
  73
  74     def __init__(self, config):
  75         self.lock = threading.Lock()
  76         self.data = {}
  77         self.file_exists = False
  78         self.path = self.init_path(config)
  79         print_error( "wallet path", self.path )
  80         if self.path:
  81             self.read(self.path)
  82
  83
  84     def init_path(self, config):
  85         """Set the path of the wallet."""
  86
  87         # command line -w option
  88         path = config.get('wallet_path')
  89         if path:
  90             return path
  91
  92         # path in config file
  93         path = config.get('default_wallet_path')
  94         if path:
  95             return path
  96
  97         # default path
  98         dirpath = os.path.join(config.path, "wallets")
  99         if not os.path.exists(dirpath):
 100             os.mkdir(dirpath)
 101
 102         new_path = os.path.join(config.path, "wallets", "default_wallet")
 103
 104         # default path in pre 1.9 versions
 105         old_path = os.path.join(config.path, "electrum.dat")
 106         if os.path.exists(old_path) and not os.path.exists(new_path):
 107             os.rename(old_path, new_path)
 108
 109         return new_path
 110
 111
 112     def read(self, path):
 113         """Read the contents of the wallet file."""
 114         try:
 115             with open(self.path, "r") as f:
 116                 data = f.read()
 117         except IOError:
 118             return
 119         try:
 120             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 121         except Exception:
 122             raise IOError("Cannot read wallet file.")
 123
 124         self.data = d
 125         self.file_exists = True
 126
 127
 128     def get(self, key, default=None):
 129         v = self.data.get(key)
 130         if v is None:
 131             v = default
 132         return v
 133
 134     def put(self, key, value, save = True):
 135
 136         with self.lock:
 137             if value is not None:
 138                 self.data[key] = value
 139             else:
 140                 self.data.pop(key)
 141             if save:
 142                 self.write()
 143
 144     def write(self):
 145         s = repr(self.data)
 146         f = open(self.path,"w")
 147         f.write( s )
 148         f.close()
 149         if 'ANDROID_DATA' not in os.environ:
 150             import stat
 151             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 152
 153
 154 class Wallet:
 155
 156     def __init__(self, storage):
 157
 158         self.storage = storage
 159         self.electrum_version = ELECTRUM_VERSION
 160         self.gap_limit_for_change = 3 # constant
 161
 162         # saved fields
 163         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 164
 165         self.gap_limit             = storage.get('gap_limit', 5)
 166         self.use_change            = storage.get('use_change',True)
 167         self.use_encryption        = storage.get('use_encryption', False)
 168         self.seed                  = storage.get('seed', '')               # encrypted
 169         self.labels                = storage.get('labels', {})
 170         self.frozen_addresses      = storage.get('frozen_addresses',[])
 171         self.addressbook           = storage.get('contacts', [])
 172
 173         self.imported_keys         = storage.get('imported_keys',{})
 174         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 175
 176         self.fee                   = int(storage.get('fee_per_kb',20000))
 177
 178         self.master_public_keys = storage.get('master_public_keys',{})
 179         self.master_private_keys = storage.get('master_private_keys', {})
 180
 181         self.next_addresses = storage.get('next_addresses',{})
 182
 183         if self.seed_version not in [4, 6]:
 184             msg = "This wallet seed is not supported."
 185             if self.seed_version in [5]:
 186                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 187                 print msg
 188                 sys.exit(1)
 189
 190         # This attribute is set when wallet.start_threads is called.
 191         self.synchronizer = None
 192
 193         self.load_accounts()
 194
 195         self.transactions = {}
 196         tx_list = self.storage.get('transactions',{})
 197         for k,v in tx_list.items():
 198             try:
 199                 tx = Transaction(v)
 200             except Exception:
 201                 print_msg("Warning: Cannot deserialize transactions. skipping")
 202                 continue
 203
 204             self.add_extra_addresses(tx)
 205             self.transactions[k] = tx
 206
 207         for h,tx in self.transactions.items():
 208             if not self.check_new_tx(h, tx):
 209                 print_error("removing unreferenced tx", h)
 210                 self.transactions.pop(h)
 211
 212
 213         # not saved
 214         self.prevout_values = {}     # my own transaction outputs
 215         self.spent_outputs = []
 216
 217         # spv
 218         self.verifier = None
 219
 220         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 221         # interface.is_up_to_date() returns true when all requests have been answered and processed
 222         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 223
 224         self.up_to_date = False
 225         self.lock = threading.Lock()
 226         self.transaction_lock = threading.Lock()
 227         self.tx_event = threading.Event()
 228
 229         for tx_hash, tx in self.transactions.items():
 230             self.update_tx_outputs(tx_hash)
 231
 232
 233     def add_extra_addresses(self, tx):
 234         h = tx.hash()
 235         # find the address corresponding to pay-to-pubkey inputs
 236         tx.add_extra_addresses(self.transactions)
 237         for o in tx.d.get('outputs'):
 238             if o.get('is_pubkey'):
 239                 for tx2 in self.transactions.values():
 240                     tx2.add_extra_addresses({h:tx})
 241
 242
 243
 244
 245     def set_up_to_date(self,b):
 246         with self.lock: self.up_to_date = b
 247
 248
 249     def is_up_to_date(self):
 250         with self.lock: return self.up_to_date
 251
 252
 253     def update(self):
 254         self.up_to_date = False
 255         while not self.is_up_to_date():
 256             time.sleep(0.1)
 257
 258
 259     def import_key(self, sec, password):
 260         # check password
 261         seed = self.get_seed(password)
 262         try:
 263             address = address_from_private_key(sec)
 264         except Exception:
 265             raise Exception('Invalid private key')
 266
 267         if self.is_mine(address):
 268             raise Exception('Address already in wallet')
 269
 270         # store the originally requested keypair into the imported keys table
 271         self.imported_keys[address] = pw_encode(sec, password )
 272         self.storage.put('imported_keys', self.imported_keys, True)
 273         if self.synchronizer:
 274             self.synchronizer.subscribe_to_addresses([address])
 275         return address
 276
 277     def delete_imported_key(self, addr):
 278         if addr in self.imported_keys:
 279             self.imported_keys.pop(addr)
 280             self.storage.put('imported_keys', self.imported_keys, True)
 281
 282
 283     def make_seed(self):
 284         import mnemonic, ecdsa
 285         entropy = ecdsa.util.randrange( pow(2,160) )
 286         nonce = 0
 287         while True:
 288             ss = "%040x"%(entropy+nonce)
 289             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 290             # we keep only 13 words, that's approximately 139 bits of entropy
 291             words = mnemonic.mn_encode(s)[0:13]
 292             seed = ' '.join(words)
 293             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 294                 break  # this removes 12 bits of entropy
 295             nonce += 1
 296
 297         return seed
 298
 299
 300     def init_seed(self, seed):
 301         import mnemonic
 302
 303         if self.seed:
 304             raise Exception("a seed exists")
 305
 306         if not seed:
 307             self.seed = random_seed(128)
 308             self.seed_version = 4
 309             return
 310
 311         # check if seed is hexadecimal
 312         seed = seed.strip()
 313         try:
 314             assert seed
 315             seed.decode('hex')
 316             self.seed_version = 4
 317             self.seed = str(seed)
 318             return
 319         except Exception:
 320             pass
 321
 322         words = seed.split()
 323         self.seed_version = 4
 324         self.seed = mnemonic.mn_decode(words)
 325
 326         if not self.seed:
 327             raise Exception("Invalid seed")
 328
 329
 330
 331     def save_seed(self, password):
 332         if password:
 333             self.seed = pw_encode( self.seed, password)
 334             self.use_encryption = True
 335         self.storage.put('seed', self.seed, True)
 336         self.storage.put('seed_version', self.seed_version, True)
 337         self.storage.put('use_encryption', self.use_encryption,True)
 338         self.create_accounts(password)
 339
 340
 341     def create_watching_only_wallet(self, params):
 342         K0, c0 = params
 343         if not K0:
 344             return
 345
 346         if not c0:
 347             self.seed_version = 4
 348             self.storage.put('seed_version', self.seed_version, True)
 349             self.create_old_account(K0)
 350             return
 351
 352         cK0 = ""
 353         self.master_public_keys = {
 354             "m/0'/": (c0, K0, cK0),
 355             }
 356         self.storage.put('master_public_keys', self.master_public_keys, True)
 357         self.storage.put('seed_version', self.seed_version, True)
 358         self.create_account('1','Main account')
 359
 360
 361     def create_accounts(self, password):
 362         seed = pw_decode(self.seed, password)
 363
 364         if self.seed_version == 4:
 365             mpk = OldAccount.mpk_from_seed(seed)
 366             self.create_old_account(mpk)
 367         else:
 368             # create default account
 369             self.create_master_keys('1', password)
 370             self.create_account('1','Main account')
 371
 372
 373     def create_master_keys(self, account_type, password):
 374         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 375         if account_type == '1':
 376             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 377             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 378             self.master_private_keys["m/0'/"] = pw_encode(k0, password)
 379         elif account_type == '2of2':
 380             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 381             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 382             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 383             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 384             self.master_private_keys["m/1'/"] = pw_encode(k1, password)
 385             self.master_private_keys["m/2'/"] = pw_encode(k2, password)
 386         elif account_type == '2of3':
 387             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 388             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 389             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 390             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 391             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 392             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 393             self.master_private_keys["m/3'/"] = pw_encode(k3, password)
 394             self.master_private_keys["m/4'/"] = pw_encode(k4, password)
 395             self.master_private_keys["m/5'/"] = pw_encode(k5, password)
 396
 397         self.storage.put('master_public_keys', self.master_public_keys, True)
 398         self.storage.put('master_private_keys', self.master_private_keys, True)
 399
 400     def has_master_public_keys(self, account_type):
 401         if account_type == '1':
 402             return "m/0'/" in self.master_public_keys
 403         elif account_type == '2of2':
 404             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 405         elif account_type == '2of3':
 406             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 407
 408     def find_root_by_master_key(self, c, K):
 409         for key, v in self.master_public_keys.items():
 410             if key == "m/":continue
 411             cc, KK, _ = v
 412             if (c == cc) and (K == KK):
 413                 return key
 414
 415     def deseed_root(self, seed, password):
 416         # for safety, we ask the user to enter their seed
 417         assert seed == self.get_seed(password)
 418         self.seed = ''
 419         self.storage.put('seed', '', True)
 420
 421
 422     def deseed_branch(self, k):
 423         # check that parent has no seed
 424         assert self.seed == ''
 425         self.master_private_keys.pop(k)
 426         self.storage.put('master_private_keys', self.master_private_keys, True)
 427
 428     def is_watching_only(self):
 429         return (self.seed == '') and (self.master_private_keys == {})
 430
 431
 432
 433     def account_id(self, account_type, i):
 434         if account_type == '1':
 435             return "m/0'/%d"%i
 436         elif account_type == '2of2':
 437             return "m/1'/%d & m/2'/%d"%(i,i)
 438         elif account_type == '2of3':
 439             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 440         else:
 441             raise Exception('unknown account type')
 442
 443
 444     def num_accounts(self, account_type):
 445         keys = self.accounts.keys()
 446         i = 0
 447         while True:
 448             account_id = self.account_id(account_type, i)
 449             if account_id not in keys: break
 450             i += 1
 451         return i
 452
 453
 454     def new_account_address(self, account_type = '1'):
 455         i = self.num_accounts(account_type)
 456         k = self.account_id(account_type,i)
 457
 458         addr = self.next_addresses.get(k)
 459         if not addr:
 460             account_id, account = self.next_account(account_type)
 461             addr = account.first_address()
 462             self.next_addresses[k] = addr
 463             self.storage.put('next_addresses',self.next_addresses)
 464
 465         return k, addr
 466
 467
 468     def next_account(self, account_type = '1'):
 469
 470         i = self.num_accounts(account_type)
 471         account_id = self.account_id(account_type,i)
 472
 473         if account_type is '1':
 474             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 475             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 476             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 477
 478         elif account_type == '2of2':
 479             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 480             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 481             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 482             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 483             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 484
 485         elif account_type == '2of3':
 486             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 487             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 488             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 489             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 490             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 491             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 492             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 493
 494         return account_id, account
 495
 496
 497     def set_label(self, name, text = None):
 498         changed = False
 499         old_text = self.labels.get(name)
 500         if text:
 501             if old_text != text:
 502                 self.labels[name] = text
 503                 changed = True
 504         else:
 505             if old_text:
 506                 self.labels.pop(name)
 507                 changed = True
 508
 509         if changed:
 510             self.storage.put('labels', self.labels, True)
 511
 512         run_hook('set_label', name, text, changed)
 513         return changed
 514
 515
 516
 517     def create_account(self, account_type = '1', name = None):
 518         k, account = self.next_account(account_type)
 519         if k in self.pending_accounts:
 520             self.pending_accounts.pop(k)
 521             self.storage.put('pending_accounts', self.pending_accounts)
 522
 523         self.accounts[k] = account
 524         self.save_accounts()
 525         if name:
 526             self.set_label(k, name)
 527
 528
 529     def create_old_account(self, mpk):
 530         self.storage.put('master_public_key', mpk, True)
 531         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 532         self.save_accounts()
 533
 534
 535     def save_accounts(self):
 536         d = {}
 537         for k, v in self.accounts.items():
 538             d[k] = v.dump()
 539         self.storage.put('accounts', d, True)
 540
 541
 542
 543     def load_accounts(self):
 544         d = self.storage.get('accounts', {})
 545         self.accounts = {}
 546         for k, v in d.items():
 547             if k == 0:
 548                 v['mpk'] = self.storage.get('master_public_key')
 549                 self.accounts[k] = OldAccount(v)
 550             elif '&' in k:
 551                 self.accounts[k] = BIP32_Account_2of2(v)
 552             else:
 553                 self.accounts[k] = BIP32_Account(v)
 554
 555         self.pending_accounts = self.storage.get('pending_accounts',{})
 556
 557
 558     def delete_pending_account(self, k):
 559         self.pending_accounts.pop(k)
 560         self.storage.put('pending_accounts', self.pending_accounts)
 561
 562     def account_is_pending(self, k):
 563         return k in self.pending_accounts
 564
 565     def create_pending_account(self, acct_type, name):
 566         k, addr = self.new_account_address(acct_type)
 567         self.set_label(k, name)
 568         self.pending_accounts[k] = addr
 569         self.storage.put('pending_accounts', self.pending_accounts)
 570
 571     def get_pending_accounts(self):
 572         return self.pending_accounts.items()
 573
 574
 575     def addresses(self, include_change = True, _next=True):
 576         o = self.get_account_addresses(-1, include_change)
 577         for a in self.accounts.keys():
 578             o += self.get_account_addresses(a, include_change)
 579
 580         if _next:
 581             for addr in self.next_addresses.values():
 582                 if addr not in o:
 583                     o += [addr]
 584         return o
 585
 586
 587     def is_mine(self, address):
 588         return address in self.addresses(True)
 589
 590
 591     def is_change(self, address):
 592         if not self.is_mine(address): return False
 593         if address in self.imported_keys.keys(): return False
 594         acct, s = self.get_address_index(address)
 595         if s is None: return False
 596         return s[0] == 1
 597
 598     def get_master_public_key(self):
 599         if self.seed_version == 4:
 600             return self.storage.get("master_public_key")
 601         else:
 602             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 603             return repr((c, K))
 604
 605     def get_master_private_key(self, account, password):
 606         master_k = pw_decode( self.master_private_keys[account], password)
 607         master_c, master_K, master_Kc = self.master_public_keys[account]
 608         try:
 609             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 610             assert K.encode('hex') == master_K
 611         except Exception:
 612             raise Exception("Invalid password")
 613         return master_k
 614
 615
 616     def get_address_index(self, address):
 617         if address in self.imported_keys.keys():
 618             return -1, None
 619
 620         for account in self.accounts.keys():
 621             for for_change in [0,1]:
 622                 addresses = self.accounts[account].get_addresses(for_change)
 623                 for addr in addresses:
 624                     if address == addr:
 625                         return account, (for_change, addresses.index(addr))
 626
 627         for k,v in self.next_addresses.items():
 628             if v == address:
 629                 return k, (0,0)
 630
 631         raise Exception("Address not found", address)
 632
 633
 634     def get_roots(self, account):
 635         roots = []
 636         for a in account.split('&'):
 637             s = a.strip()
 638             m = re.match("(m/\d+'/)(\d+)", s)
 639             roots.append( m.group(1) )
 640         return roots
 641
 642     def is_seeded(self, account):
 643         if type(account) is int:
 644             return self.seed is not None
 645
 646         for root in self.get_roots(account):
 647             if root not in self.master_private_keys.keys():
 648                 return False
 649         return True
 650
 651     def rebase_sequence(self, account, sequence):
 652         c, i = sequence
 653         dd = []
 654         for a in account.split('&'):
 655             s = a.strip()
 656             m = re.match("(m/\d+'/)(\d+)", s)
 657             root = m.group(1)
 658             num = int(m.group(2))
 659             dd.append( (root, [num,c,i] ) )
 660         return dd
 661
 662
 663     def get_keyID(self, account, sequence):
 664         if account == 0:
 665             a, b = sequence
 666             mpk = self.storage.get('master_public_key')
 667             return 'old(%s,%d,%d)'%(mpk,a,b)
 668
 669         rs = self.rebase_sequence(account, sequence)
 670         dd = []
 671         for root, public_sequence in rs:
 672             c, K, _ = self.master_public_keys[root]
 673             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 674             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 675         return '&'.join(dd)
 676
 677
 678
 679     def get_seed(self, password):
 680         s = pw_decode(self.seed, password)
 681         if self.seed_version == 4:
 682             seed = s
 683             self.accounts[0].check_seed(seed)
 684         else:
 685             seed = mnemonic_hash(s)
 686         return seed
 687
 688
 689     def get_mnemonic(self, password):
 690         import mnemonic
 691         s = pw_decode(self.seed, password)
 692         if self.seed_version == 4:
 693             return ' '.join(mnemonic.mn_encode(s))
 694         else:
 695             return s
 696
 697
 698
 699     def get_private_key(self, address, password):
 700         if self.is_watching_only():
 701             return []
 702
 703         # first check the provided password
 704         seed = self.get_seed(password)
 705
 706         out = []
 707         if address in self.imported_keys.keys():
 708             out.append( pw_decode( self.imported_keys[address], password ) )
 709         else:
 710             account, sequence = self.get_address_index(address)
 711             if account == 0:
 712                 pk = self.accounts[account].get_private_key(seed, sequence)
 713                 out.append(pk)
 714                 return out
 715
 716             # assert address == self.accounts[account].get_address(*sequence)
 717             rs = self.rebase_sequence( account, sequence)
 718             for root, public_sequence in rs:
 719
 720                 if root not in self.master_private_keys.keys(): continue
 721                 master_k = self.get_master_private_key(root, password)
 722                 master_c, _, _ = self.master_public_keys[root]
 723                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 724                 out.append(pk)
 725
 726         return out
 727
 728
 729     def add_keypairs_from_wallet(self, tx, keypairs, password):
 730         for txin in tx.inputs:
 731             address = txin['address']
 732             if not self.is_mine(address):
 733                 continue
 734             private_keys = self.get_private_key(address, password)
 735             for sec in private_keys:
 736                 pubkey = public_key_from_private_key(sec)
 737                 keypairs[ pubkey ] = sec
 738                 if address in self.imported_keys.keys():
 739                     txin['redeemPubkey'] = pubkey
 740
 741
 742     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 743         # first check the provided password
 744         seed = self.get_seed(password)
 745
 746         for txin in tx.inputs:
 747             keyid = txin.get('KeyID')
 748             if keyid:
 749
 750                 if self.seed_version == 4:
 751                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 752                     if not m: continue
 753                     mpk = m.group(1)
 754                     if mpk != self.storage.get('master_public_key'): continue
 755                     for_change = int(m.group(2))
 756                     num = int(m.group(3))
 757                     account = self.accounts[0]
 758                     addr = account.get_address(for_change, num)
 759                     txin['address'] = addr # fixme: side effect
 760                     pk = account.get_private_key(seed, (for_change, num))
 761                     pubkey = public_key_from_private_key(pk)
 762                     keypairs[pubkey] = pk
 763                     continue
 764
 765
 766                 roots = []
 767                 for s in keyid.split('&'):
 768                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 769                     if not m: continue
 770                     c = m.group(1)
 771                     K = m.group(2)
 772                     sequence = m.group(3)
 773                     root = self.find_root_by_master_key(c,K)
 774                     if not root: continue
 775                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 776                     root = root + '%d'%sequence[0]
 777                     sequence = sequence[1:]
 778                     roots.append((root,sequence))
 779
 780                 account_id = " & ".join( map(lambda x:x[0], roots) )
 781                 account = self.accounts.get(account_id)
 782                 if not account: continue
 783                 addr = account.get_address(*sequence)
 784                 txin['address'] = addr # fixme: side effect
 785                 pk = self.get_private_key(addr, password)
 786                 for sec in pk:
 787                     pubkey = public_key_from_private_key(sec)
 788                     keypairs[pubkey] = sec
 789
 790
 791
 792     def signrawtransaction(self, tx, input_info, private_keys, password):
 793
 794         # check that the password is correct
 795         seed = self.get_seed(password)
 796
 797         # add input info
 798         tx.add_input_info(input_info)
 799
 800         # add redeem script for coins that are in the wallet
 801         # FIXME: add redeemPubkey too!
 802         unspent_coins = self.get_unspent_coins()
 803         for txin in tx.inputs:
 804             for item in unspent_coins:
 805                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 806                     print_error( "tx input is in unspent coins" )
 807                     txin['scriptPubKey'] = item['scriptPubKey']
 808                     account, sequence = self.get_address_index(item['address'])
 809                     if account != -1:
 810                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 811                         print_error("added redeemScript", txin['redeemScript'])
 812                     break
 813
 814
 815         # build a list of public/private keys
 816         keypairs = {}
 817
 818         # add private keys from parameter
 819         for sec in private_keys:
 820             pubkey = public_key_from_private_key(sec)
 821             keypairs[ pubkey ] = sec
 822
 823         # add private_keys from KeyID
 824         self.add_keypairs_from_KeyID(tx, keypairs, password)
 825
 826         # add private keys from wallet
 827         self.add_keypairs_from_wallet(tx, keypairs, password)
 828         self.sign_transaction(tx, keypairs)
 829
 830
 831     def sign_message(self, address, message, password):
 832         keys = self.get_private_key(address, password)
 833         assert len(keys) == 1
 834         sec = keys[0]
 835         key = regenerate_key(sec)
 836         compressed = is_compressed(sec)
 837         return key.sign_message(message, compressed, address)
 838
 839
 840     def change_gap_limit(self, value):
 841         if value >= self.gap_limit:
 842             self.gap_limit = value
 843             self.storage.put('gap_limit', self.gap_limit, True)
 844             #self.interface.poke('synchronizer')
 845             return True
 846
 847         elif value >= self.min_acceptable_gap():
 848             for key, account in self.accounts.items():
 849                 addresses = account[0]
 850                 k = self.num_unused_trailing_addresses(addresses)
 851                 n = len(addresses) - k + value
 852                 addresses = addresses[0:n]
 853                 self.accounts[key][0] = addresses
 854
 855             self.gap_limit = value
 856             self.storage.put('gap_limit', self.gap_limit, True)
 857             self.save_accounts()
 858             return True
 859         else:
 860             return False
 861
 862     def num_unused_trailing_addresses(self, addresses):
 863         k = 0
 864         for a in addresses[::-1]:
 865             if self.history.get(a):break
 866             k = k + 1
 867         return k
 868
 869     def min_acceptable_gap(self):
 870         # fixme: this assumes wallet is synchronized
 871         n = 0
 872         nmax = 0
 873
 874         for account in self.accounts.values():
 875             addresses = account.get_addresses(0)
 876             k = self.num_unused_trailing_addresses(addresses)
 877             for a in addresses[0:-k]:
 878                 if self.history.get(a):
 879                     n = 0
 880                 else:
 881                     n += 1
 882                     if n > nmax: nmax = n
 883         return nmax + 1
 884
 885
 886     def address_is_old(self, address):
 887         age = -1
 888         h = self.history.get(address, [])
 889         if h == ['*']:
 890             return True
 891         for tx_hash, tx_height in h:
 892             if tx_height == 0:
 893                 tx_age = 0
 894             else:
 895                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 896             if tx_age > age:
 897                 age = tx_age
 898         return age > 2
 899
 900
 901     def synchronize_sequence(self, account, for_change):
 902         limit = self.gap_limit_for_change if for_change else self.gap_limit
 903         new_addresses = []
 904         while True:
 905             addresses = account.get_addresses(for_change)
 906             if len(addresses) < limit:
 907                 address = account.create_new_address(for_change)
 908                 self.history[address] = []
 909                 new_addresses.append( address )
 910                 continue
 911
 912             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 913                 break
 914             else:
 915                 address = account.create_new_address(for_change)
 916                 self.history[address] = []
 917                 new_addresses.append( address )
 918
 919         return new_addresses
 920
 921
 922
 923     def create_pending_accounts(self):
 924         for account_type in ['1','2of2','2of3']:
 925             if not self.has_master_public_keys(account_type):
 926                 continue
 927             k, a = self.new_account_address(account_type)
 928             if self.address_is_old(a):
 929                 print_error( "creating account", a )
 930                 self.create_account(account_type)
 931                 self.next_addresses.pop(k)
 932
 933
 934     def synchronize_account(self, account):
 935         new = []
 936         new += self.synchronize_sequence(account, 0)
 937         new += self.synchronize_sequence(account, 1)
 938         return new
 939
 940
 941     def synchronize(self):
 942         if self.master_public_keys:
 943             self.create_pending_accounts()
 944         new = []
 945         for account in self.accounts.values():
 946             new += self.synchronize_account(account)
 947         if new:
 948             self.save_accounts()
 949             self.storage.put('addr_history', self.history, True)
 950         return new
 951
 952
 953     def is_found(self):
 954         return self.history.values() != [[]] * len(self.history)
 955
 956
 957     def add_contact(self, address, label=None):
 958         self.addressbook.append(address)
 959         self.storage.put('contacts', self.addressbook, True)
 960         if label:
 961             self.set_label(address, label)
 962
 963
 964     def delete_contact(self, addr):
 965         if addr in self.addressbook:
 966             self.addressbook.remove(addr)
 967             self.storage.put('addressbook', self.addressbook, True)
 968
 969
 970     def fill_addressbook(self):
 971         for tx_hash, tx in self.transactions.items():
 972             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 973             if is_send:
 974                 for addr, v in tx.outputs:
 975                     if not self.is_mine(addr) and addr not in self.addressbook:
 976                         self.addressbook.append(addr)
 977         # redo labels
 978         # self.update_tx_labels()
 979
 980     def get_num_tx(self, address):
 981         n = 0
 982         for tx in self.transactions.values():
 983             if address in map(lambda x:x[0], tx.outputs): n += 1
 984         return n
 985
 986
 987     def get_address_flags(self, addr):
 988         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 989         flags += "F" if addr in self.frozen_addresses else "-"
 990         return flags
 991
 992
 993     def get_tx_value(self, tx, account=None):
 994         domain = self.get_account_addresses(account)
 995         return tx.get_value(domain, self.prevout_values)
 996
 997
 998     def update_tx_outputs(self, tx_hash):
 999         tx = self.transactions.get(tx_hash)
1000
1001         for i, (addr, value) in enumerate(tx.outputs):
1002             key = tx_hash+ ':%d'%i
1003             self.prevout_values[key] = value
1004
1005         for item in tx.inputs:
1006             if self.is_mine(item.get('address')):
1007                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1008                 self.spent_outputs.append(key)
1009
1010
1011     def get_addr_balance(self, address):
1012         assert self.is_mine(address)
1013         h = self.history.get(address,[])
1014         if h == ['*']: return 0,0
1015         c = u = 0
1016         received_coins = []   # list of coins received at address
1017
1018         for tx_hash, tx_height in h:
1019             tx = self.transactions.get(tx_hash)
1020             if not tx: continue
1021
1022             for i, (addr, value) in enumerate(tx.outputs):
1023                 if addr == address:
1024                     key = tx_hash + ':%d'%i
1025                     received_coins.append(key)
1026
1027         for tx_hash, tx_height in h:
1028             tx = self.transactions.get(tx_hash)
1029             if not tx: continue
1030             v = 0
1031
1032             for item in tx.inputs:
1033                 addr = item.get('address')
1034                 if addr == address:
1035                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1036                     value = self.prevout_values.get( key )
1037                     if key in received_coins:
1038                         v -= value
1039
1040             for i, (addr, value) in enumerate(tx.outputs):
1041                 key = tx_hash + ':%d'%i
1042                 if addr == address:
1043                     v += value
1044
1045             if tx_height:
1046                 c += v
1047             else:
1048                 u += v
1049         return c, u
1050
1051
1052     def get_account_name(self, k):
1053         if k == 0:
1054             if self.seed_version == 4:
1055                 name = 'Main account'
1056             else:
1057                 name = 'Old account'
1058         else:
1059             default = "Unnamed account"
1060             m = re.match("m/0'/(\d+)", k)
1061             if m:
1062                 num = m.group(1)
1063                 if num == '0':
1064                     default = "Main account"
1065                 else:
1066                     default = "Account %s"%num
1067
1068             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1069             if m:
1070                 num = m.group(1)
1071                 default = "2of2 account %s"%num
1072             name = self.labels.get(k, default)
1073
1074         return name
1075
1076     def get_account_names(self):
1077         accounts = {}
1078         for k, account in self.accounts.items():
1079             accounts[k] = self.get_account_name(k)
1080         if self.imported_keys:
1081             accounts[-1] = 'Imported keys'
1082         return accounts
1083
1084     def get_account_addresses(self, a, include_change=True):
1085         if a is None:
1086             o = self.addresses(True)
1087         elif a == -1:
1088             o = self.imported_keys.keys()
1089         else:
1090             ac = self.accounts[a]
1091             o = ac.get_addresses(0)
1092             if include_change: o += ac.get_addresses(1)
1093         return o
1094
1095     def get_imported_balance(self):
1096         return self.get_balance(self.imported_keys.keys())
1097
1098     def get_account_balance(self, account):
1099         return self.get_balance(self.get_account_addresses(account))
1100
1101     def get_frozen_balance(self):
1102         return self.get_balance(self.frozen_addresses)
1103
1104     def get_balance(self, domain=None):
1105         if domain is None: domain = self.addresses(True)
1106         cc = uu = 0
1107         for addr in domain:
1108             c, u = self.get_addr_balance(addr)
1109             cc += c
1110             uu += u
1111         return cc, uu
1112
1113
1114     def get_unspent_coins(self, domain=None):
1115         coins = []
1116         if domain is None: domain = self.addresses(True)
1117         for addr in domain:
1118             h = self.history.get(addr, [])
1119             if h == ['*']: continue
1120             for tx_hash, tx_height in h:
1121                 tx = self.transactions.get(tx_hash)
1122                 if tx is None: raise Exception("Wallet not synchronized")
1123                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1124                 for o in tx.d.get('outputs'):
1125                     output = o.copy()
1126                     if output.get('address') != addr: continue
1127                     key = tx_hash + ":%d" % output.get('prevout_n')
1128                     if key in self.spent_outputs: continue
1129                     output['prevout_hash'] = tx_hash
1130                     output['height'] = tx_height
1131                     output['coinbase'] = is_coinbase
1132                     coins.append((tx_height, output))
1133
1134         # sort by age
1135         if coins:
1136             coins = sorted(coins)
1137             if coins[-1][0] != 0:
1138                 while coins[0][0] == 0:
1139                     coins = coins[1:] + [ coins[0] ]
1140         return [x[1] for x in coins]
1141
1142
1143     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1144         """ todo: minimize tx size """
1145         total = 0
1146         fee = self.fee if fixed_fee is None else fixed_fee
1147         if domain is None:
1148             domain = self.addresses(True)
1149
1150         for i in self.frozen_addresses:
1151             if i in domain: domain.remove(i)
1152
1153         coins = self.get_unspent_coins(domain)
1154         inputs = []
1155
1156         for item in coins:
1157             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1158                 continue
1159             addr = item.get('address')
1160             v = item.get('value')
1161             total += v
1162             inputs.append(item)
1163             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1164             if total >= amount + fee: break
1165         else:
1166             inputs = []
1167
1168         return inputs, total, fee
1169
1170
1171     def set_fee(self, fee):
1172         if self.fee != fee:
1173             self.fee = fee
1174             self.storage.put('fee_per_kb', self.fee, True)
1175
1176     def estimated_fee(self, inputs):
1177         estimated_size =  len(inputs) * 180 + 80    # this assumes non-compressed keys
1178         fee = self.fee * int(math.ceil(estimated_size/1000.))
1179         return fee
1180
1181
1182     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1183         "add change to a transaction"
1184         change_amount = total - ( amount + fee )
1185         if change_amount > DUST_THRESHOLD:
1186             if not change_addr:
1187
1188                 # send change to one of the accounts involved in the tx
1189                 address = inputs[0].get('address')
1190                 account, _ = self.get_address_index(address)
1191
1192                 if not self.use_change or account == -1:
1193                     change_addr = inputs[-1]['address']
1194                 else:
1195                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1196
1197             # Insert the change output at a random position in the outputs
1198             posn = random.randint(0, len(outputs))
1199             outputs[posn:posn] = [( change_addr,  change_amount)]
1200         return outputs
1201
1202
1203     def get_history(self, address):
1204         with self.lock:
1205             return self.history.get(address)
1206
1207
1208     def get_status(self, h):
1209         if not h: return None
1210         if h == ['*']: return '*'
1211         status = ''
1212         for tx_hash, height in h:
1213             status += tx_hash + ':%d:' % height
1214         return hashlib.sha256( status ).digest().encode('hex')
1215
1216
1217     def receive_tx_callback(self, tx_hash, tx, tx_height):
1218
1219         with self.transaction_lock:
1220             self.add_extra_addresses(tx)
1221             if not self.check_new_tx(tx_hash, tx):
1222                 # may happen due to pruning
1223                 print_error("received transaction that is no longer referenced in history", tx_hash)
1224                 return
1225             self.transactions[tx_hash] = tx
1226             self.network.interface.pending_transactions_for_notifications.append(tx)
1227             self.save_transactions()
1228             if self.verifier and tx_height>0:
1229                 self.verifier.add(tx_hash, tx_height)
1230             self.update_tx_outputs(tx_hash)
1231
1232
1233     def save_transactions(self):
1234         tx = {}
1235         for k,v in self.transactions.items():
1236             tx[k] = str(v)
1237         self.storage.put('transactions', tx, True)
1238
1239     def receive_history_callback(self, addr, hist):
1240
1241         if not self.check_new_history(addr, hist):
1242             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1243
1244         with self.lock:
1245             self.history[addr] = hist
1246             self.storage.put('addr_history', self.history, True)
1247
1248         if hist != ['*']:
1249             for tx_hash, tx_height in hist:
1250                 if tx_height>0:
1251                     # add it in case it was previously unconfirmed
1252                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1253
1254
1255     def get_tx_history(self, account=None):
1256         if not self.verifier:
1257             return []
1258
1259         with self.transaction_lock:
1260             history = self.transactions.items()
1261             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1262             result = []
1263
1264             balance = 0
1265             for tx_hash, tx in history:
1266                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1267                 if v is not None: balance += v
1268
1269             c, u = self.get_account_balance(account)
1270
1271             if balance != c+u:
1272                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1273
1274             balance = c + u - balance
1275             for tx_hash, tx in history:
1276                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1277                 if not is_relevant:
1278                     continue
1279                 if value is not None:
1280                     balance += value
1281
1282                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1283                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1284
1285         return result
1286
1287
1288     def get_label(self, tx_hash):
1289         label = self.labels.get(tx_hash)
1290         is_default = (label == '') or (label is None)
1291         if is_default: label = self.get_default_label(tx_hash)
1292         return label, is_default
1293
1294
1295     def get_default_label(self, tx_hash):
1296         tx = self.transactions.get(tx_hash)
1297         default_label = ''
1298         if tx:
1299             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1300             if is_mine:
1301                 for o in tx.outputs:
1302                     o_addr, _ = o
1303                     if not self.is_mine(o_addr):
1304                         try:
1305                             default_label = self.labels[o_addr]
1306                         except KeyError:
1307                             default_label = o_addr
1308                         break
1309                 else:
1310                     default_label = '(internal)'
1311             else:
1312                 for o in tx.outputs:
1313                     o_addr, _ = o
1314                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1315                         break
1316                 else:
1317                     for o in tx.outputs:
1318                         o_addr, _ = o
1319                         if self.is_mine(o_addr):
1320                             break
1321                     else:
1322                         o_addr = None
1323
1324                 if o_addr:
1325                     dest_label = self.labels.get(o_addr)
1326                     try:
1327                         default_label = self.labels[o_addr]
1328                     except KeyError:
1329                         default_label = o_addr
1330
1331         return default_label
1332
1333
1334     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1335         for address, x in outputs:
1336             assert is_valid(address), "Address " + address + " is invalid!"
1337         amount = sum( map(lambda x:x[1], outputs) )
1338         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1339         if not inputs:
1340             raise ValueError("Not enough funds")
1341         self.add_input_info(inputs)
1342         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1343         return Transaction.from_io(inputs, outputs)
1344
1345
1346     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1347         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1348         keypairs = {}
1349         self.add_keypairs_from_wallet(tx, keypairs, password)
1350         if keypairs:
1351             self.sign_transaction(tx, keypairs)
1352         return tx
1353
1354
1355     def add_input_info(self, inputs):
1356         for txin in inputs:
1357             address = txin['address']
1358             if address in self.imported_keys.keys():
1359                 continue
1360             account, sequence = self.get_address_index(address)
1361             txin['KeyID'] = self.get_keyID(account, sequence)
1362             redeemScript = self.accounts[account].redeem_script(sequence)
1363             if redeemScript:
1364                 txin['redeemScript'] = redeemScript
1365             else:
1366                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1367
1368
1369     def sign_transaction(self, tx, keypairs):
1370         tx.sign(keypairs)
1371         run_hook('sign_transaction', tx)
1372
1373
1374     def sendtx(self, tx):
1375         # synchronous
1376         h = self.send_tx(tx)
1377         self.tx_event.wait()
1378         return self.receive_tx(h, tx)
1379
1380     def send_tx(self, tx):
1381         # asynchronous
1382         self.tx_event.clear()
1383         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1384         return tx.hash()
1385
1386     def on_broadcast(self, i, r):
1387         self.tx_result = r.get('result')
1388         self.tx_event.set()
1389
1390     def receive_tx(self, tx_hash, tx):
1391         out = self.tx_result
1392         if out != tx_hash:
1393             return False, "error: " + out
1394         run_hook('receive_tx', tx, self)
1395         return True, out
1396
1397
1398
1399     def update_password(self, old_password, new_password):
1400         if new_password == '': new_password = None
1401         decoded = self.get_seed(old_password)
1402         self.seed = pw_encode( decoded, new_password)
1403         self.storage.put('seed', self.seed, True)
1404         self.use_encryption = (new_password != None)
1405         self.storage.put('use_encryption', self.use_encryption,True)
1406         for k in self.imported_keys.keys():
1407             a = self.imported_keys[k]
1408             b = pw_decode(a, old_password)
1409             c = pw_encode(b, new_password)
1410             self.imported_keys[k] = c
1411         self.storage.put('imported_keys', self.imported_keys, True)
1412
1413         for k, v in self.master_private_keys.items():
1414             b = pw_decode(v, old_password)
1415             c = pw_encode(b, new_password)
1416             self.master_private_keys[k] = c
1417         self.storage.put('master_private_keys', self.master_private_keys, True)
1418
1419
1420     def freeze(self,addr):
1421         if self.is_mine(addr) and addr not in self.frozen_addresses:
1422             self.frozen_addresses.append(addr)
1423             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1424             return True
1425         else:
1426             return False
1427
1428     def unfreeze(self,addr):
1429         if self.is_mine(addr) and addr in self.frozen_addresses:
1430             self.frozen_addresses.remove(addr)
1431             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1432             return True
1433         else:
1434             return False
1435
1436
1437     def set_verifier(self, verifier):
1438         self.verifier = verifier
1439
1440         # review transactions that are in the history
1441         for addr, hist in self.history.items():
1442             if hist == ['*']: continue
1443             for tx_hash, tx_height in hist:
1444                 if tx_height>0:
1445                     # add it in case it was previously unconfirmed
1446                     self.verifier.add(tx_hash, tx_height)
1447
1448
1449         # if we are on a pruning server, remove unverified transactions
1450         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1451         for tx_hash in self.transactions.keys():
1452             if tx_hash not in vr:
1453                 self.transactions.pop(tx_hash)
1454
1455
1456
1457     def check_new_history(self, addr, hist):
1458
1459         # check that all tx in hist are relevant
1460         if hist != ['*']:
1461             for tx_hash, height in hist:
1462                 tx = self.transactions.get(tx_hash)
1463                 if not tx: continue
1464                 if not tx.has_address(addr):
1465                     return False
1466
1467         # check that we are not "orphaning" a transaction
1468         old_hist = self.history.get(addr,[])
1469         if old_hist == ['*']: return True
1470
1471         for tx_hash, height in old_hist:
1472             if tx_hash in map(lambda x:x[0], hist): continue
1473             found = False
1474             for _addr, _hist in self.history.items():
1475                 if _addr == addr: continue
1476                 if _hist == ['*']: continue
1477                 _tx_hist = map(lambda x:x[0], _hist)
1478                 if tx_hash in _tx_hist:
1479                     found = True
1480                     break
1481
1482             if not found:
1483                 tx = self.transactions.get(tx_hash)
1484                 # tx might not be there
1485                 if not tx: continue
1486
1487                 # already verified?
1488                 if self.verifier.get_height(tx_hash):
1489                     continue
1490                 # unconfirmed tx
1491                 print_error("new history is orphaning transaction:", tx_hash)
1492                 # check that all outputs are not mine, request histories
1493                 ext_requests = []
1494                 for _addr, _v in tx.outputs:
1495                     # assert not self.is_mine(_addr)
1496                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1497
1498                 ext_h = self.network.synchronous_get(ext_requests)
1499                 print_error("sync:", ext_requests, ext_h)
1500                 height = None
1501                 for h in ext_h:
1502                     if h == ['*']: continue
1503                     for item in h:
1504                         if item.get('tx_hash') == tx_hash:
1505                             height = item.get('height')
1506                 if height:
1507                     print_error("found height for", tx_hash, height)
1508                     self.verifier.add(tx_hash, height)
1509                 else:
1510                     print_error("removing orphaned tx from history", tx_hash)
1511                     self.transactions.pop(tx_hash)
1512
1513         return True
1514
1515
1516
1517     def check_new_tx(self, tx_hash, tx):
1518         # 1 check that tx is referenced in addr_history.
1519         addresses = []
1520         for addr, hist in self.history.items():
1521             if hist == ['*']:continue
1522             for txh, height in hist:
1523                 if txh == tx_hash:
1524                     addresses.append(addr)
1525
1526         if not addresses:
1527             return False
1528
1529         # 2 check that referencing addresses are in the tx
1530         for addr in addresses:
1531             if not tx.has_address(addr):
1532                 return False
1533
1534         return True
1535
1536
1537     def start_threads(self, network):
1538         from verifier import TxVerifier
1539         self.network = network
1540         if self.network:
1541             self.verifier = TxVerifier(self.network, self.storage)
1542             self.verifier.start()
1543             self.set_verifier(self.verifier)
1544             self.synchronizer = WalletSynchronizer(self, network)
1545             self.synchronizer.start()
1546         else:
1547             self.verifier = None
1548             self.synchronizer =None
1549
1550     def stop_threads(self):
1551         if self.network:
1552             self.verifier.stop()
1553             self.synchronizer.stop()
1554
1555
1556     def restore(self, callback):
1557         from i18n import _
1558         def wait_for_wallet():
1559             self.set_up_to_date(False)
1560             while not self.is_up_to_date():
1561                 msg = "%s\n%s %d\n%s %.1f"%(
1562                     _("Please wait..."),
1563                     _("Addresses generated:"),
1564                     len(self.addresses(True)),
1565                     _("Kilobytes received:"),
1566                     self.network.interface.bytes_received/1024.)
1567
1568                 apply(callback, (msg,))
1569                 time.sleep(0.1)
1570
1571         def wait_for_network():
1572             while not self.network.is_connected():
1573                 msg = "%s \n" % (_("Connecting..."))
1574                 apply(callback, (msg,))
1575                 time.sleep(0.1)
1576
1577         # wait until we are connected, because the user might have selected another server
1578         if self.network:
1579             wait_for_network()
1580             wait_for_wallet()
1581         else:
1582             self.synchronize()
1583
1584         self.fill_addressbook()
1585
1586
1587
1588
1589 class WalletSynchronizer(threading.Thread):
1590
1591
1592     def __init__(self, wallet, network):
1593         threading.Thread.__init__(self)
1594         self.daemon = True
1595         self.wallet = wallet
1596         self.network = network
1597         self.was_updated = True
1598         self.running = False
1599         self.lock = threading.Lock()
1600         self.queue = Queue.Queue()
1601
1602     def stop(self):
1603         with self.lock: self.running = False
1604
1605     def is_running(self):
1606         with self.lock: return self.running
1607
1608
1609     def subscribe_to_addresses(self, addresses):
1610         messages = []
1611         for addr in addresses:
1612             messages.append(('blockchain.address.subscribe', [addr]))
1613         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1614
1615
1616     def run(self):
1617         with self.lock:
1618             self.running = True
1619
1620         while self.is_running():
1621
1622             if not self.network.is_connected():
1623                 self.network.wait_until_connected()
1624
1625             self.run_interface(self.network.interface)
1626
1627
1628     def run_interface(self, interface):
1629
1630         print_error("synchronizer: connected to", interface.server)
1631
1632         requested_tx = []
1633         missing_tx = []
1634         requested_histories = {}
1635
1636         # request any missing transactions
1637         for history in self.wallet.history.values():
1638             if history == ['*']: continue
1639             for tx_hash, tx_height in history:
1640                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1641                     missing_tx.append( (tx_hash, tx_height) )
1642
1643         if missing_tx:
1644             print_error("missing tx", missing_tx)
1645
1646         # subscriptions
1647         self.subscribe_to_addresses(self.wallet.addresses(True))
1648
1649         while self.is_running():
1650             # 1. create new addresses
1651             new_addresses = self.wallet.synchronize()
1652
1653             # request missing addresses
1654             if new_addresses:
1655                 self.subscribe_to_addresses(new_addresses)
1656
1657             # request missing transactions
1658             for tx_hash, tx_height in missing_tx:
1659                 if (tx_hash, tx_height) not in requested_tx:
1660                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1661                     requested_tx.append( (tx_hash, tx_height) )
1662             missing_tx = []
1663
1664             # detect if situation has changed
1665             if interface.is_up_to_date() and self.queue.empty():
1666                 if not self.wallet.is_up_to_date():
1667                     self.wallet.set_up_to_date(True)
1668                     self.was_updated = True
1669             else:
1670                 if self.wallet.is_up_to_date():
1671                     self.wallet.set_up_to_date(False)
1672                     self.was_updated = True
1673
1674             if self.was_updated:
1675                 self.wallet.network.trigger_callback('updated')
1676                 self.was_updated = False
1677
1678             # 2. get a response
1679             try:
1680                 r = self.queue.get(block=True, timeout=1)
1681             except Queue.Empty:
1682                 continue
1683
1684             if interface != self.network.interface:
1685                 break
1686
1687             if not r:
1688                 continue
1689
1690             # 3. handle response
1691             method = r['method']
1692             params = r['params']
1693             result = r.get('result')
1694             error = r.get('error')
1695             if error:
1696                 print "error", r
1697                 continue
1698
1699             if method == 'blockchain.address.subscribe':
1700                 addr = params[0]
1701                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1702                     if requested_histories.get(addr) is None:
1703                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1704                         requested_histories[addr] = result
1705
1706             elif method == 'blockchain.address.get_history':
1707                 addr = params[0]
1708                 print_error("receiving history", addr, result)
1709                 if result == ['*']:
1710                     assert requested_histories.pop(addr) == '*'
1711                     self.wallet.receive_history_callback(addr, result)
1712                 else:
1713                     hist = []
1714                     # check that txids are unique
1715                     txids = []
1716                     for item in result:
1717                         tx_hash = item['tx_hash']
1718                         if tx_hash not in txids:
1719                             txids.append(tx_hash)
1720                             hist.append( (tx_hash, item['height']) )
1721
1722                     if len(hist) != len(result):
1723                         raise Exception("error: server sent history with non-unique txid", result)
1724
1725                     # check that the status corresponds to what was announced
1726                     rs = requested_histories.pop(addr)
1727                     if self.wallet.get_status(hist) != rs:
1728                         raise Exception("error: status mismatch: %s"%addr)
1729
1730                     # store received history
1731                     self.wallet.receive_history_callback(addr, hist)
1732
1733                     # request transactions that we don't have
1734                     for tx_hash, tx_height in hist:
1735                         if self.wallet.transactions.get(tx_hash) is None:
1736                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1737                                 missing_tx.append( (tx_hash, tx_height) )
1738
1739             elif method == 'blockchain.transaction.get':
1740                 tx_hash = params[0]
1741                 tx_height = params[1]
1742                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1743                 tx = Transaction(result)
1744                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1745                 self.was_updated = True
1746                 requested_tx.remove( (tx_hash, tx_height) )
1747                 print_error("received tx:", tx_hash, len(tx.raw))
1748
1749             else:
1750                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1751
1752             if self.was_updated and not requested_tx:
1753                 self.wallet.network.trigger_callback('updated')
1754                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1755
1756                 self.was_updated = False