lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40
  41 # AES encryption
  42 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  43 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  44
  45 def pw_encode(s, password):
  46     if password:
  47         secret = Hash(password)
  48         return EncodeAES(secret, s)
  49     else:
  50         return s
  51
  52 def pw_decode(s, password):
  53     if password is not None:
  54         secret = Hash(password)
  55         try:
  56             d = DecodeAES(secret, s)
  57         except:
  58             raise BaseException('Invalid password')
  59         return d
  60     else:
  61         return s
  62
  63
  64
  65
  66
  67 from version import *
  68
  69
  70 class WalletStorage:
  71
  72     def __init__(self, config):
  73         self.lock = threading.Lock()
  74         self.data = {}
  75         self.file_exists = False
  76         self.path = self.init_path(config)
  77         print_error( "wallet path", self.path )
  78         if self.path:
  79             self.read(self.path)
  80
  81
  82     def init_path(self, config):
  83         """Set the path of the wallet."""
  84
  85         # command line -w option
  86         path = config.get('wallet_path')
  87         if path:
  88             return path
  89
  90         # path in config file
  91         path = config.get('default_wallet_path')
  92         if path:
  93             return path
  94
  95         # default path
  96         dirpath = os.path.join(config.path, "wallets")
  97         if not os.path.exists(dirpath):
  98             os.mkdir(dirpath)
  99
 100         new_path = os.path.join(config.path, "wallets", "default_wallet")
 101
 102         # default path in pre 1.9 versions
 103         old_path = os.path.join(config.path, "electrum.dat")
 104         if os.path.exists(old_path) and not os.path.exists(new_path):
 105             os.rename(old_path, new_path)
 106
 107         return new_path
 108
 109
 110     def read(self, path):
 111         """Read the contents of the wallet file."""
 112         try:
 113             with open(self.path, "r") as f:
 114                 data = f.read()
 115         except IOError:
 116             return
 117         try:
 118             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 119         except:
 120             raise IOError("Cannot read wallet file.")
 121
 122         self.data = d
 123         self.file_exists = True
 124
 125
 126     def get(self, key, default=None):
 127         return self.data.get(key, default)
 128
 129     def put(self, key, value, save = True):
 130
 131         with self.lock:
 132             if value is not None:
 133                 self.data[key] = value
 134             else:
 135                 self.data.pop[key]
 136             if save:
 137                 self.write()
 138
 139     def write(self):
 140         s = repr(self.data)
 141         f = open(self.path,"w")
 142         f.write( s )
 143         f.close()
 144         if 'ANDROID_DATA' not in os.environ:
 145             import stat
 146             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 147
 148
 149 class Wallet:
 150
 151     def __init__(self, storage):
 152
 153         self.storage = storage
 154         self.electrum_version = ELECTRUM_VERSION
 155         self.gap_limit_for_change = 3 # constant
 156
 157         # saved fields
 158         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 159
 160         self.gap_limit             = storage.get('gap_limit', 5)
 161         self.use_change            = storage.get('use_change',True)
 162         self.use_encryption        = storage.get('use_encryption', False)
 163         self.seed                  = storage.get('seed', '')               # encrypted
 164         self.labels                = storage.get('labels', {})
 165         self.frozen_addresses      = storage.get('frozen_addresses',[])
 166         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 167         self.addressbook           = storage.get('contacts', [])
 168
 169         self.imported_keys         = storage.get('imported_keys',{})
 170         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 171
 172         self.fee                   = int(storage.get('fee_per_kb',20000))
 173
 174         self.master_public_keys = storage.get('master_public_keys',{})
 175         self.master_private_keys = storage.get('master_private_keys', {})
 176
 177         self.next_addresses = storage.get('next_addresses',{})
 178
 179         if self.seed_version not in [4, 6]:
 180             msg = "This wallet seed is not supported."
 181             if self.seed_version in [5]: msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 182             raise ValueError(msg)
 183
 184         self.load_accounts()
 185
 186         self.transactions = {}
 187         tx_list = self.storage.get('transactions',{})
 188         for k,v in tx_list.items():
 189             try:
 190                 tx = Transaction(v)
 191             except:
 192                 print_msg("Warning: Cannot deserialize transactions. skipping")
 193                 continue
 194
 195             self.add_extra_addresses(tx)
 196             self.transactions[k] = tx
 197
 198         for h,tx in self.transactions.items():
 199             if not self.check_new_tx(h, tx):
 200                 print_error("removing unreferenced tx", h)
 201                 self.transactions.pop(h)
 202
 203
 204         # not saved
 205         self.prevout_values = {}     # my own transaction outputs
 206         self.spent_outputs = []
 207
 208         # spv
 209         self.verifier = None
 210
 211         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 212         # interface.is_up_to_date() returns true when all requests have been answered and processed
 213         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 214
 215         self.up_to_date = False
 216         self.lock = threading.Lock()
 217         self.transaction_lock = threading.Lock()
 218         self.tx_event = threading.Event()
 219
 220         for tx_hash, tx in self.transactions.items():
 221             self.update_tx_outputs(tx_hash)
 222
 223
 224     def add_extra_addresses(self, tx):
 225         h = tx.hash()
 226         # find the address corresponding to pay-to-pubkey inputs
 227         tx.add_extra_addresses(self.transactions)
 228         for o in tx.d.get('outputs'):
 229             if o.get('is_pubkey'):
 230                 for tx2 in self.transactions.values():
 231                     tx2.add_extra_addresses({h:tx})
 232
 233
 234
 235
 236     def set_up_to_date(self,b):
 237         with self.lock: self.up_to_date = b
 238
 239
 240     def is_up_to_date(self):
 241         with self.lock: return self.up_to_date
 242
 243
 244     def update(self):
 245         self.up_to_date = False
 246         while not self.is_up_to_date():
 247             time.sleep(0.1)
 248
 249
 250     def import_key(self, sec, password):
 251         # check password
 252         seed = self.get_seed(password)
 253         try:
 254             address = address_from_private_key(sec)
 255         except:
 256             raise BaseException('Invalid private key')
 257
 258         if self.is_mine(address):
 259             raise BaseException('Address already in wallet')
 260
 261         # store the originally requested keypair into the imported keys table
 262         self.imported_keys[address] = pw_encode(sec, password )
 263         self.storage.put('imported_keys', self.imported_keys, True)
 264         if self.synchronizer:
 265             self.synchronizer.subscribe_to_addresses([address])
 266         return address
 267
 268     def delete_imported_key(self, addr):
 269         if addr in self.imported_keys:
 270             self.imported_keys.pop(addr)
 271             self.storage.put('imported_keys', self.imported_keys, True)
 272
 273
 274     def make_seed(self):
 275         import mnemonic, ecdsa
 276         entropy = ecdsa.util.randrange( pow(2,160) )
 277         nonce = 0
 278         while True:
 279             ss = "%040x"%(entropy+nonce)
 280             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 281             # we keep only 13 words, that's approximately 139 bits of entropy
 282             words = mnemonic.mn_encode(s)[0:13]
 283             seed = ' '.join(words)
 284             if mnemonic_hash(seed)[0:3] == SEED_PREFIX:
 285                 break  # this removes 12 bits of entropy
 286             nonce += 1
 287
 288         return seed
 289
 290
 291     def init_seed(self, seed):
 292         if self.seed:
 293             raise BaseException("a seed exists")
 294
 295         if not seed:
 296             self.seed = self.make_seed()
 297             self.seed_version = SEED_VERSION
 298             return
 299
 300         # find out what kind of wallet we are
 301         try:
 302             seed.decode('hex')
 303             self.seed_version = 4
 304             return
 305         except:
 306             pass
 307
 308         words = seed.split()
 309         try:
 310             mnemonic.mn_decode(words)
 311             uses_electrum_words = True
 312         except:
 313             uses_electrum_words = False
 314
 315         if uses_electrum_words and len(words) != 13:
 316             self.seed_version = 4
 317             self.seed = mnemonic.mn_encode(seed)
 318         else:
 319             assert mnemonic_hash(seed)[0:3] == SEED_PREFIX
 320             self.seed_version = SEED_VERSION
 321             self.seed = seed
 322
 323
 324     def save_seed(self):
 325         self.storage.put('seed', self.seed, True)
 326         self.storage.put('seed_version', self.seed_version, True)
 327
 328     def create_watching_only_wallet(self, c0, K0):
 329         cK0 = ""
 330         self.master_public_keys = {
 331             "m/0'/": (c0, K0, cK0),
 332             }
 333         self.storage.put('master_public_keys', self.master_public_keys, True)
 334         self.create_account('1','Main account')
 335
 336
 337     def create_accounts(self):
 338         # create default account
 339         self.create_master_keys('1')
 340         self.create_account('1','Main account')
 341
 342
 343     def create_master_keys(self, account_type):
 344         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 345         if account_type == '1':
 346             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 347             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 348             self.master_private_keys["m/0'/"] = k0
 349         elif account_type == '2of2':
 350             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 351             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 352             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 353             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 354             self.master_private_keys["m/1'/"] = k1
 355             self.master_private_keys["m/2'/"] = k2
 356         elif account_type == '2of3':
 357             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 358             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 359             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 360             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 361             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 362             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 363             self.master_private_keys["m/3'/"] = k3
 364             self.master_private_keys["m/4'/"] = k4
 365             self.master_private_keys["m/5'/"] = k5
 366
 367         self.storage.put('master_public_keys', self.master_public_keys, True)
 368         self.storage.put('master_private_keys', self.master_private_keys, True)
 369
 370     def has_master_public_keys(self, account_type):
 371         if account_type == '1':
 372             return "m/0'/" in self.master_public_keys
 373         elif account_type == '2of2':
 374             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 375         elif account_type == '2of3':
 376             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 377
 378     def find_root_by_master_key(self, c, K):
 379         for key, v in self.master_public_keys.items():
 380             if key == "m/":continue
 381             cc, KK, _ = v
 382             if (c == cc) and (K == KK):
 383                 return key
 384
 385     def deseed_root(self, seed, password):
 386         # for safety, we ask the user to enter their seed
 387         assert seed == self.get_seed(password)
 388         self.seed = ''
 389         self.storage.put('seed', '', True)
 390
 391
 392     def deseed_branch(self, k):
 393         # check that parent has no seed
 394         assert self.seed == ''
 395         self.master_private_keys.pop(k)
 396         self.storage.put('master_private_keys', self.master_private_keys, True)
 397
 398     def is_watching_only(self):
 399         return (self.seed == '') and (self.master_private_keys == {})
 400
 401
 402
 403     def account_id(self, account_type, i):
 404         if account_type == '1':
 405             return "m/0'/%d"%i
 406         elif account_type == '2of2':
 407             return "m/1'/%d & m/2'/%d"%(i,i)
 408         elif account_type == '2of3':
 409             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 410         else:
 411             raise BaseException('unknown account type')
 412
 413
 414     def num_accounts(self, account_type):
 415         keys = self.accounts.keys()
 416         i = 0
 417         while True:
 418             account_id = self.account_id(account_type, i)
 419             if account_id not in keys: break
 420             i += 1
 421         return i
 422
 423
 424     def new_account_address(self, account_type = '1'):
 425         i = self.num_accounts(account_type)
 426         k = self.account_id(account_type,i)
 427
 428         addr = self.next_addresses.get(k)
 429         if not addr:
 430             account_id, account = self.next_account(account_type)
 431             addr = account.first_address()
 432             self.next_addresses[k] = addr
 433             self.storage.put('next_addresses',self.next_addresses)
 434
 435         return k, addr
 436
 437
 438     def next_account(self, account_type = '1'):
 439
 440         i = self.num_accounts(account_type)
 441         account_id = self.account_id(account_type,i)
 442
 443         if account_type is '1':
 444             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 445             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 446             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 447
 448         elif account_type == '2of2':
 449             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 450             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 451             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 452             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 453             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 454
 455         elif account_type == '2of3':
 456             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 457             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 458             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 459             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 460             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 461             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 462             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 463
 464         return account_id, account
 465
 466
 467     def set_label(self, name, text = None):
 468         changed = False
 469         old_text = self.labels.get(name)
 470         if text:
 471             if old_text != text:
 472                 self.labels[name] = text
 473                 changed = True
 474         else:
 475             if old_text:
 476                 self.labels.pop(name)
 477                 changed = True
 478
 479         if changed:
 480             self.storage.put('labels', self.labels, True)
 481
 482         run_hook('set_label', name, text, changed)
 483         return changed
 484
 485
 486
 487     def create_account(self, account_type = '1', name = None):
 488         k, account = self.next_account(account_type)
 489         if k in self.pending_accounts:
 490             self.pending_accounts.pop(k)
 491             self.storage.put('pending_accounts', self.pending_accounts)
 492
 493         self.accounts[k] = account
 494         self.save_accounts()
 495         if name:
 496             self.set_label(k, name)
 497
 498
 499     def create_old_account(self):
 500         mpk = OldAccount.mpk_from_seed(self.seed)
 501         self.storage.put('master_public_key', mpk, True)
 502         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 503         self.save_accounts()
 504
 505
 506     def save_accounts(self):
 507         d = {}
 508         for k, v in self.accounts.items():
 509             d[k] = v.dump()
 510         self.storage.put('accounts', d, True)
 511
 512
 513
 514     def load_accounts(self):
 515         d = self.storage.get('accounts', {})
 516         self.accounts = {}
 517         for k, v in d.items():
 518             if k == 0:
 519                 v['mpk'] = self.storage.get('master_public_key')
 520                 self.accounts[k] = OldAccount(v)
 521             elif '&' in k:
 522                 self.accounts[k] = BIP32_Account_2of2(v)
 523             else:
 524                 self.accounts[k] = BIP32_Account(v)
 525
 526         self.pending_accounts = self.storage.get('pending_accounts',{})
 527
 528
 529     def delete_pending_account(self, k):
 530         self.pending_accounts.pop(k)
 531         self.storage.put('pending_accounts', self.pending_accounts)
 532
 533     def account_is_pending(self, k):
 534         return k in self.pending_accounts
 535
 536     def create_pending_account(self, acct_type, name):
 537         k, addr = self.new_account_address(acct_type)
 538         self.set_label(k, name)
 539         self.pending_accounts[k] = addr
 540         self.storage.put('pending_accounts', self.pending_accounts)
 541
 542     def get_pending_accounts(self):
 543         return self.pending_accounts.items()
 544
 545
 546     def addresses(self, include_change = True, _next=True):
 547         o = self.get_account_addresses(-1, include_change)
 548         for a in self.accounts.keys():
 549             o += self.get_account_addresses(a, include_change)
 550
 551         if _next:
 552             for addr in self.next_addresses.values():
 553                 if addr not in o:
 554                     o += [addr]
 555         return o
 556
 557
 558     def is_mine(self, address):
 559         return address in self.addresses(True)
 560
 561
 562     def is_change(self, address):
 563         if not self.is_mine(address): return False
 564         if address in self.imported_keys.keys(): return False
 565         acct, s = self.get_address_index(address)
 566         if s is None: return False
 567         return s[0] == 1
 568
 569     def get_master_public_key(self):
 570         if self.seed_version == 4:
 571             return self.storage.get("master_public_key")
 572         else:
 573             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 574             return repr((c, K))
 575
 576     def get_master_private_key(self, account, password):
 577         master_k = pw_decode( self.master_private_keys[account], password)
 578         master_c, master_K, master_Kc = self.master_public_keys[account]
 579         try:
 580             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 581             assert K.encode('hex') == master_K
 582         except:
 583             raise BaseException("Invalid password")
 584         return master_k
 585
 586
 587     def get_address_index(self, address):
 588         if address in self.imported_keys.keys():
 589             return -1, None
 590
 591         for account in self.accounts.keys():
 592             for for_change in [0,1]:
 593                 addresses = self.accounts[account].get_addresses(for_change)
 594                 for addr in addresses:
 595                     if address == addr:
 596                         return account, (for_change, addresses.index(addr))
 597
 598         for k,v in self.next_addresses.items():
 599             if v == address:
 600                 return k, (0,0)
 601
 602         raise BaseException("Address not found", address)
 603
 604
 605     def get_roots(self, account):
 606         roots = []
 607         for a in account.split('&'):
 608             s = a.strip()
 609             m = re.match("(m/\d+'/)(\d+)", s)
 610             roots.append( m.group(1) )
 611         return roots
 612
 613     def is_seeded(self, account):
 614         if type(account) is int:
 615             return self.seed is not None
 616
 617         for root in self.get_roots(account):
 618             if root not in self.master_private_keys.keys():
 619                 return False
 620         return True
 621
 622     def rebase_sequence(self, account, sequence):
 623         c, i = sequence
 624         dd = []
 625         for a in account.split('&'):
 626             s = a.strip()
 627             m = re.match("(m/\d+'/)(\d+)", s)
 628             root = m.group(1)
 629             num = int(m.group(2))
 630             dd.append( (root, [num,c,i] ) )
 631         return dd
 632
 633
 634     def get_keyID(self, account, sequence):
 635         if account == 0:
 636             return 'old'
 637
 638         rs = self.rebase_sequence(account, sequence)
 639         dd = []
 640         for root, public_sequence in rs:
 641             c, K, _ = self.master_public_keys[root]
 642             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 643             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 644         return '&'.join(dd)
 645
 646
 647
 648     def get_seed(self, password):
 649         s = pw_decode(self.seed, password)
 650         if self.seed_version == 4:
 651             seed = s
 652         else:
 653             seed = mnemonic_hash(s)
 654         #todo:  #self.sequences[0].check_seed(seed)
 655         return seed
 656
 657
 658     def get_mnemonic(self, password):
 659         import mnemonic
 660         s = pw_decode(self.seed, password)
 661         if self.seed_version == 4:
 662             return ' '.join(mnemonic.mn_encode(s))
 663         else:
 664             return s
 665
 666
 667
 668     def get_private_key(self, address, password):
 669         out = []
 670         if address in self.imported_keys.keys():
 671             out.append( pw_decode( self.imported_keys[address], password ) )
 672         else:
 673             account, sequence = self.get_address_index(address)
 674             if account == 0:
 675                 seed = self.get_seed(password)
 676                 pk = self.accounts[account].get_private_key(seed, sequence)
 677                 out.append(pk)
 678                 return out
 679
 680             # assert address == self.accounts[account].get_address(*sequence)
 681             rs = self.rebase_sequence( account, sequence)
 682             for root, public_sequence in rs:
 683
 684                 if root not in self.master_private_keys.keys(): continue
 685                 master_k = self.get_master_private_key(root, password)
 686                 master_c, _, _ = self.master_public_keys[root]
 687                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 688                 out.append(pk)
 689
 690         return out
 691
 692
 693     def add_keypairs_from_wallet(self, tx, keypairs, password):
 694         for txin in tx.inputs:
 695             address = txin['address']
 696             private_keys = self.get_private_key(address, password)
 697             for sec in private_keys:
 698                 pubkey = public_key_from_private_key(sec)
 699                 keypairs[ pubkey ] = sec
 700
 701
 702     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 703         for txin in tx.inputs:
 704             keyid = txin.get('KeyID')
 705             if keyid:
 706                 roots = []
 707                 for s in keyid.split('&'):
 708                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 709                     if not m: continue
 710                     c = m.group(1)
 711                     K = m.group(2)
 712                     sequence = m.group(3)
 713                     root = self.find_root_by_master_key(c,K)
 714                     if not root: continue
 715                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 716                     root = root + '%d'%sequence[0]
 717                     sequence = sequence[1:]
 718                     roots.append((root,sequence))
 719
 720                 account_id = " & ".join( map(lambda x:x[0], roots) )
 721                 account = self.accounts.get(account_id)
 722                 if not account: continue
 723                 addr = account.get_address(*sequence)
 724                 txin['address'] = addr # fixme: side effect
 725                 pk = self.get_private_key(addr, password)
 726                 for sec in pk:
 727                     pubkey = public_key_from_private_key(sec)
 728                     keypairs[pubkey] = sec
 729
 730
 731
 732     def signrawtransaction(self, tx, input_info, private_keys, password):
 733
 734         # check that the password is correct
 735         seed = self.get_seed(password)
 736
 737         # add input info
 738         tx.add_input_info(input_info)
 739
 740         # add redeem script for coins that are in the wallet
 741         # FIXME: add redeemPubkey too!
 742         unspent_coins = self.get_unspent_coins()
 743         for txin in tx.inputs:
 744             for item in unspent_coins:
 745                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 746                     print_error( "tx input is in unspent coins" )
 747                     txin['scriptPubKey'] = item['scriptPubKey']
 748                     account, sequence = self.get_address_index(item['address'])
 749                     if account != -1:
 750                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 751                         print_error("added redeemScript", txin['redeemScript'])
 752                     break
 753
 754
 755         # build a list of public/private keys
 756         keypairs = {}
 757
 758         # add private keys from parameter
 759         for sec in private_keys:
 760             pubkey = public_key_from_private_key(sec)
 761             keypairs[ pubkey ] = sec
 762
 763         # add private_keys from KeyID
 764         self.add_keypairs_from_KeyID(tx, keypairs, password)
 765
 766         # add private keys from wallet
 767         self.add_keypairs_from_wallet(tx, keypairs, password)
 768         self.sign_transaction(tx, keypairs)
 769
 770
 771     def sign_message(self, address, message, password):
 772         keys = self.get_private_key(address, password)
 773         assert len(keys) == 1
 774         sec = keys[0]
 775         key = regenerate_key(sec)
 776         compressed = is_compressed(sec)
 777         return key.sign_message(message, compressed, address)
 778
 779
 780     def change_gap_limit(self, value):
 781         if value >= self.gap_limit:
 782             self.gap_limit = value
 783             self.storage.put('gap_limit', self.gap_limit, True)
 784             #self.interface.poke('synchronizer')
 785             return True
 786
 787         elif value >= self.min_acceptable_gap():
 788             for key, account in self.accounts.items():
 789                 addresses = account[0]
 790                 k = self.num_unused_trailing_addresses(addresses)
 791                 n = len(addresses) - k + value
 792                 addresses = addresses[0:n]
 793                 self.accounts[key][0] = addresses
 794
 795             self.gap_limit = value
 796             self.storage.put('gap_limit', self.gap_limit, True)
 797             self.save_accounts()
 798             return True
 799         else:
 800             return False
 801
 802     def num_unused_trailing_addresses(self, addresses):
 803         k = 0
 804         for a in addresses[::-1]:
 805             if self.history.get(a):break
 806             k = k + 1
 807         return k
 808
 809     def min_acceptable_gap(self):
 810         # fixme: this assumes wallet is synchronized
 811         n = 0
 812         nmax = 0
 813
 814         for account in self.accounts.values():
 815             addresses = account.get_addresses(0)
 816             k = self.num_unused_trailing_addresses(addresses)
 817             for a in addresses[0:-k]:
 818                 if self.history.get(a):
 819                     n = 0
 820                 else:
 821                     n += 1
 822                     if n > nmax: nmax = n
 823         return nmax + 1
 824
 825
 826     def address_is_old(self, address):
 827         age = -1
 828         h = self.history.get(address, [])
 829         if h == ['*']:
 830             return True
 831         for tx_hash, tx_height in h:
 832             if tx_height == 0:
 833                 tx_age = 0
 834             else:
 835                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 836             if tx_age > age:
 837                 age = tx_age
 838         return age > 2
 839
 840
 841     def synchronize_sequence(self, account, for_change):
 842         limit = self.gap_limit_for_change if for_change else self.gap_limit
 843         new_addresses = []
 844         while True:
 845             addresses = account.get_addresses(for_change)
 846             if len(addresses) < limit:
 847                 address = account.create_new_address(for_change)
 848                 self.history[address] = []
 849                 new_addresses.append( address )
 850                 continue
 851
 852             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 853                 break
 854             else:
 855                 address = account.create_new_address(for_change)
 856                 self.history[address] = []
 857                 new_addresses.append( address )
 858
 859         return new_addresses
 860
 861
 862
 863     def create_pending_accounts(self):
 864         for account_type in ['1','2of2','2of3']:
 865             if not self.has_master_public_keys(account_type):
 866                 continue
 867             k, a = self.new_account_address(account_type)
 868             if self.address_is_old(a):
 869                 print_error( "creating account", a )
 870                 self.create_account(account_type)
 871                 self.next_addresses.pop(k)
 872
 873
 874     def synchronize_account(self, account):
 875         new = []
 876         new += self.synchronize_sequence(account, 0)
 877         new += self.synchronize_sequence(account, 1)
 878         return new
 879
 880
 881     def synchronize(self):
 882         if self.master_public_keys:
 883             self.create_pending_accounts()
 884         new = []
 885         for account in self.accounts.values():
 886             new += self.synchronize_account(account)
 887         if new:
 888             self.save_accounts()
 889             self.storage.put('addr_history', self.history, True)
 890         return new
 891
 892
 893     def is_found(self):
 894         return self.history.values() != [[]] * len(self.history)
 895
 896
 897     def add_contact(self, address, label=None):
 898         self.addressbook.append(address)
 899         self.storage.put('contacts', self.addressbook, True)
 900         if label:
 901             self.set_label(address, label)
 902
 903
 904     def delete_contact(self, addr):
 905         if addr in self.addressbook:
 906             self.addressbook.remove(addr)
 907             self.storage.put('addressbook', self.addressbook, True)
 908
 909
 910     def fill_addressbook(self):
 911         for tx_hash, tx in self.transactions.items():
 912             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 913             if is_send:
 914                 for addr, v in tx.outputs:
 915                     if not self.is_mine(addr) and addr not in self.addressbook:
 916                         self.addressbook.append(addr)
 917         # redo labels
 918         # self.update_tx_labels()
 919
 920     def get_num_tx(self, address):
 921         n = 0
 922         for tx in self.transactions.values():
 923             if address in map(lambda x:x[0], tx.outputs): n += 1
 924         return n
 925
 926
 927     def get_address_flags(self, addr):
 928         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 929         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 930         return flags
 931
 932
 933     def get_tx_value(self, tx, account=None):
 934         domain = self.get_account_addresses(account)
 935         return tx.get_value(domain, self.prevout_values)
 936
 937
 938     def update_tx_outputs(self, tx_hash):
 939         tx = self.transactions.get(tx_hash)
 940
 941         for i, (addr, value) in enumerate(tx.outputs):
 942             key = tx_hash+ ':%d'%i
 943             self.prevout_values[key] = value
 944
 945         for item in tx.inputs:
 946             if self.is_mine(item.get('address')):
 947                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 948                 self.spent_outputs.append(key)
 949
 950
 951     def get_addr_balance(self, address):
 952         assert self.is_mine(address)
 953         h = self.history.get(address,[])
 954         if h == ['*']: return 0,0
 955         c = u = 0
 956         received_coins = []   # list of coins received at address
 957
 958         for tx_hash, tx_height in h:
 959             tx = self.transactions.get(tx_hash)
 960             if not tx: continue
 961
 962             for i, (addr, value) in enumerate(tx.outputs):
 963                 if addr == address:
 964                     key = tx_hash + ':%d'%i
 965                     received_coins.append(key)
 966
 967         for tx_hash, tx_height in h:
 968             tx = self.transactions.get(tx_hash)
 969             if not tx: continue
 970             v = 0
 971
 972             for item in tx.inputs:
 973                 addr = item.get('address')
 974                 if addr == address:
 975                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 976                     value = self.prevout_values.get( key )
 977                     if key in received_coins:
 978                         v -= value
 979
 980             for i, (addr, value) in enumerate(tx.outputs):
 981                 key = tx_hash + ':%d'%i
 982                 if addr == address:
 983                     v += value
 984
 985             if tx_height:
 986                 c += v
 987             else:
 988                 u += v
 989         return c, u
 990
 991
 992     def get_account_name(self, k):
 993         if k == 0:
 994             if self.seed_version == 4:
 995                 name = 'Main account'
 996             else:
 997                 name = 'Old account'
 998         else:
 999             default = "Unnamed account"
1000             m = re.match("m/0'/(\d+)", k)
1001             if m:
1002                 num = m.group(1)
1003                 if num == '0':
1004                     default = "Main account"
1005                 else:
1006                     default = "Account %s"%num
1007
1008             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1009             if m:
1010                 num = m.group(1)
1011                 default = "2of2 account %s"%num
1012             name = self.labels.get(k, default)
1013
1014         return name
1015
1016     def get_account_names(self):
1017         accounts = {}
1018         for k, account in self.accounts.items():
1019             accounts[k] = self.get_account_name(k)
1020         if self.imported_keys:
1021             accounts[-1] = 'Imported keys'
1022         return accounts
1023
1024     def get_account_addresses(self, a, include_change=True):
1025         if a is None:
1026             o = self.addresses(True)
1027         elif a == -1:
1028             o = self.imported_keys.keys()
1029         else:
1030             ac = self.accounts[a]
1031             o = ac.get_addresses(0)
1032             if include_change: o += ac.get_addresses(1)
1033         return o
1034
1035     def get_imported_balance(self):
1036         return self.get_balance(self.imported_keys.keys())
1037
1038     def get_account_balance(self, account):
1039         return self.get_balance(self.get_account_addresses(account))
1040
1041     def get_frozen_balance(self):
1042         return self.get_balance(self.frozen_addresses)
1043
1044     def get_balance(self, domain=None):
1045         if domain is None: domain = self.addresses(True)
1046         cc = uu = 0
1047         for addr in domain:
1048             c, u = self.get_addr_balance(addr)
1049             cc += c
1050             uu += u
1051         return cc, uu
1052
1053
1054     def get_unspent_coins(self, domain=None):
1055         coins = []
1056         if domain is None: domain = self.addresses(True)
1057         for addr in domain:
1058             h = self.history.get(addr, [])
1059             if h == ['*']: continue
1060             for tx_hash, tx_height in h:
1061                 tx = self.transactions.get(tx_hash)
1062                 if tx is None: raise BaseException("Wallet not synchronized")
1063                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1064                 for output in tx.d.get('outputs'):
1065                     if output.get('address') != addr: continue
1066                     key = tx_hash + ":%d" % output.get('prevout_n')
1067                     if key in self.spent_outputs: continue
1068                     output['prevout_hash'] = tx_hash
1069                     output['height'] = tx_height
1070                     output['coinbase'] = is_coinbase
1071                     coins.append((tx_height, output))
1072
1073         # sort by age
1074         if coins:
1075             coins = sorted(coins)
1076             if coins[-1][0] != 0:
1077                 while coins[0][0] == 0:
1078                     coins = coins[1:] + [ coins[0] ]
1079         return [x[1] for x in coins]
1080
1081
1082
1083     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1084         domain = self.get_account_addresses(account) if account else None
1085         return self.choose_tx_inputs( amount, fixed_fee, domain )
1086
1087
1088     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1089         """ todo: minimize tx size """
1090         total = 0
1091         fee = self.fee if fixed_fee is None else fixed_fee
1092         if domain is None:
1093             domain = self.addresses(True)
1094
1095         for i in self.frozen_addresses:
1096             if i in domain: domain.remove(i)
1097
1098         prioritized = []
1099         for i in self.prioritized_addresses:
1100             if i in domain:
1101                 domain.remove(i)
1102                 prioritized.append(i)
1103
1104         coins = self.get_unspent_coins(domain)
1105         prioritized_coins = self.get_unspent_coins(prioritized)
1106
1107         inputs = []
1108         coins = prioritized_coins + coins
1109
1110         for item in coins:
1111             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1112                 continue
1113             addr = item.get('address')
1114             v = item.get('value')
1115             total += v
1116             inputs.append(item)
1117             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1118             if total >= amount + fee: break
1119         else:
1120             inputs = []
1121
1122         return inputs, total, fee
1123
1124
1125     def set_fee(self, fee):
1126         if self.fee != fee:
1127             self.fee = fee
1128             self.storage.put('fee_per_kb', self.fee, True)
1129
1130     def estimated_fee(self, inputs):
1131         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1132         fee = self.fee * int(round(estimated_size/1024.))
1133         if fee == 0: fee = self.fee
1134         return fee
1135
1136
1137     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1138         "add change to a transaction"
1139         change_amount = total - ( amount + fee )
1140         if change_amount != 0:
1141             if not change_addr:
1142
1143                 # send change to one of the accounts involved in the tx
1144                 address = inputs[0].get('address')
1145                 account, _ = self.get_address_index(address)
1146
1147                 if not self.use_change or account == -1:
1148                     change_addr = inputs[-1]['address']
1149                 else:
1150                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1151
1152             # Insert the change output at a random position in the outputs
1153             posn = random.randint(0, len(outputs))
1154             outputs[posn:posn] = [( change_addr,  change_amount)]
1155         return outputs
1156
1157
1158     def get_history(self, address):
1159         with self.lock:
1160             return self.history.get(address)
1161
1162
1163     def get_status(self, h):
1164         if not h: return None
1165         if h == ['*']: return '*'
1166         status = ''
1167         for tx_hash, height in h:
1168             status += tx_hash + ':%d:' % height
1169         return hashlib.sha256( status ).digest().encode('hex')
1170
1171
1172     def receive_tx_callback(self, tx_hash, tx, tx_height):
1173
1174         with self.transaction_lock:
1175             self.add_extra_addresses(tx)
1176             if not self.check_new_tx(tx_hash, tx):
1177                 # may happen due to pruning
1178                 print_error("received transaction that is no longer referenced in history", tx_hash)
1179                 return
1180             self.transactions[tx_hash] = tx
1181             self.network.interface.pending_transactions_for_notifications.append(tx)
1182             self.save_transactions()
1183             if self.verifier and tx_height>0:
1184                 self.verifier.add(tx_hash, tx_height)
1185             self.update_tx_outputs(tx_hash)
1186
1187
1188     def save_transactions(self):
1189         tx = {}
1190         for k,v in self.transactions.items():
1191             tx[k] = str(v)
1192         self.storage.put('transactions', tx, True)
1193
1194     def receive_history_callback(self, addr, hist):
1195
1196         if not self.check_new_history(addr, hist):
1197             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1198
1199         with self.lock:
1200             self.history[addr] = hist
1201             self.storage.put('addr_history', self.history, True)
1202
1203         if hist != ['*']:
1204             for tx_hash, tx_height in hist:
1205                 if tx_height>0:
1206                     # add it in case it was previously unconfirmed
1207                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1208
1209
1210     def get_tx_history(self, account=None):
1211         with self.transaction_lock:
1212             history = self.transactions.items()
1213             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1214             result = []
1215
1216             balance = 0
1217             for tx_hash, tx in history:
1218                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1219                 if v is not None: balance += v
1220
1221             c, u = self.get_account_balance(account)
1222
1223             if balance != c+u:
1224                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1225
1226             balance = c + u - balance
1227             for tx_hash, tx in history:
1228                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1229                 if not is_relevant:
1230                     continue
1231                 if value is not None:
1232                     balance += value
1233
1234                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1235                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1236
1237         return result
1238
1239
1240     def get_label(self, tx_hash):
1241         label = self.labels.get(tx_hash)
1242         is_default = (label == '') or (label is None)
1243         if is_default: label = self.get_default_label(tx_hash)
1244         return label, is_default
1245
1246
1247     def get_default_label(self, tx_hash):
1248         tx = self.transactions.get(tx_hash)
1249         default_label = ''
1250         if tx:
1251             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1252             if is_mine:
1253                 for o in tx.outputs:
1254                     o_addr, _ = o
1255                     if not self.is_mine(o_addr):
1256                         try:
1257                             default_label = self.labels[o_addr]
1258                         except KeyError:
1259                             default_label = o_addr
1260                         break
1261                 else:
1262                     default_label = '(internal)'
1263             else:
1264                 for o in tx.outputs:
1265                     o_addr, _ = o
1266                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1267                         break
1268                 else:
1269                     for o in tx.outputs:
1270                         o_addr, _ = o
1271                         if self.is_mine(o_addr):
1272                             break
1273                     else:
1274                         o_addr = None
1275
1276                 if o_addr:
1277                     dest_label = self.labels.get(o_addr)
1278                     try:
1279                         default_label = self.labels[o_addr]
1280                     except KeyError:
1281                         default_label = o_addr
1282
1283         return default_label
1284
1285
1286     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1287         for address, x in outputs:
1288             assert is_valid(address)
1289         amount = sum( map(lambda x:x[1], outputs) )
1290         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1291         if not inputs:
1292             raise ValueError("Not enough funds")
1293         self.add_input_info(inputs)
1294         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1295         return Transaction.from_io(inputs, outputs)
1296
1297
1298     def mktx_from_account(self, outputs, password, fee=None, account=None):
1299         domain = self.get_account_addresses(account) if account else None
1300         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1301
1302
1303     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1304         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1305         keypairs = {}
1306         self.add_keypairs_from_wallet(tx, keypairs, password)
1307         if keypairs:
1308             self.sign_transaction(tx, keypairs)
1309         return tx
1310
1311
1312     def add_input_info(self, inputs):
1313         for txin in inputs:
1314             address = txin['address']
1315             account, sequence = self.get_address_index(address)
1316             txin['KeyID'] = self.get_keyID(account, sequence)
1317             redeemScript = self.accounts[account].redeem_script(sequence)
1318             if redeemScript:
1319                 txin['redeemScript'] = redeemScript
1320             else:
1321                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1322
1323
1324     def sign_transaction(self, tx, keypairs):
1325         tx.sign(keypairs)
1326         run_hook('sign_transaction', tx)
1327
1328
1329     def sendtx(self, tx):
1330         # synchronous
1331         h = self.send_tx(tx)
1332         self.tx_event.wait()
1333         return self.receive_tx(h)
1334
1335     def send_tx(self, tx):
1336         # asynchronous
1337         self.tx_event.clear()
1338         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1339         return tx.hash()
1340
1341     def on_broadcast(self, i, r):
1342         self.tx_result = r.get('result')
1343         self.tx_event.set()
1344
1345     def receive_tx(self,tx_hash):
1346         out = self.tx_result
1347         if out != tx_hash:
1348             return False, "error: " + out
1349         return True, out
1350
1351
1352
1353     def update_password(self, old_password, new_password):
1354         if new_password == '': new_password = None
1355         # this will throw an exception if unicode cannot be converted
1356         decoded = pw_decode(self.seed, old_password)
1357         self.seed = pw_encode( decoded, new_password)
1358         self.storage.put('seed', self.seed, True)
1359         self.use_encryption = (new_password != None)
1360         self.storage.put('use_encryption', self.use_encryption,True)
1361         for k in self.imported_keys.keys():
1362             a = self.imported_keys[k]
1363             b = pw_decode(a, old_password)
1364             c = pw_encode(b, new_password)
1365             self.imported_keys[k] = c
1366         self.storage.put('imported_keys', self.imported_keys, True)
1367
1368         for k, v in self.master_private_keys.items():
1369             b = pw_decode(v, old_password)
1370             c = pw_encode(b, new_password)
1371             self.master_private_keys[k] = c
1372         self.storage.put('master_private_keys', self.master_private_keys, True)
1373
1374
1375     def freeze(self,addr):
1376         if self.is_mine(addr) and addr not in self.frozen_addresses:
1377             self.unprioritize(addr)
1378             self.frozen_addresses.append(addr)
1379             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1380             return True
1381         else:
1382             return False
1383
1384     def unfreeze(self,addr):
1385         if self.is_mine(addr) and addr in self.frozen_addresses:
1386             self.frozen_addresses.remove(addr)
1387             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1388             return True
1389         else:
1390             return False
1391
1392     def prioritize(self,addr):
1393         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1394             self.unfreeze(addr)
1395             self.prioritized_addresses.append(addr)
1396             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1397             return True
1398         else:
1399             return False
1400
1401     def unprioritize(self,addr):
1402         if self.is_mine(addr) and addr in self.prioritized_addresses:
1403             self.prioritized_addresses.remove(addr)
1404             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1405             return True
1406         else:
1407             return False
1408
1409
1410     def set_verifier(self, verifier):
1411         self.verifier = verifier
1412
1413         # review transactions that are in the history
1414         for addr, hist in self.history.items():
1415             if hist == ['*']: continue
1416             for tx_hash, tx_height in hist:
1417                 if tx_height>0:
1418                     # add it in case it was previously unconfirmed
1419                     self.verifier.add(tx_hash, tx_height)
1420
1421
1422         # if we are on a pruning server, remove unverified transactions
1423         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1424         for tx_hash in self.transactions.keys():
1425             if tx_hash not in vr:
1426                 self.transactions.pop(tx_hash)
1427
1428
1429
1430     def check_new_history(self, addr, hist):
1431
1432         # check that all tx in hist are relevant
1433         if hist != ['*']:
1434             for tx_hash, height in hist:
1435                 tx = self.transactions.get(tx_hash)
1436                 if not tx: continue
1437                 if not tx.has_address(addr):
1438                     return False
1439
1440         # check that we are not "orphaning" a transaction
1441         old_hist = self.history.get(addr,[])
1442         if old_hist == ['*']: return True
1443
1444         for tx_hash, height in old_hist:
1445             if tx_hash in map(lambda x:x[0], hist): continue
1446             found = False
1447             for _addr, _hist in self.history.items():
1448                 if _addr == addr: continue
1449                 if _hist == ['*']: continue
1450                 _tx_hist = map(lambda x:x[0], _hist)
1451                 if tx_hash in _tx_hist:
1452                     found = True
1453                     break
1454
1455             if not found:
1456                 tx = self.transactions.get(tx_hash)
1457                 # tx might not be there
1458                 if not tx: continue
1459
1460                 # already verified?
1461                 if self.verifier.get_height(tx_hash):
1462                     continue
1463                 # unconfirmed tx
1464                 print_error("new history is orphaning transaction:", tx_hash)
1465                 # check that all outputs are not mine, request histories
1466                 ext_requests = []
1467                 for _addr, _v in tx.outputs:
1468                     # assert not self.is_mine(_addr)
1469                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1470
1471                 ext_h = self.network.synchronous_get(ext_requests)
1472                 print_error("sync:", ext_requests, ext_h)
1473                 height = None
1474                 for h in ext_h:
1475                     if h == ['*']: continue
1476                     for item in h:
1477                         if item.get('tx_hash') == tx_hash:
1478                             height = item.get('height')
1479                 if height:
1480                     print_error("found height for", tx_hash, height)
1481                     self.verifier.add(tx_hash, height)
1482                 else:
1483                     print_error("removing orphaned tx from history", tx_hash)
1484                     self.transactions.pop(tx_hash)
1485
1486         return True
1487
1488
1489
1490     def check_new_tx(self, tx_hash, tx):
1491         # 1 check that tx is referenced in addr_history.
1492         addresses = []
1493         for addr, hist in self.history.items():
1494             if hist == ['*']:continue
1495             for txh, height in hist:
1496                 if txh == tx_hash:
1497                     addresses.append(addr)
1498
1499         if not addresses:
1500             return False
1501
1502         # 2 check that referencing addresses are in the tx
1503         for addr in addresses:
1504             if not tx.has_address(addr):
1505                 return False
1506
1507         return True
1508
1509
1510     def start_threads(self, network):
1511         from verifier import TxVerifier
1512         self.network = network
1513         self.verifier = TxVerifier(self.network, self.storage)
1514         self.verifier.start()
1515         self.set_verifier(self.verifier)
1516         self.synchronizer = WalletSynchronizer(self, network)
1517         self.synchronizer.start()
1518
1519     def stop_threads(self):
1520         self.verifier.stop()
1521         self.synchronizer.stop()
1522
1523
1524
1525     def restore(self, callback):
1526         from i18n import _
1527         def wait_for_wallet():
1528             self.set_up_to_date(False)
1529             while not self.is_up_to_date():
1530                 msg = "%s\n%s %d\n%s %.1f"%(
1531                     _("Please wait..."),
1532                     _("Addresses generated:"),
1533                     len(self.addresses(True)),_("Kilobytes received:"),
1534                     self.network.interface.bytes_received/1024.)
1535
1536                 apply(callback, (msg,))
1537                 time.sleep(0.1)
1538
1539         def wait_for_network():
1540             while not self.network.interface.is_connected:
1541                 msg = "%s \n" % (_("Connecting..."))
1542                 apply(callback, (msg,))
1543                 time.sleep(0.1)
1544
1545         # wait until we are connected, because the user might have selected another server
1546         wait_for_network()
1547
1548
1549         if self.seed_version == 4:
1550             self.create_old_account()
1551         else:
1552             self.create_accounts()
1553         wait_for_wallet()
1554
1555
1556
1557
1558 class WalletSynchronizer(threading.Thread):
1559
1560
1561     def __init__(self, wallet, network):
1562         threading.Thread.__init__(self)
1563         self.daemon = True
1564         self.wallet = wallet
1565         self.network = network
1566         self.was_updated = True
1567         self.running = False
1568         self.lock = threading.Lock()
1569         self.queue = Queue.Queue()
1570
1571     def stop(self):
1572         with self.lock: self.running = False
1573
1574     def is_running(self):
1575         with self.lock: return self.running
1576
1577
1578     def subscribe_to_addresses(self, addresses):
1579         messages = []
1580         for addr in addresses:
1581             messages.append(('blockchain.address.subscribe', [addr]))
1582         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1583
1584
1585     def run(self):
1586         with self.lock:
1587             self.running = True
1588
1589         while self.is_running():
1590
1591             if not self.network.is_connected():
1592                 print_error("synchronizer: waiting for interface")
1593                 self.network.wait_until_connected()
1594
1595             self.run_interface(self.network.interface)
1596
1597
1598     def run_interface(self, interface):
1599
1600         print_error("synchronizer: connected to", interface.server)
1601
1602         requested_tx = []
1603         missing_tx = []
1604         requested_histories = {}
1605
1606         # request any missing transactions
1607         for history in self.wallet.history.values():
1608             if history == ['*']: continue
1609             for tx_hash, tx_height in history:
1610                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1611                     missing_tx.append( (tx_hash, tx_height) )
1612
1613         if missing_tx:
1614             print_error("missing tx", missing_tx)
1615
1616         # subscriptions
1617         self.subscribe_to_addresses(self.wallet.addresses(True))
1618
1619         while self.is_running():
1620             # 1. create new addresses
1621             new_addresses = self.wallet.synchronize()
1622
1623             # request missing addresses
1624             if new_addresses:
1625                 self.subscribe_to_addresses(new_addresses)
1626
1627             # request missing transactions
1628             for tx_hash, tx_height in missing_tx:
1629                 if (tx_hash, tx_height) not in requested_tx:
1630                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1631                     requested_tx.append( (tx_hash, tx_height) )
1632             missing_tx = []
1633
1634             # detect if situation has changed
1635             if interface.is_up_to_date() and self.queue.empty():
1636                 if not self.wallet.is_up_to_date():
1637                     self.wallet.set_up_to_date(True)
1638                     self.was_updated = True
1639             else:
1640                 if self.wallet.is_up_to_date():
1641                     self.wallet.set_up_to_date(False)
1642                     self.was_updated = True
1643
1644             if self.was_updated:
1645                 self.wallet.network.trigger_callback('updated')
1646                 self.was_updated = False
1647
1648             # 2. get a response
1649             try:
1650                 r = self.queue.get(block=True, timeout=1)
1651             except Queue.Empty:
1652                 continue
1653
1654             if interface != self.network.interface:
1655                 break
1656
1657             if not r:
1658                 continue
1659
1660             # 3. handle response
1661             method = r['method']
1662             params = r['params']
1663             result = r.get('result')
1664             error = r.get('error')
1665             if error:
1666                 print "error", r
1667                 continue
1668
1669             if method == 'blockchain.address.subscribe':
1670                 addr = params[0]
1671                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1672                     if requested_histories.get(addr) is None:
1673                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1674                         requested_histories[addr] = result
1675
1676             elif method == 'blockchain.address.get_history':
1677                 addr = params[0]
1678                 print_error("receiving history", addr, result)
1679                 if result == ['*']:
1680                     assert requested_histories.pop(addr) == '*'
1681                     self.wallet.receive_history_callback(addr, result)
1682                 else:
1683                     hist = []
1684                     # check that txids are unique
1685                     txids = []
1686                     for item in result:
1687                         tx_hash = item['tx_hash']
1688                         if tx_hash not in txids:
1689                             txids.append(tx_hash)
1690                             hist.append( (tx_hash, item['height']) )
1691
1692                     if len(hist) != len(result):
1693                         raise BaseException("error: server sent history with non-unique txid", result)
1694
1695                     # check that the status corresponds to what was announced
1696                     rs = requested_histories.pop(addr)
1697                     if self.wallet.get_status(hist) != rs:
1698                         raise BaseException("error: status mismatch: %s"%addr)
1699
1700                     # store received history
1701                     self.wallet.receive_history_callback(addr, hist)
1702
1703                     # request transactions that we don't have
1704                     for tx_hash, tx_height in hist:
1705                         if self.wallet.transactions.get(tx_hash) is None:
1706                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1707                                 missing_tx.append( (tx_hash, tx_height) )
1708
1709             elif method == 'blockchain.transaction.get':
1710                 tx_hash = params[0]
1711                 tx_height = params[1]
1712                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1713                 tx = Transaction(result)
1714                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1715                 self.was_updated = True
1716                 requested_tx.remove( (tx_hash, tx_height) )
1717                 print_error("received tx:", tx_hash, len(tx.raw))
1718
1719             else:
1720                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1721
1722             if self.was_updated and not requested_tx:
1723                 self.wallet.network.trigger_callback('updated')
1724                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1725
1726                 self.was_updated = False