lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40
  41 # AES encryption
  42 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  43 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  44
  45 def pw_encode(s, password):
  46     if password:
  47         secret = Hash(password)
  48         return EncodeAES(secret, s)
  49     else:
  50         return s
  51
  52 def pw_decode(s, password):
  53     if password is not None:
  54         secret = Hash(password)
  55         try:
  56             d = DecodeAES(secret, s)
  57         except:
  58             raise BaseException('Invalid password')
  59         return d
  60     else:
  61         return s
  62
  63
  64
  65
  66
  67 from version import *
  68
  69
  70 class WalletStorage:
  71
  72     def __init__(self, config):
  73         self.lock = threading.Lock()
  74         self.data = {}
  75         self.file_exists = False
  76         self.path = self.init_path(config)
  77         print_error( "wallet path", self.path )
  78         if self.path:
  79             self.read(self.path)
  80
  81
  82     def init_path(self, config):
  83         """Set the path of the wallet."""
  84
  85         # command line -w option
  86         path = config.get('wallet_path')
  87         if path:
  88             return path
  89
  90         # path in config file
  91         path = config.get('default_wallet_path')
  92         if path:
  93             return path
  94
  95         # default path
  96         dirpath = os.path.join(config.path, "wallets")
  97         if not os.path.exists(dirpath):
  98             os.mkdir(dirpath)
  99
 100         new_path = os.path.join(config.path, "wallets", "default_wallet")
 101
 102         # default path in pre 1.9 versions
 103         old_path = os.path.join(config.path, "electrum.dat")
 104         if os.path.exists(old_path) and not os.path.exists(new_path):
 105             os.rename(old_path, new_path)
 106
 107         return new_path
 108
 109
 110     def read(self, path):
 111         """Read the contents of the wallet file."""
 112         try:
 113             with open(self.path, "r") as f:
 114                 data = f.read()
 115         except IOError:
 116             return
 117         try:
 118             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 119         except:
 120             raise IOError("Cannot read wallet file.")
 121
 122         self.data = d
 123         self.file_exists = True
 124
 125
 126     def get(self, key, default=None):
 127         return self.data.get(key, default)
 128
 129     def put(self, key, value, save = True):
 130
 131         with self.lock:
 132             if value is not None:
 133                 self.data[key] = value
 134             else:
 135                 self.data.pop[key]
 136             if save:
 137                 self.write()
 138
 139     def write(self):
 140         s = repr(self.data)
 141         f = open(self.path,"w")
 142         f.write( s )
 143         f.close()
 144         if 'ANDROID_DATA' not in os.environ:
 145             import stat
 146             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 147
 148
 149 class Wallet:
 150
 151     def __init__(self, storage):
 152
 153         self.storage = storage
 154         self.electrum_version = ELECTRUM_VERSION
 155         self.gap_limit_for_change = 3 # constant
 156
 157         # saved fields
 158         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 159
 160         self.gap_limit             = storage.get('gap_limit', 5)
 161         self.use_change            = storage.get('use_change',True)
 162         self.use_encryption        = storage.get('use_encryption', False)
 163         self.seed                  = storage.get('seed', '')               # encrypted
 164         self.labels                = storage.get('labels', {})
 165         self.frozen_addresses      = storage.get('frozen_addresses',[])
 166         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 167         self.addressbook           = storage.get('contacts', [])
 168
 169         self.imported_keys         = storage.get('imported_keys',{})
 170         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 171
 172         self.fee                   = int(storage.get('fee_per_kb',20000))
 173
 174         self.master_public_keys = storage.get('master_public_keys',{})
 175         self.master_private_keys = storage.get('master_private_keys', {})
 176
 177         self.next_addresses = storage.get('next_addresses',{})
 178
 179         if self.seed_version not in [4, 6]:
 180             msg = "This wallet seed is not supported."
 181             if self.seed_version in [5]: msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 182             raise ValueError(msg)
 183
 184         self.load_accounts()
 185
 186         self.transactions = {}
 187         tx_list = self.storage.get('transactions',{})
 188         for k,v in tx_list.items():
 189             try:
 190                 tx = Transaction(v)
 191             except:
 192                 print_msg("Warning: Cannot deserialize transactions. skipping")
 193                 continue
 194
 195             self.add_extra_addresses(tx)
 196             self.transactions[k] = tx
 197
 198         for h,tx in self.transactions.items():
 199             if not self.check_new_tx(h, tx):
 200                 print_error("removing unreferenced tx", h)
 201                 self.transactions.pop(h)
 202
 203
 204         # not saved
 205         self.prevout_values = {}     # my own transaction outputs
 206         self.spent_outputs = []
 207
 208         # spv
 209         self.verifier = None
 210
 211         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 212         # interface.is_up_to_date() returns true when all requests have been answered and processed
 213         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 214
 215         self.up_to_date = False
 216         self.lock = threading.Lock()
 217         self.transaction_lock = threading.Lock()
 218         self.tx_event = threading.Event()
 219
 220         for tx_hash, tx in self.transactions.items():
 221             self.update_tx_outputs(tx_hash)
 222
 223
 224     def add_extra_addresses(self, tx):
 225         h = tx.hash()
 226         # find the address corresponding to pay-to-pubkey inputs
 227         tx.add_extra_addresses(self.transactions)
 228         for o in tx.d.get('outputs'):
 229             if o.get('is_pubkey'):
 230                 for tx2 in self.transactions.values():
 231                     tx2.add_extra_addresses({h:tx})
 232
 233
 234
 235
 236     def set_up_to_date(self,b):
 237         with self.lock: self.up_to_date = b
 238
 239
 240     def is_up_to_date(self):
 241         with self.lock: return self.up_to_date
 242
 243
 244     def update(self):
 245         self.up_to_date = False
 246         while not self.is_up_to_date():
 247             time.sleep(0.1)
 248
 249
 250     def import_key(self, sec, password):
 251         # check password
 252         seed = self.get_seed(password)
 253         try:
 254             address = address_from_private_key(sec)
 255         except:
 256             raise BaseException('Invalid private key')
 257
 258         if self.is_mine(address):
 259             raise BaseException('Address already in wallet')
 260
 261         # store the originally requested keypair into the imported keys table
 262         self.imported_keys[address] = pw_encode(sec, password )
 263         self.storage.put('imported_keys', self.imported_keys, True)
 264         if self.synchronizer:
 265             self.synchronizer.subscribe_to_addresses([address])
 266         return address
 267
 268     def delete_imported_key(self, addr):
 269         if addr in self.imported_keys:
 270             self.imported_keys.pop(addr)
 271             self.storage.put('imported_keys', self.imported_keys, True)
 272
 273
 274     def make_seed(self):
 275         import mnemonic, ecdsa
 276         entropy = ecdsa.util.randrange( pow(2,160) )
 277         nonce = 0
 278         while True:
 279             ss = "%040x"%(entropy+nonce)
 280             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 281             # we keep only 13 words, that's approximately 139 bits of entropy
 282             words = mnemonic.mn_encode(s)[0:13]
 283             seed = ' '.join(words)
 284             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 285                 break  # this removes 12 bits of entropy
 286             nonce += 1
 287
 288         return seed
 289
 290
 291     def init_seed(self, seed):
 292         if self.seed:
 293             raise BaseException("a seed exists")
 294
 295         if not seed:
 296             self.seed = random_seed(128)
 297             self.seed_version = 4
 298             return
 299
 300         #if not seed:
 301         #    self.seed = self.make_seed()
 302         #    self.seed_version = SEED_VERSION
 303         #    return
 304
 305         # find out what kind of wallet we are
 306         try:
 307             seed.decode('hex')
 308             self.seed_version = 4
 309             return
 310         except:
 311             pass
 312
 313         words = seed.split()
 314         try:
 315             mnemonic.mn_decode(words)
 316             uses_electrum_words = True
 317         except:
 318             uses_electrum_words = False
 319
 320         if uses_electrum_words and len(words) != 13:
 321             self.seed_version = 4
 322             self.seed = mnemonic.mn_encode(seed)
 323         else:
 324             assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 325             self.seed_version = SEED_VERSION
 326             self.seed = seed
 327
 328
 329     def save_seed(self):
 330         self.storage.put('seed', self.seed, True)
 331         self.storage.put('seed_version', self.seed_version, True)
 332
 333     def create_watching_only_wallet(self, c0, K0):
 334         cK0 = ""
 335         self.master_public_keys = {
 336             "m/0'/": (c0, K0, cK0),
 337             }
 338         self.storage.put('master_public_keys', self.master_public_keys, True)
 339         self.create_account('1','Main account')
 340
 341
 342     def create_accounts(self):
 343         if self.seed_version == 4:
 344             self.create_old_account()
 345         else:
 346             # create default account
 347             self.create_master_keys('1')
 348             self.create_account('1','Main account')
 349
 350
 351     def create_master_keys(self, account_type):
 352         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 353         if account_type == '1':
 354             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 355             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 356             self.master_private_keys["m/0'/"] = k0
 357         elif account_type == '2of2':
 358             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 359             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 360             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 361             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 362             self.master_private_keys["m/1'/"] = k1
 363             self.master_private_keys["m/2'/"] = k2
 364         elif account_type == '2of3':
 365             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 366             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 367             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 368             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 369             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 370             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 371             self.master_private_keys["m/3'/"] = k3
 372             self.master_private_keys["m/4'/"] = k4
 373             self.master_private_keys["m/5'/"] = k5
 374
 375         self.storage.put('master_public_keys', self.master_public_keys, True)
 376         self.storage.put('master_private_keys', self.master_private_keys, True)
 377
 378     def has_master_public_keys(self, account_type):
 379         if account_type == '1':
 380             return "m/0'/" in self.master_public_keys
 381         elif account_type == '2of2':
 382             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 383         elif account_type == '2of3':
 384             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 385
 386     def find_root_by_master_key(self, c, K):
 387         for key, v in self.master_public_keys.items():
 388             if key == "m/":continue
 389             cc, KK, _ = v
 390             if (c == cc) and (K == KK):
 391                 return key
 392
 393     def deseed_root(self, seed, password):
 394         # for safety, we ask the user to enter their seed
 395         assert seed == self.get_seed(password)
 396         self.seed = ''
 397         self.storage.put('seed', '', True)
 398
 399
 400     def deseed_branch(self, k):
 401         # check that parent has no seed
 402         assert self.seed == ''
 403         self.master_private_keys.pop(k)
 404         self.storage.put('master_private_keys', self.master_private_keys, True)
 405
 406     def is_watching_only(self):
 407         return (self.seed == '') and (self.master_private_keys == {})
 408
 409
 410
 411     def account_id(self, account_type, i):
 412         if account_type == '1':
 413             return "m/0'/%d"%i
 414         elif account_type == '2of2':
 415             return "m/1'/%d & m/2'/%d"%(i,i)
 416         elif account_type == '2of3':
 417             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 418         else:
 419             raise BaseException('unknown account type')
 420
 421
 422     def num_accounts(self, account_type):
 423         keys = self.accounts.keys()
 424         i = 0
 425         while True:
 426             account_id = self.account_id(account_type, i)
 427             if account_id not in keys: break
 428             i += 1
 429         return i
 430
 431
 432     def new_account_address(self, account_type = '1'):
 433         i = self.num_accounts(account_type)
 434         k = self.account_id(account_type,i)
 435
 436         addr = self.next_addresses.get(k)
 437         if not addr:
 438             account_id, account = self.next_account(account_type)
 439             addr = account.first_address()
 440             self.next_addresses[k] = addr
 441             self.storage.put('next_addresses',self.next_addresses)
 442
 443         return k, addr
 444
 445
 446     def next_account(self, account_type = '1'):
 447
 448         i = self.num_accounts(account_type)
 449         account_id = self.account_id(account_type,i)
 450
 451         if account_type is '1':
 452             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 453             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 454             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 455
 456         elif account_type == '2of2':
 457             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 458             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 459             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 460             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 461             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 462
 463         elif account_type == '2of3':
 464             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 465             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 466             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 467             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 468             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 469             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 470             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 471
 472         return account_id, account
 473
 474
 475     def set_label(self, name, text = None):
 476         changed = False
 477         old_text = self.labels.get(name)
 478         if text:
 479             if old_text != text:
 480                 self.labels[name] = text
 481                 changed = True
 482         else:
 483             if old_text:
 484                 self.labels.pop(name)
 485                 changed = True
 486
 487         if changed:
 488             self.storage.put('labels', self.labels, True)
 489
 490         run_hook('set_label', name, text, changed)
 491         return changed
 492
 493
 494
 495     def create_account(self, account_type = '1', name = None):
 496         k, account = self.next_account(account_type)
 497         if k in self.pending_accounts:
 498             self.pending_accounts.pop(k)
 499             self.storage.put('pending_accounts', self.pending_accounts)
 500
 501         self.accounts[k] = account
 502         self.save_accounts()
 503         if name:
 504             self.set_label(k, name)
 505
 506
 507     def create_old_account(self):
 508         mpk = OldAccount.mpk_from_seed(self.seed)
 509         self.storage.put('master_public_key', mpk, True)
 510         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 511         self.save_accounts()
 512
 513
 514     def save_accounts(self):
 515         d = {}
 516         for k, v in self.accounts.items():
 517             d[k] = v.dump()
 518         self.storage.put('accounts', d, True)
 519
 520
 521
 522     def load_accounts(self):
 523         d = self.storage.get('accounts', {})
 524         self.accounts = {}
 525         for k, v in d.items():
 526             if k == 0:
 527                 v['mpk'] = self.storage.get('master_public_key')
 528                 self.accounts[k] = OldAccount(v)
 529             elif '&' in k:
 530                 self.accounts[k] = BIP32_Account_2of2(v)
 531             else:
 532                 self.accounts[k] = BIP32_Account(v)
 533
 534         self.pending_accounts = self.storage.get('pending_accounts',{})
 535
 536
 537     def delete_pending_account(self, k):
 538         self.pending_accounts.pop(k)
 539         self.storage.put('pending_accounts', self.pending_accounts)
 540
 541     def account_is_pending(self, k):
 542         return k in self.pending_accounts
 543
 544     def create_pending_account(self, acct_type, name):
 545         k, addr = self.new_account_address(acct_type)
 546         self.set_label(k, name)
 547         self.pending_accounts[k] = addr
 548         self.storage.put('pending_accounts', self.pending_accounts)
 549
 550     def get_pending_accounts(self):
 551         return self.pending_accounts.items()
 552
 553
 554     def addresses(self, include_change = True, _next=True):
 555         o = self.get_account_addresses(-1, include_change)
 556         for a in self.accounts.keys():
 557             o += self.get_account_addresses(a, include_change)
 558
 559         if _next:
 560             for addr in self.next_addresses.values():
 561                 if addr not in o:
 562                     o += [addr]
 563         return o
 564
 565
 566     def is_mine(self, address):
 567         return address in self.addresses(True)
 568
 569
 570     def is_change(self, address):
 571         if not self.is_mine(address): return False
 572         if address in self.imported_keys.keys(): return False
 573         acct, s = self.get_address_index(address)
 574         if s is None: return False
 575         return s[0] == 1
 576
 577     def get_master_public_key(self):
 578         if self.seed_version == 4:
 579             return self.storage.get("master_public_key")
 580         else:
 581             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 582             return repr((c, K))
 583
 584     def get_master_private_key(self, account, password):
 585         master_k = pw_decode( self.master_private_keys[account], password)
 586         master_c, master_K, master_Kc = self.master_public_keys[account]
 587         try:
 588             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 589             assert K.encode('hex') == master_K
 590         except:
 591             raise BaseException("Invalid password")
 592         return master_k
 593
 594
 595     def get_address_index(self, address):
 596         if address in self.imported_keys.keys():
 597             return -1, None
 598
 599         for account in self.accounts.keys():
 600             for for_change in [0,1]:
 601                 addresses = self.accounts[account].get_addresses(for_change)
 602                 for addr in addresses:
 603                     if address == addr:
 604                         return account, (for_change, addresses.index(addr))
 605
 606         for k,v in self.next_addresses.items():
 607             if v == address:
 608                 return k, (0,0)
 609
 610         raise BaseException("Address not found", address)
 611
 612
 613     def get_roots(self, account):
 614         roots = []
 615         for a in account.split('&'):
 616             s = a.strip()
 617             m = re.match("(m/\d+'/)(\d+)", s)
 618             roots.append( m.group(1) )
 619         return roots
 620
 621     def is_seeded(self, account):
 622         if type(account) is int:
 623             return self.seed is not None
 624
 625         for root in self.get_roots(account):
 626             if root not in self.master_private_keys.keys():
 627                 return False
 628         return True
 629
 630     def rebase_sequence(self, account, sequence):
 631         c, i = sequence
 632         dd = []
 633         for a in account.split('&'):
 634             s = a.strip()
 635             m = re.match("(m/\d+'/)(\d+)", s)
 636             root = m.group(1)
 637             num = int(m.group(2))
 638             dd.append( (root, [num,c,i] ) )
 639         return dd
 640
 641
 642     def get_keyID(self, account, sequence):
 643         if account == 0:
 644             return 'old'
 645
 646         rs = self.rebase_sequence(account, sequence)
 647         dd = []
 648         for root, public_sequence in rs:
 649             c, K, _ = self.master_public_keys[root]
 650             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 651             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 652         return '&'.join(dd)
 653
 654
 655
 656     def get_seed(self, password):
 657         s = pw_decode(self.seed, password)
 658         if self.seed_version == 4:
 659             seed = s
 660         else:
 661             seed = mnemonic_hash(s)
 662         #todo:  #self.sequences[0].check_seed(seed)
 663         return seed
 664
 665
 666     def get_mnemonic(self, password):
 667         import mnemonic
 668         s = pw_decode(self.seed, password)
 669         if self.seed_version == 4:
 670             return ' '.join(mnemonic.mn_encode(s))
 671         else:
 672             return s
 673
 674
 675
 676     def get_private_key(self, address, password):
 677         out = []
 678         if address in self.imported_keys.keys():
 679             out.append( pw_decode( self.imported_keys[address], password ) )
 680         else:
 681             account, sequence = self.get_address_index(address)
 682             if account == 0:
 683                 seed = self.get_seed(password)
 684                 pk = self.accounts[account].get_private_key(seed, sequence)
 685                 out.append(pk)
 686                 return out
 687
 688             # assert address == self.accounts[account].get_address(*sequence)
 689             rs = self.rebase_sequence( account, sequence)
 690             for root, public_sequence in rs:
 691
 692                 if root not in self.master_private_keys.keys(): continue
 693                 master_k = self.get_master_private_key(root, password)
 694                 master_c, _, _ = self.master_public_keys[root]
 695                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 696                 out.append(pk)
 697
 698         return out
 699
 700
 701     def add_keypairs_from_wallet(self, tx, keypairs, password):
 702         for txin in tx.inputs:
 703             address = txin['address']
 704             private_keys = self.get_private_key(address, password)
 705             for sec in private_keys:
 706                 pubkey = public_key_from_private_key(sec)
 707                 keypairs[ pubkey ] = sec
 708
 709
 710     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 711         for txin in tx.inputs:
 712             keyid = txin.get('KeyID')
 713             if keyid:
 714                 roots = []
 715                 for s in keyid.split('&'):
 716                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 717                     if not m: continue
 718                     c = m.group(1)
 719                     K = m.group(2)
 720                     sequence = m.group(3)
 721                     root = self.find_root_by_master_key(c,K)
 722                     if not root: continue
 723                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 724                     root = root + '%d'%sequence[0]
 725                     sequence = sequence[1:]
 726                     roots.append((root,sequence))
 727
 728                 account_id = " & ".join( map(lambda x:x[0], roots) )
 729                 account = self.accounts.get(account_id)
 730                 if not account: continue
 731                 addr = account.get_address(*sequence)
 732                 txin['address'] = addr # fixme: side effect
 733                 pk = self.get_private_key(addr, password)
 734                 for sec in pk:
 735                     pubkey = public_key_from_private_key(sec)
 736                     keypairs[pubkey] = sec
 737
 738
 739
 740     def signrawtransaction(self, tx, input_info, private_keys, password):
 741
 742         # check that the password is correct
 743         seed = self.get_seed(password)
 744
 745         # add input info
 746         tx.add_input_info(input_info)
 747
 748         # add redeem script for coins that are in the wallet
 749         # FIXME: add redeemPubkey too!
 750         unspent_coins = self.get_unspent_coins()
 751         for txin in tx.inputs:
 752             for item in unspent_coins:
 753                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 754                     print_error( "tx input is in unspent coins" )
 755                     txin['scriptPubKey'] = item['scriptPubKey']
 756                     account, sequence = self.get_address_index(item['address'])
 757                     if account != -1:
 758                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 759                         print_error("added redeemScript", txin['redeemScript'])
 760                     break
 761
 762
 763         # build a list of public/private keys
 764         keypairs = {}
 765
 766         # add private keys from parameter
 767         for sec in private_keys:
 768             pubkey = public_key_from_private_key(sec)
 769             keypairs[ pubkey ] = sec
 770
 771         # add private_keys from KeyID
 772         self.add_keypairs_from_KeyID(tx, keypairs, password)
 773
 774         # add private keys from wallet
 775         self.add_keypairs_from_wallet(tx, keypairs, password)
 776         self.sign_transaction(tx, keypairs)
 777
 778
 779     def sign_message(self, address, message, password):
 780         keys = self.get_private_key(address, password)
 781         assert len(keys) == 1
 782         sec = keys[0]
 783         key = regenerate_key(sec)
 784         compressed = is_compressed(sec)
 785         return key.sign_message(message, compressed, address)
 786
 787
 788     def change_gap_limit(self, value):
 789         if value >= self.gap_limit:
 790             self.gap_limit = value
 791             self.storage.put('gap_limit', self.gap_limit, True)
 792             #self.interface.poke('synchronizer')
 793             return True
 794
 795         elif value >= self.min_acceptable_gap():
 796             for key, account in self.accounts.items():
 797                 addresses = account[0]
 798                 k = self.num_unused_trailing_addresses(addresses)
 799                 n = len(addresses) - k + value
 800                 addresses = addresses[0:n]
 801                 self.accounts[key][0] = addresses
 802
 803             self.gap_limit = value
 804             self.storage.put('gap_limit', self.gap_limit, True)
 805             self.save_accounts()
 806             return True
 807         else:
 808             return False
 809
 810     def num_unused_trailing_addresses(self, addresses):
 811         k = 0
 812         for a in addresses[::-1]:
 813             if self.history.get(a):break
 814             k = k + 1
 815         return k
 816
 817     def min_acceptable_gap(self):
 818         # fixme: this assumes wallet is synchronized
 819         n = 0
 820         nmax = 0
 821
 822         for account in self.accounts.values():
 823             addresses = account.get_addresses(0)
 824             k = self.num_unused_trailing_addresses(addresses)
 825             for a in addresses[0:-k]:
 826                 if self.history.get(a):
 827                     n = 0
 828                 else:
 829                     n += 1
 830                     if n > nmax: nmax = n
 831         return nmax + 1
 832
 833
 834     def address_is_old(self, address):
 835         age = -1
 836         h = self.history.get(address, [])
 837         if h == ['*']:
 838             return True
 839         for tx_hash, tx_height in h:
 840             if tx_height == 0:
 841                 tx_age = 0
 842             else:
 843                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 844             if tx_age > age:
 845                 age = tx_age
 846         return age > 2
 847
 848
 849     def synchronize_sequence(self, account, for_change):
 850         limit = self.gap_limit_for_change if for_change else self.gap_limit
 851         new_addresses = []
 852         while True:
 853             addresses = account.get_addresses(for_change)
 854             if len(addresses) < limit:
 855                 address = account.create_new_address(for_change)
 856                 self.history[address] = []
 857                 new_addresses.append( address )
 858                 continue
 859
 860             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 861                 break
 862             else:
 863                 address = account.create_new_address(for_change)
 864                 self.history[address] = []
 865                 new_addresses.append( address )
 866
 867         return new_addresses
 868
 869
 870
 871     def create_pending_accounts(self):
 872         for account_type in ['1','2of2','2of3']:
 873             if not self.has_master_public_keys(account_type):
 874                 continue
 875             k, a = self.new_account_address(account_type)
 876             if self.address_is_old(a):
 877                 print_error( "creating account", a )
 878                 self.create_account(account_type)
 879                 self.next_addresses.pop(k)
 880
 881
 882     def synchronize_account(self, account):
 883         new = []
 884         new += self.synchronize_sequence(account, 0)
 885         new += self.synchronize_sequence(account, 1)
 886         return new
 887
 888
 889     def synchronize(self):
 890         if self.master_public_keys:
 891             self.create_pending_accounts()
 892         new = []
 893         for account in self.accounts.values():
 894             new += self.synchronize_account(account)
 895         if new:
 896             self.save_accounts()
 897             self.storage.put('addr_history', self.history, True)
 898         return new
 899
 900
 901     def is_found(self):
 902         return self.history.values() != [[]] * len(self.history)
 903
 904
 905     def add_contact(self, address, label=None):
 906         self.addressbook.append(address)
 907         self.storage.put('contacts', self.addressbook, True)
 908         if label:
 909             self.set_label(address, label)
 910
 911
 912     def delete_contact(self, addr):
 913         if addr in self.addressbook:
 914             self.addressbook.remove(addr)
 915             self.storage.put('addressbook', self.addressbook, True)
 916
 917
 918     def fill_addressbook(self):
 919         for tx_hash, tx in self.transactions.items():
 920             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 921             if is_send:
 922                 for addr, v in tx.outputs:
 923                     if not self.is_mine(addr) and addr not in self.addressbook:
 924                         self.addressbook.append(addr)
 925         # redo labels
 926         # self.update_tx_labels()
 927
 928     def get_num_tx(self, address):
 929         n = 0
 930         for tx in self.transactions.values():
 931             if address in map(lambda x:x[0], tx.outputs): n += 1
 932         return n
 933
 934
 935     def get_address_flags(self, addr):
 936         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 937         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 938         return flags
 939
 940
 941     def get_tx_value(self, tx, account=None):
 942         domain = self.get_account_addresses(account)
 943         return tx.get_value(domain, self.prevout_values)
 944
 945
 946     def update_tx_outputs(self, tx_hash):
 947         tx = self.transactions.get(tx_hash)
 948
 949         for i, (addr, value) in enumerate(tx.outputs):
 950             key = tx_hash+ ':%d'%i
 951             self.prevout_values[key] = value
 952
 953         for item in tx.inputs:
 954             if self.is_mine(item.get('address')):
 955                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 956                 self.spent_outputs.append(key)
 957
 958
 959     def get_addr_balance(self, address):
 960         assert self.is_mine(address)
 961         h = self.history.get(address,[])
 962         if h == ['*']: return 0,0
 963         c = u = 0
 964         received_coins = []   # list of coins received at address
 965
 966         for tx_hash, tx_height in h:
 967             tx = self.transactions.get(tx_hash)
 968             if not tx: continue
 969
 970             for i, (addr, value) in enumerate(tx.outputs):
 971                 if addr == address:
 972                     key = tx_hash + ':%d'%i
 973                     received_coins.append(key)
 974
 975         for tx_hash, tx_height in h:
 976             tx = self.transactions.get(tx_hash)
 977             if not tx: continue
 978             v = 0
 979
 980             for item in tx.inputs:
 981                 addr = item.get('address')
 982                 if addr == address:
 983                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 984                     value = self.prevout_values.get( key )
 985                     if key in received_coins:
 986                         v -= value
 987
 988             for i, (addr, value) in enumerate(tx.outputs):
 989                 key = tx_hash + ':%d'%i
 990                 if addr == address:
 991                     v += value
 992
 993             if tx_height:
 994                 c += v
 995             else:
 996                 u += v
 997         return c, u
 998
 999
1000     def get_account_name(self, k):
1001         if k == 0:
1002             if self.seed_version == 4:
1003                 name = 'Main account'
1004             else:
1005                 name = 'Old account'
1006         else:
1007             default = "Unnamed account"
1008             m = re.match("m/0'/(\d+)", k)
1009             if m:
1010                 num = m.group(1)
1011                 if num == '0':
1012                     default = "Main account"
1013                 else:
1014                     default = "Account %s"%num
1015
1016             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1017             if m:
1018                 num = m.group(1)
1019                 default = "2of2 account %s"%num
1020             name = self.labels.get(k, default)
1021
1022         return name
1023
1024     def get_account_names(self):
1025         accounts = {}
1026         for k, account in self.accounts.items():
1027             accounts[k] = self.get_account_name(k)
1028         if self.imported_keys:
1029             accounts[-1] = 'Imported keys'
1030         return accounts
1031
1032     def get_account_addresses(self, a, include_change=True):
1033         if a is None:
1034             o = self.addresses(True)
1035         elif a == -1:
1036             o = self.imported_keys.keys()
1037         else:
1038             ac = self.accounts[a]
1039             o = ac.get_addresses(0)
1040             if include_change: o += ac.get_addresses(1)
1041         return o
1042
1043     def get_imported_balance(self):
1044         return self.get_balance(self.imported_keys.keys())
1045
1046     def get_account_balance(self, account):
1047         return self.get_balance(self.get_account_addresses(account))
1048
1049     def get_frozen_balance(self):
1050         return self.get_balance(self.frozen_addresses)
1051
1052     def get_balance(self, domain=None):
1053         if domain is None: domain = self.addresses(True)
1054         cc = uu = 0
1055         for addr in domain:
1056             c, u = self.get_addr_balance(addr)
1057             cc += c
1058             uu += u
1059         return cc, uu
1060
1061
1062     def get_unspent_coins(self, domain=None):
1063         coins = []
1064         if domain is None: domain = self.addresses(True)
1065         for addr in domain:
1066             h = self.history.get(addr, [])
1067             if h == ['*']: continue
1068             for tx_hash, tx_height in h:
1069                 tx = self.transactions.get(tx_hash)
1070                 if tx is None: raise BaseException("Wallet not synchronized")
1071                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1072                 for output in tx.d.get('outputs'):
1073                     if output.get('address') != addr: continue
1074                     key = tx_hash + ":%d" % output.get('prevout_n')
1075                     if key in self.spent_outputs: continue
1076                     output['prevout_hash'] = tx_hash
1077                     output['height'] = tx_height
1078                     output['coinbase'] = is_coinbase
1079                     coins.append((tx_height, output))
1080
1081         # sort by age
1082         if coins:
1083             coins = sorted(coins)
1084             if coins[-1][0] != 0:
1085                 while coins[0][0] == 0:
1086                     coins = coins[1:] + [ coins[0] ]
1087         return [x[1] for x in coins]
1088
1089
1090
1091     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1092         domain = self.get_account_addresses(account) if account else None
1093         return self.choose_tx_inputs( amount, fixed_fee, domain )
1094
1095
1096     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1097         """ todo: minimize tx size """
1098         total = 0
1099         fee = self.fee if fixed_fee is None else fixed_fee
1100         if domain is None:
1101             domain = self.addresses(True)
1102
1103         for i in self.frozen_addresses:
1104             if i in domain: domain.remove(i)
1105
1106         prioritized = []
1107         for i in self.prioritized_addresses:
1108             if i in domain:
1109                 domain.remove(i)
1110                 prioritized.append(i)
1111
1112         coins = self.get_unspent_coins(domain)
1113         prioritized_coins = self.get_unspent_coins(prioritized)
1114
1115         inputs = []
1116         coins = prioritized_coins + coins
1117
1118         for item in coins:
1119             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1120                 continue
1121             addr = item.get('address')
1122             v = item.get('value')
1123             total += v
1124             inputs.append(item)
1125             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1126             if total >= amount + fee: break
1127         else:
1128             inputs = []
1129
1130         return inputs, total, fee
1131
1132
1133     def set_fee(self, fee):
1134         if self.fee != fee:
1135             self.fee = fee
1136             self.storage.put('fee_per_kb', self.fee, True)
1137
1138     def estimated_fee(self, inputs):
1139         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1140         fee = self.fee * int(round(estimated_size/1024.))
1141         if fee == 0: fee = self.fee
1142         return fee
1143
1144
1145     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1146         "add change to a transaction"
1147         change_amount = total - ( amount + fee )
1148         if change_amount != 0:
1149             if not change_addr:
1150
1151                 # send change to one of the accounts involved in the tx
1152                 address = inputs[0].get('address')
1153                 account, _ = self.get_address_index(address)
1154
1155                 if not self.use_change or account == -1:
1156                     change_addr = inputs[-1]['address']
1157                 else:
1158                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1159
1160             # Insert the change output at a random position in the outputs
1161             posn = random.randint(0, len(outputs))
1162             outputs[posn:posn] = [( change_addr,  change_amount)]
1163         return outputs
1164
1165
1166     def get_history(self, address):
1167         with self.lock:
1168             return self.history.get(address)
1169
1170
1171     def get_status(self, h):
1172         if not h: return None
1173         if h == ['*']: return '*'
1174         status = ''
1175         for tx_hash, height in h:
1176             status += tx_hash + ':%d:' % height
1177         return hashlib.sha256( status ).digest().encode('hex')
1178
1179
1180     def receive_tx_callback(self, tx_hash, tx, tx_height):
1181
1182         with self.transaction_lock:
1183             self.add_extra_addresses(tx)
1184             if not self.check_new_tx(tx_hash, tx):
1185                 # may happen due to pruning
1186                 print_error("received transaction that is no longer referenced in history", tx_hash)
1187                 return
1188             self.transactions[tx_hash] = tx
1189             self.network.interface.pending_transactions_for_notifications.append(tx)
1190             self.save_transactions()
1191             if self.verifier and tx_height>0:
1192                 self.verifier.add(tx_hash, tx_height)
1193             self.update_tx_outputs(tx_hash)
1194
1195
1196     def save_transactions(self):
1197         tx = {}
1198         for k,v in self.transactions.items():
1199             tx[k] = str(v)
1200         self.storage.put('transactions', tx, True)
1201
1202     def receive_history_callback(self, addr, hist):
1203
1204         if not self.check_new_history(addr, hist):
1205             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1206
1207         with self.lock:
1208             self.history[addr] = hist
1209             self.storage.put('addr_history', self.history, True)
1210
1211         if hist != ['*']:
1212             for tx_hash, tx_height in hist:
1213                 if tx_height>0:
1214                     # add it in case it was previously unconfirmed
1215                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1216
1217
1218     def get_tx_history(self, account=None):
1219         with self.transaction_lock:
1220             history = self.transactions.items()
1221             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1222             result = []
1223
1224             balance = 0
1225             for tx_hash, tx in history:
1226                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1227                 if v is not None: balance += v
1228
1229             c, u = self.get_account_balance(account)
1230
1231             if balance != c+u:
1232                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1233
1234             balance = c + u - balance
1235             for tx_hash, tx in history:
1236                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1237                 if not is_relevant:
1238                     continue
1239                 if value is not None:
1240                     balance += value
1241
1242                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1243                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1244
1245         return result
1246
1247
1248     def get_label(self, tx_hash):
1249         label = self.labels.get(tx_hash)
1250         is_default = (label == '') or (label is None)
1251         if is_default: label = self.get_default_label(tx_hash)
1252         return label, is_default
1253
1254
1255     def get_default_label(self, tx_hash):
1256         tx = self.transactions.get(tx_hash)
1257         default_label = ''
1258         if tx:
1259             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1260             if is_mine:
1261                 for o in tx.outputs:
1262                     o_addr, _ = o
1263                     if not self.is_mine(o_addr):
1264                         try:
1265                             default_label = self.labels[o_addr]
1266                         except KeyError:
1267                             default_label = o_addr
1268                         break
1269                 else:
1270                     default_label = '(internal)'
1271             else:
1272                 for o in tx.outputs:
1273                     o_addr, _ = o
1274                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1275                         break
1276                 else:
1277                     for o in tx.outputs:
1278                         o_addr, _ = o
1279                         if self.is_mine(o_addr):
1280                             break
1281                     else:
1282                         o_addr = None
1283
1284                 if o_addr:
1285                     dest_label = self.labels.get(o_addr)
1286                     try:
1287                         default_label = self.labels[o_addr]
1288                     except KeyError:
1289                         default_label = o_addr
1290
1291         return default_label
1292
1293
1294     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1295         for address, x in outputs:
1296             assert is_valid(address)
1297         amount = sum( map(lambda x:x[1], outputs) )
1298         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1299         if not inputs:
1300             raise ValueError("Not enough funds")
1301         self.add_input_info(inputs)
1302         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1303         return Transaction.from_io(inputs, outputs)
1304
1305
1306     def mktx_from_account(self, outputs, password, fee=None, account=None):
1307         domain = self.get_account_addresses(account) if account else None
1308         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1309
1310
1311     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1312         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1313         keypairs = {}
1314         self.add_keypairs_from_wallet(tx, keypairs, password)
1315         if keypairs:
1316             self.sign_transaction(tx, keypairs)
1317         return tx
1318
1319
1320     def add_input_info(self, inputs):
1321         for txin in inputs:
1322             address = txin['address']
1323             account, sequence = self.get_address_index(address)
1324             txin['KeyID'] = self.get_keyID(account, sequence)
1325             redeemScript = self.accounts[account].redeem_script(sequence)
1326             if redeemScript:
1327                 txin['redeemScript'] = redeemScript
1328             else:
1329                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1330
1331
1332     def sign_transaction(self, tx, keypairs):
1333         tx.sign(keypairs)
1334         run_hook('sign_transaction', tx)
1335
1336
1337     def sendtx(self, tx):
1338         # synchronous
1339         h = self.send_tx(tx)
1340         self.tx_event.wait()
1341         return self.receive_tx(h)
1342
1343     def send_tx(self, tx):
1344         # asynchronous
1345         self.tx_event.clear()
1346         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1347         return tx.hash()
1348
1349     def on_broadcast(self, i, r):
1350         self.tx_result = r.get('result')
1351         self.tx_event.set()
1352
1353     def receive_tx(self,tx_hash):
1354         out = self.tx_result
1355         if out != tx_hash:
1356             return False, "error: " + out
1357         return True, out
1358
1359
1360
1361     def update_password(self, old_password, new_password):
1362         if new_password == '': new_password = None
1363         # this will throw an exception if unicode cannot be converted
1364         decoded = pw_decode(self.seed, old_password)
1365         self.seed = pw_encode( decoded, new_password)
1366         self.storage.put('seed', self.seed, True)
1367         self.use_encryption = (new_password != None)
1368         self.storage.put('use_encryption', self.use_encryption,True)
1369         for k in self.imported_keys.keys():
1370             a = self.imported_keys[k]
1371             b = pw_decode(a, old_password)
1372             c = pw_encode(b, new_password)
1373             self.imported_keys[k] = c
1374         self.storage.put('imported_keys', self.imported_keys, True)
1375
1376         for k, v in self.master_private_keys.items():
1377             b = pw_decode(v, old_password)
1378             c = pw_encode(b, new_password)
1379             self.master_private_keys[k] = c
1380         self.storage.put('master_private_keys', self.master_private_keys, True)
1381
1382
1383     def freeze(self,addr):
1384         if self.is_mine(addr) and addr not in self.frozen_addresses:
1385             self.unprioritize(addr)
1386             self.frozen_addresses.append(addr)
1387             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1388             return True
1389         else:
1390             return False
1391
1392     def unfreeze(self,addr):
1393         if self.is_mine(addr) and addr in self.frozen_addresses:
1394             self.frozen_addresses.remove(addr)
1395             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1396             return True
1397         else:
1398             return False
1399
1400     def prioritize(self,addr):
1401         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1402             self.unfreeze(addr)
1403             self.prioritized_addresses.append(addr)
1404             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1405             return True
1406         else:
1407             return False
1408
1409     def unprioritize(self,addr):
1410         if self.is_mine(addr) and addr in self.prioritized_addresses:
1411             self.prioritized_addresses.remove(addr)
1412             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1413             return True
1414         else:
1415             return False
1416
1417
1418     def set_verifier(self, verifier):
1419         self.verifier = verifier
1420
1421         # review transactions that are in the history
1422         for addr, hist in self.history.items():
1423             if hist == ['*']: continue
1424             for tx_hash, tx_height in hist:
1425                 if tx_height>0:
1426                     # add it in case it was previously unconfirmed
1427                     self.verifier.add(tx_hash, tx_height)
1428
1429
1430         # if we are on a pruning server, remove unverified transactions
1431         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1432         for tx_hash in self.transactions.keys():
1433             if tx_hash not in vr:
1434                 self.transactions.pop(tx_hash)
1435
1436
1437
1438     def check_new_history(self, addr, hist):
1439
1440         # check that all tx in hist are relevant
1441         if hist != ['*']:
1442             for tx_hash, height in hist:
1443                 tx = self.transactions.get(tx_hash)
1444                 if not tx: continue
1445                 if not tx.has_address(addr):
1446                     return False
1447
1448         # check that we are not "orphaning" a transaction
1449         old_hist = self.history.get(addr,[])
1450         if old_hist == ['*']: return True
1451
1452         for tx_hash, height in old_hist:
1453             if tx_hash in map(lambda x:x[0], hist): continue
1454             found = False
1455             for _addr, _hist in self.history.items():
1456                 if _addr == addr: continue
1457                 if _hist == ['*']: continue
1458                 _tx_hist = map(lambda x:x[0], _hist)
1459                 if tx_hash in _tx_hist:
1460                     found = True
1461                     break
1462
1463             if not found:
1464                 tx = self.transactions.get(tx_hash)
1465                 # tx might not be there
1466                 if not tx: continue
1467
1468                 # already verified?
1469                 if self.verifier.get_height(tx_hash):
1470                     continue
1471                 # unconfirmed tx
1472                 print_error("new history is orphaning transaction:", tx_hash)
1473                 # check that all outputs are not mine, request histories
1474                 ext_requests = []
1475                 for _addr, _v in tx.outputs:
1476                     # assert not self.is_mine(_addr)
1477                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1478
1479                 ext_h = self.network.synchronous_get(ext_requests)
1480                 print_error("sync:", ext_requests, ext_h)
1481                 height = None
1482                 for h in ext_h:
1483                     if h == ['*']: continue
1484                     for item in h:
1485                         if item.get('tx_hash') == tx_hash:
1486                             height = item.get('height')
1487                 if height:
1488                     print_error("found height for", tx_hash, height)
1489                     self.verifier.add(tx_hash, height)
1490                 else:
1491                     print_error("removing orphaned tx from history", tx_hash)
1492                     self.transactions.pop(tx_hash)
1493
1494         return True
1495
1496
1497
1498     def check_new_tx(self, tx_hash, tx):
1499         # 1 check that tx is referenced in addr_history.
1500         addresses = []
1501         for addr, hist in self.history.items():
1502             if hist == ['*']:continue
1503             for txh, height in hist:
1504                 if txh == tx_hash:
1505                     addresses.append(addr)
1506
1507         if not addresses:
1508             return False
1509
1510         # 2 check that referencing addresses are in the tx
1511         for addr in addresses:
1512             if not tx.has_address(addr):
1513                 return False
1514
1515         return True
1516
1517
1518     def start_threads(self, network):
1519         from verifier import TxVerifier
1520         self.network = network
1521         self.verifier = TxVerifier(self.network, self.storage)
1522         self.verifier.start()
1523         self.set_verifier(self.verifier)
1524         self.synchronizer = WalletSynchronizer(self, network)
1525         self.synchronizer.start()
1526
1527     def stop_threads(self):
1528         self.verifier.stop()
1529         self.synchronizer.stop()
1530
1531
1532
1533     def restore(self, callback):
1534         from i18n import _
1535         def wait_for_wallet():
1536             self.set_up_to_date(False)
1537             while not self.is_up_to_date():
1538                 msg = "%s\n%s %d\n%s %.1f"%(
1539                     _("Please wait..."),
1540                     _("Addresses generated:"),
1541                     len(self.addresses(True)),_("Kilobytes received:"),
1542                     self.network.interface.bytes_received/1024.)
1543
1544                 apply(callback, (msg,))
1545                 time.sleep(0.1)
1546
1547         def wait_for_network():
1548             while not self.network.interface.is_connected:
1549                 msg = "%s \n" % (_("Connecting..."))
1550                 apply(callback, (msg,))
1551                 time.sleep(0.1)
1552
1553         # wait until we are connected, because the user might have selected another server
1554         wait_for_network()
1555
1556
1557         self.create_accounts()
1558         wait_for_wallet()
1559
1560
1561
1562
1563 class WalletSynchronizer(threading.Thread):
1564
1565
1566     def __init__(self, wallet, network):
1567         threading.Thread.__init__(self)
1568         self.daemon = True
1569         self.wallet = wallet
1570         self.network = network
1571         self.was_updated = True
1572         self.running = False
1573         self.lock = threading.Lock()
1574         self.queue = Queue.Queue()
1575
1576     def stop(self):
1577         with self.lock: self.running = False
1578
1579     def is_running(self):
1580         with self.lock: return self.running
1581
1582
1583     def subscribe_to_addresses(self, addresses):
1584         messages = []
1585         for addr in addresses:
1586             messages.append(('blockchain.address.subscribe', [addr]))
1587         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1588
1589
1590     def run(self):
1591         with self.lock:
1592             self.running = True
1593
1594         while self.is_running():
1595
1596             if not self.network.is_connected():
1597                 print_error("synchronizer: waiting for interface")
1598                 self.network.wait_until_connected()
1599
1600             self.run_interface(self.network.interface)
1601
1602
1603     def run_interface(self, interface):
1604
1605         print_error("synchronizer: connected to", interface.server)
1606
1607         requested_tx = []
1608         missing_tx = []
1609         requested_histories = {}
1610
1611         # request any missing transactions
1612         for history in self.wallet.history.values():
1613             if history == ['*']: continue
1614             for tx_hash, tx_height in history:
1615                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1616                     missing_tx.append( (tx_hash, tx_height) )
1617
1618         if missing_tx:
1619             print_error("missing tx", missing_tx)
1620
1621         # subscriptions
1622         self.subscribe_to_addresses(self.wallet.addresses(True))
1623
1624         while self.is_running():
1625             # 1. create new addresses
1626             new_addresses = self.wallet.synchronize()
1627
1628             # request missing addresses
1629             if new_addresses:
1630                 self.subscribe_to_addresses(new_addresses)
1631
1632             # request missing transactions
1633             for tx_hash, tx_height in missing_tx:
1634                 if (tx_hash, tx_height) not in requested_tx:
1635                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1636                     requested_tx.append( (tx_hash, tx_height) )
1637             missing_tx = []
1638
1639             # detect if situation has changed
1640             if interface.is_up_to_date() and self.queue.empty():
1641                 if not self.wallet.is_up_to_date():
1642                     self.wallet.set_up_to_date(True)
1643                     self.was_updated = True
1644             else:
1645                 if self.wallet.is_up_to_date():
1646                     self.wallet.set_up_to_date(False)
1647                     self.was_updated = True
1648
1649             if self.was_updated:
1650                 self.wallet.network.trigger_callback('updated')
1651                 self.was_updated = False
1652
1653             # 2. get a response
1654             try:
1655                 r = self.queue.get(block=True, timeout=1)
1656             except Queue.Empty:
1657                 continue
1658
1659             if interface != self.network.interface:
1660                 break
1661
1662             if not r:
1663                 continue
1664
1665             # 3. handle response
1666             method = r['method']
1667             params = r['params']
1668             result = r.get('result')
1669             error = r.get('error')
1670             if error:
1671                 print "error", r
1672                 continue
1673
1674             if method == 'blockchain.address.subscribe':
1675                 addr = params[0]
1676                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1677                     if requested_histories.get(addr) is None:
1678                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1679                         requested_histories[addr] = result
1680
1681             elif method == 'blockchain.address.get_history':
1682                 addr = params[0]
1683                 print_error("receiving history", addr, result)
1684                 if result == ['*']:
1685                     assert requested_histories.pop(addr) == '*'
1686                     self.wallet.receive_history_callback(addr, result)
1687                 else:
1688                     hist = []
1689                     # check that txids are unique
1690                     txids = []
1691                     for item in result:
1692                         tx_hash = item['tx_hash']
1693                         if tx_hash not in txids:
1694                             txids.append(tx_hash)
1695                             hist.append( (tx_hash, item['height']) )
1696
1697                     if len(hist) != len(result):
1698                         raise BaseException("error: server sent history with non-unique txid", result)
1699
1700                     # check that the status corresponds to what was announced
1701                     rs = requested_histories.pop(addr)
1702                     if self.wallet.get_status(hist) != rs:
1703                         raise BaseException("error: status mismatch: %s"%addr)
1704
1705                     # store received history
1706                     self.wallet.receive_history_callback(addr, hist)
1707
1708                     # request transactions that we don't have
1709                     for tx_hash, tx_height in hist:
1710                         if self.wallet.transactions.get(tx_hash) is None:
1711                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1712                                 missing_tx.append( (tx_hash, tx_height) )
1713
1714             elif method == 'blockchain.transaction.get':
1715                 tx_hash = params[0]
1716                 tx_height = params[1]
1717                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1718                 tx = Transaction(result)
1719                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1720                 self.was_updated = True
1721                 requested_tx.remove( (tx_hash, tx_height) )
1722                 print_error("received tx:", tx_hash, len(tx.raw))
1723
1724             else:
1725                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1726
1727             if self.was_updated and not requested_tx:
1728                 self.wallet.network.trigger_callback('updated')
1729                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1730
1731                 self.was_updated = False