lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40
  41 # AES encryption
  42 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  43 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  44
  45 def pw_encode(s, password):
  46     if password:
  47         secret = Hash(password)
  48         return EncodeAES(secret, s)
  49     else:
  50         return s
  51
  52 def pw_decode(s, password):
  53     if password is not None:
  54         secret = Hash(password)
  55         try:
  56             d = DecodeAES(secret, s)
  57         except:
  58             raise BaseException('Invalid password')
  59         return d
  60     else:
  61         return s
  62
  63
  64
  65
  66
  67 from version import *
  68
  69
  70 class WalletStorage:
  71
  72     def __init__(self, config):
  73         self.lock = threading.Lock()
  74         self.data = {}
  75         self.file_exists = False
  76         self.path = self.init_path(config)
  77         print_error( "wallet path", self.path )
  78         if self.path:
  79             self.read(self.path)
  80
  81
  82     def init_path(self, config):
  83         """Set the path of the wallet."""
  84
  85         # command line -w option
  86         path = config.get('wallet_path')
  87         if path:
  88             return path
  89
  90         # path in config file
  91         path = config.get('default_wallet_path')
  92         if path:
  93             return path
  94
  95         # default path
  96         dirpath = os.path.join(config.path, "wallets")
  97         if not os.path.exists(dirpath):
  98             os.mkdir(dirpath)
  99
 100         new_path = os.path.join(config.path, "wallets", "default_wallet")
 101
 102         # default path in pre 1.9 versions
 103         old_path = os.path.join(config.path, "electrum.dat")
 104         if os.path.exists(old_path) and not os.path.exists(new_path):
 105             os.rename(old_path, new_path)
 106
 107         return new_path
 108
 109
 110     def read(self, path):
 111         """Read the contents of the wallet file."""
 112         try:
 113             with open(self.path, "r") as f:
 114                 data = f.read()
 115         except IOError:
 116             return
 117         try:
 118             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 119         except:
 120             raise IOError("Cannot read wallet file.")
 121
 122         self.data = d
 123         self.file_exists = True
 124
 125
 126     def get(self, key, default=None):
 127         return self.data.get(key, default)
 128
 129     def put(self, key, value, save = True):
 130
 131         with self.lock:
 132             if value is not None:
 133                 self.data[key] = value
 134             else:
 135                 self.data.pop[key]
 136             if save:
 137                 self.write()
 138
 139     def write(self):
 140         s = repr(self.data)
 141         f = open(self.path,"w")
 142         f.write( s )
 143         f.close()
 144         if 'ANDROID_DATA' not in os.environ:
 145             import stat
 146             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 147
 148
 149 class Wallet:
 150
 151     def __init__(self, storage):
 152
 153         self.storage = storage
 154         self.electrum_version = ELECTRUM_VERSION
 155         self.gap_limit_for_change = 3 # constant
 156
 157         # saved fields
 158         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 159
 160         self.gap_limit             = storage.get('gap_limit', 5)
 161         self.use_change            = storage.get('use_change',True)
 162         self.use_encryption        = storage.get('use_encryption', False)
 163         self.seed                  = storage.get('seed', '')               # encrypted
 164         self.labels                = storage.get('labels', {})
 165         self.frozen_addresses      = storage.get('frozen_addresses',[])
 166         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 167         self.addressbook           = storage.get('contacts', [])
 168
 169         self.imported_keys         = storage.get('imported_keys',{})
 170         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 171
 172         self.fee                   = int(storage.get('fee_per_kb',20000))
 173
 174         self.master_public_keys = storage.get('master_public_keys',{})
 175         self.master_private_keys = storage.get('master_private_keys', {})
 176
 177         self.next_addresses = storage.get('next_addresses',{})
 178
 179         if self.seed_version not in [4, 6]:
 180             msg = "This wallet seed is not supported."
 181             if self.seed_version in [5]: msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 182             raise ValueError(msg)
 183
 184         self.load_accounts()
 185
 186         self.transactions = {}
 187         tx_list = self.storage.get('transactions',{})
 188         for k,v in tx_list.items():
 189             try:
 190                 tx = Transaction(v)
 191             except:
 192                 print_msg("Warning: Cannot deserialize transactions. skipping")
 193                 continue
 194
 195             self.add_extra_addresses(tx)
 196             self.transactions[k] = tx
 197
 198         for h,tx in self.transactions.items():
 199             if not self.check_new_tx(h, tx):
 200                 print_error("removing unreferenced tx", h)
 201                 self.transactions.pop(h)
 202
 203
 204         # not saved
 205         self.prevout_values = {}     # my own transaction outputs
 206         self.spent_outputs = []
 207
 208         # spv
 209         self.verifier = None
 210
 211         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 212         # interface.is_up_to_date() returns true when all requests have been answered and processed
 213         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 214
 215         self.up_to_date = False
 216         self.lock = threading.Lock()
 217         self.transaction_lock = threading.Lock()
 218         self.tx_event = threading.Event()
 219
 220         for tx_hash, tx in self.transactions.items():
 221             self.update_tx_outputs(tx_hash)
 222
 223
 224     def add_extra_addresses(self, tx):
 225         h = tx.hash()
 226         # find the address corresponding to pay-to-pubkey inputs
 227         tx.add_extra_addresses(self.transactions)
 228         for o in tx.d.get('outputs'):
 229             if o.get('is_pubkey'):
 230                 for tx2 in self.transactions.values():
 231                     tx2.add_extra_addresses({h:tx})
 232
 233
 234
 235
 236     def set_up_to_date(self,b):
 237         with self.lock: self.up_to_date = b
 238
 239
 240     def is_up_to_date(self):
 241         with self.lock: return self.up_to_date
 242
 243
 244     def update(self):
 245         self.up_to_date = False
 246         while not self.is_up_to_date():
 247             time.sleep(0.1)
 248
 249
 250     def import_key(self, sec, password):
 251         # check password
 252         seed = self.get_seed(password)
 253         try:
 254             address = address_from_private_key(sec)
 255         except:
 256             raise BaseException('Invalid private key')
 257
 258         if self.is_mine(address):
 259             raise BaseException('Address already in wallet')
 260
 261         # store the originally requested keypair into the imported keys table
 262         self.imported_keys[address] = pw_encode(sec, password )
 263         self.storage.put('imported_keys', self.imported_keys, True)
 264         if self.synchronizer:
 265             self.synchronizer.subscribe_to_addresses([address])
 266         return address
 267
 268     def delete_imported_key(self, addr):
 269         if addr in self.imported_keys:
 270             self.imported_keys.pop(addr)
 271             self.storage.put('imported_keys', self.imported_keys, True)
 272
 273
 274     def make_seed(self):
 275         import mnemonic, ecdsa
 276         entropy = ecdsa.util.randrange( pow(2,160) )
 277         nonce = 0
 278         while True:
 279             ss = "%040x"%(entropy+nonce)
 280             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 281             # we keep only 13 words, that's approximately 139 bits of entropy
 282             words = mnemonic.mn_encode(s)[0:13]
 283             seed = ' '.join(words)
 284             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 285                 break  # this removes 12 bits of entropy
 286             nonce += 1
 287
 288         return seed
 289
 290
 291     def init_seed(self, seed):
 292         import mnemonic
 293
 294         if self.seed:
 295             raise BaseException("a seed exists")
 296
 297         if not seed:
 298             self.seed = random_seed(128)
 299             self.seed_version = 4
 300             return
 301
 302         #if not seed:
 303         #    self.seed = self.make_seed()
 304         #    self.seed_version = SEED_VERSION
 305         #    return
 306
 307         # find out what kind of wallet we are
 308         try:
 309             seed.decode('hex')
 310             self.seed_version = 4
 311             return
 312         except:
 313             pass
 314
 315         words = seed.split()
 316         self.seed_version = 4
 317         self.seed = mnemonic.mn_decode(words)
 318
 319         #try:
 320         #    mnemonic.mn_decode(words)
 321         #    uses_electrum_words = True
 322         #except:
 323         #    uses_electrum_words = False
 324         #
 325         #if uses_electrum_words and len(words) != 13:
 326         #    self.seed_version = 4
 327         #    self.seed = mnemonic.mn_decode(words)
 328         #else:
 329         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 330         #    self.seed_version = SEED_VERSION
 331         #    self.seed = seed
 332
 333
 334     def save_seed(self):
 335         self.storage.put('seed', self.seed, True)
 336         self.storage.put('seed_version', self.seed_version, True)
 337
 338     def create_watching_only_wallet(self, c0, K0):
 339         cK0 = ""
 340         self.master_public_keys = {
 341             "m/0'/": (c0, K0, cK0),
 342             }
 343         self.storage.put('master_public_keys', self.master_public_keys, True)
 344         self.create_account('1','Main account')
 345
 346
 347     def create_accounts(self):
 348         if self.seed_version == 4:
 349             self.create_old_account()
 350         else:
 351             # create default account
 352             self.create_master_keys('1')
 353             self.create_account('1','Main account')
 354
 355
 356     def create_master_keys(self, account_type):
 357         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 358         if account_type == '1':
 359             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 360             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 361             self.master_private_keys["m/0'/"] = k0
 362         elif account_type == '2of2':
 363             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 364             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 365             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 366             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 367             self.master_private_keys["m/1'/"] = k1
 368             self.master_private_keys["m/2'/"] = k2
 369         elif account_type == '2of3':
 370             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 371             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 372             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 373             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 374             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 375             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 376             self.master_private_keys["m/3'/"] = k3
 377             self.master_private_keys["m/4'/"] = k4
 378             self.master_private_keys["m/5'/"] = k5
 379
 380         self.storage.put('master_public_keys', self.master_public_keys, True)
 381         self.storage.put('master_private_keys', self.master_private_keys, True)
 382
 383     def has_master_public_keys(self, account_type):
 384         if account_type == '1':
 385             return "m/0'/" in self.master_public_keys
 386         elif account_type == '2of2':
 387             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 388         elif account_type == '2of3':
 389             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 390
 391     def find_root_by_master_key(self, c, K):
 392         for key, v in self.master_public_keys.items():
 393             if key == "m/":continue
 394             cc, KK, _ = v
 395             if (c == cc) and (K == KK):
 396                 return key
 397
 398     def deseed_root(self, seed, password):
 399         # for safety, we ask the user to enter their seed
 400         assert seed == self.get_seed(password)
 401         self.seed = ''
 402         self.storage.put('seed', '', True)
 403
 404
 405     def deseed_branch(self, k):
 406         # check that parent has no seed
 407         assert self.seed == ''
 408         self.master_private_keys.pop(k)
 409         self.storage.put('master_private_keys', self.master_private_keys, True)
 410
 411     def is_watching_only(self):
 412         return (self.seed == '') and (self.master_private_keys == {})
 413
 414
 415
 416     def account_id(self, account_type, i):
 417         if account_type == '1':
 418             return "m/0'/%d"%i
 419         elif account_type == '2of2':
 420             return "m/1'/%d & m/2'/%d"%(i,i)
 421         elif account_type == '2of3':
 422             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 423         else:
 424             raise BaseException('unknown account type')
 425
 426
 427     def num_accounts(self, account_type):
 428         keys = self.accounts.keys()
 429         i = 0
 430         while True:
 431             account_id = self.account_id(account_type, i)
 432             if account_id not in keys: break
 433             i += 1
 434         return i
 435
 436
 437     def new_account_address(self, account_type = '1'):
 438         i = self.num_accounts(account_type)
 439         k = self.account_id(account_type,i)
 440
 441         addr = self.next_addresses.get(k)
 442         if not addr:
 443             account_id, account = self.next_account(account_type)
 444             addr = account.first_address()
 445             self.next_addresses[k] = addr
 446             self.storage.put('next_addresses',self.next_addresses)
 447
 448         return k, addr
 449
 450
 451     def next_account(self, account_type = '1'):
 452
 453         i = self.num_accounts(account_type)
 454         account_id = self.account_id(account_type,i)
 455
 456         if account_type is '1':
 457             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 458             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 459             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 460
 461         elif account_type == '2of2':
 462             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 463             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 464             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 465             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 466             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 467
 468         elif account_type == '2of3':
 469             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 470             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 471             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 472             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 473             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 474             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 475             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 476
 477         return account_id, account
 478
 479
 480     def set_label(self, name, text = None):
 481         changed = False
 482         old_text = self.labels.get(name)
 483         if text:
 484             if old_text != text:
 485                 self.labels[name] = text
 486                 changed = True
 487         else:
 488             if old_text:
 489                 self.labels.pop(name)
 490                 changed = True
 491
 492         if changed:
 493             self.storage.put('labels', self.labels, True)
 494
 495         run_hook('set_label', name, text, changed)
 496         return changed
 497
 498
 499
 500     def create_account(self, account_type = '1', name = None):
 501         k, account = self.next_account(account_type)
 502         if k in self.pending_accounts:
 503             self.pending_accounts.pop(k)
 504             self.storage.put('pending_accounts', self.pending_accounts)
 505
 506         self.accounts[k] = account
 507         self.save_accounts()
 508         if name:
 509             self.set_label(k, name)
 510
 511
 512     def create_old_account(self):
 513         mpk = OldAccount.mpk_from_seed(self.seed)
 514         self.storage.put('master_public_key', mpk, True)
 515         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 516         self.save_accounts()
 517
 518
 519     def save_accounts(self):
 520         d = {}
 521         for k, v in self.accounts.items():
 522             d[k] = v.dump()
 523         self.storage.put('accounts', d, True)
 524
 525
 526
 527     def load_accounts(self):
 528         d = self.storage.get('accounts', {})
 529         self.accounts = {}
 530         for k, v in d.items():
 531             if k == 0:
 532                 v['mpk'] = self.storage.get('master_public_key')
 533                 self.accounts[k] = OldAccount(v)
 534             elif '&' in k:
 535                 self.accounts[k] = BIP32_Account_2of2(v)
 536             else:
 537                 self.accounts[k] = BIP32_Account(v)
 538
 539         self.pending_accounts = self.storage.get('pending_accounts',{})
 540
 541
 542     def delete_pending_account(self, k):
 543         self.pending_accounts.pop(k)
 544         self.storage.put('pending_accounts', self.pending_accounts)
 545
 546     def account_is_pending(self, k):
 547         return k in self.pending_accounts
 548
 549     def create_pending_account(self, acct_type, name):
 550         k, addr = self.new_account_address(acct_type)
 551         self.set_label(k, name)
 552         self.pending_accounts[k] = addr
 553         self.storage.put('pending_accounts', self.pending_accounts)
 554
 555     def get_pending_accounts(self):
 556         return self.pending_accounts.items()
 557
 558
 559     def addresses(self, include_change = True, _next=True):
 560         o = self.get_account_addresses(-1, include_change)
 561         for a in self.accounts.keys():
 562             o += self.get_account_addresses(a, include_change)
 563
 564         if _next:
 565             for addr in self.next_addresses.values():
 566                 if addr not in o:
 567                     o += [addr]
 568         return o
 569
 570
 571     def is_mine(self, address):
 572         return address in self.addresses(True)
 573
 574
 575     def is_change(self, address):
 576         if not self.is_mine(address): return False
 577         if address in self.imported_keys.keys(): return False
 578         acct, s = self.get_address_index(address)
 579         if s is None: return False
 580         return s[0] == 1
 581
 582     def get_master_public_key(self):
 583         if self.seed_version == 4:
 584             return self.storage.get("master_public_key")
 585         else:
 586             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 587             return repr((c, K))
 588
 589     def get_master_private_key(self, account, password):
 590         master_k = pw_decode( self.master_private_keys[account], password)
 591         master_c, master_K, master_Kc = self.master_public_keys[account]
 592         try:
 593             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 594             assert K.encode('hex') == master_K
 595         except:
 596             raise BaseException("Invalid password")
 597         return master_k
 598
 599
 600     def get_address_index(self, address):
 601         if address in self.imported_keys.keys():
 602             return -1, None
 603
 604         for account in self.accounts.keys():
 605             for for_change in [0,1]:
 606                 addresses = self.accounts[account].get_addresses(for_change)
 607                 for addr in addresses:
 608                     if address == addr:
 609                         return account, (for_change, addresses.index(addr))
 610
 611         for k,v in self.next_addresses.items():
 612             if v == address:
 613                 return k, (0,0)
 614
 615         raise BaseException("Address not found", address)
 616
 617
 618     def get_roots(self, account):
 619         roots = []
 620         for a in account.split('&'):
 621             s = a.strip()
 622             m = re.match("(m/\d+'/)(\d+)", s)
 623             roots.append( m.group(1) )
 624         return roots
 625
 626     def is_seeded(self, account):
 627         if type(account) is int:
 628             return self.seed is not None
 629
 630         for root in self.get_roots(account):
 631             if root not in self.master_private_keys.keys():
 632                 return False
 633         return True
 634
 635     def rebase_sequence(self, account, sequence):
 636         c, i = sequence
 637         dd = []
 638         for a in account.split('&'):
 639             s = a.strip()
 640             m = re.match("(m/\d+'/)(\d+)", s)
 641             root = m.group(1)
 642             num = int(m.group(2))
 643             dd.append( (root, [num,c,i] ) )
 644         return dd
 645
 646
 647     def get_keyID(self, account, sequence):
 648         if account == 0:
 649             return 'old'
 650
 651         rs = self.rebase_sequence(account, sequence)
 652         dd = []
 653         for root, public_sequence in rs:
 654             c, K, _ = self.master_public_keys[root]
 655             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 656             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 657         return '&'.join(dd)
 658
 659
 660
 661     def get_seed(self, password):
 662         s = pw_decode(self.seed, password)
 663         if self.seed_version == 4:
 664             seed = s
 665         else:
 666             seed = mnemonic_hash(s)
 667         #todo:  #self.sequences[0].check_seed(seed)
 668         return seed
 669
 670
 671     def get_mnemonic(self, password):
 672         import mnemonic
 673         s = pw_decode(self.seed, password)
 674         if self.seed_version == 4:
 675             return ' '.join(mnemonic.mn_encode(s))
 676         else:
 677             return s
 678
 679
 680
 681     def get_private_key(self, address, password):
 682         out = []
 683         if address in self.imported_keys.keys():
 684             out.append( pw_decode( self.imported_keys[address], password ) )
 685         else:
 686             account, sequence = self.get_address_index(address)
 687             if account == 0:
 688                 seed = self.get_seed(password)
 689                 pk = self.accounts[account].get_private_key(seed, sequence)
 690                 out.append(pk)
 691                 return out
 692
 693             # assert address == self.accounts[account].get_address(*sequence)
 694             rs = self.rebase_sequence( account, sequence)
 695             for root, public_sequence in rs:
 696
 697                 if root not in self.master_private_keys.keys(): continue
 698                 master_k = self.get_master_private_key(root, password)
 699                 master_c, _, _ = self.master_public_keys[root]
 700                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 701                 out.append(pk)
 702
 703         return out
 704
 705
 706     def add_keypairs_from_wallet(self, tx, keypairs, password):
 707         for txin in tx.inputs:
 708             address = txin['address']
 709             private_keys = self.get_private_key(address, password)
 710             for sec in private_keys:
 711                 pubkey = public_key_from_private_key(sec)
 712                 keypairs[ pubkey ] = sec
 713
 714
 715     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 716         for txin in tx.inputs:
 717             keyid = txin.get('KeyID')
 718             if keyid:
 719                 roots = []
 720                 for s in keyid.split('&'):
 721                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 722                     if not m: continue
 723                     c = m.group(1)
 724                     K = m.group(2)
 725                     sequence = m.group(3)
 726                     root = self.find_root_by_master_key(c,K)
 727                     if not root: continue
 728                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 729                     root = root + '%d'%sequence[0]
 730                     sequence = sequence[1:]
 731                     roots.append((root,sequence))
 732
 733                 account_id = " & ".join( map(lambda x:x[0], roots) )
 734                 account = self.accounts.get(account_id)
 735                 if not account: continue
 736                 addr = account.get_address(*sequence)
 737                 txin['address'] = addr # fixme: side effect
 738                 pk = self.get_private_key(addr, password)
 739                 for sec in pk:
 740                     pubkey = public_key_from_private_key(sec)
 741                     keypairs[pubkey] = sec
 742
 743
 744
 745     def signrawtransaction(self, tx, input_info, private_keys, password):
 746
 747         # check that the password is correct
 748         seed = self.get_seed(password)
 749
 750         # add input info
 751         tx.add_input_info(input_info)
 752
 753         # add redeem script for coins that are in the wallet
 754         # FIXME: add redeemPubkey too!
 755         unspent_coins = self.get_unspent_coins()
 756         for txin in tx.inputs:
 757             for item in unspent_coins:
 758                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 759                     print_error( "tx input is in unspent coins" )
 760                     txin['scriptPubKey'] = item['scriptPubKey']
 761                     account, sequence = self.get_address_index(item['address'])
 762                     if account != -1:
 763                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 764                         print_error("added redeemScript", txin['redeemScript'])
 765                     break
 766
 767
 768         # build a list of public/private keys
 769         keypairs = {}
 770
 771         # add private keys from parameter
 772         for sec in private_keys:
 773             pubkey = public_key_from_private_key(sec)
 774             keypairs[ pubkey ] = sec
 775
 776         # add private_keys from KeyID
 777         self.add_keypairs_from_KeyID(tx, keypairs, password)
 778
 779         # add private keys from wallet
 780         self.add_keypairs_from_wallet(tx, keypairs, password)
 781         self.sign_transaction(tx, keypairs)
 782
 783
 784     def sign_message(self, address, message, password):
 785         keys = self.get_private_key(address, password)
 786         assert len(keys) == 1
 787         sec = keys[0]
 788         key = regenerate_key(sec)
 789         compressed = is_compressed(sec)
 790         return key.sign_message(message, compressed, address)
 791
 792
 793     def change_gap_limit(self, value):
 794         if value >= self.gap_limit:
 795             self.gap_limit = value
 796             self.storage.put('gap_limit', self.gap_limit, True)
 797             #self.interface.poke('synchronizer')
 798             return True
 799
 800         elif value >= self.min_acceptable_gap():
 801             for key, account in self.accounts.items():
 802                 addresses = account[0]
 803                 k = self.num_unused_trailing_addresses(addresses)
 804                 n = len(addresses) - k + value
 805                 addresses = addresses[0:n]
 806                 self.accounts[key][0] = addresses
 807
 808             self.gap_limit = value
 809             self.storage.put('gap_limit', self.gap_limit, True)
 810             self.save_accounts()
 811             return True
 812         else:
 813             return False
 814
 815     def num_unused_trailing_addresses(self, addresses):
 816         k = 0
 817         for a in addresses[::-1]:
 818             if self.history.get(a):break
 819             k = k + 1
 820         return k
 821
 822     def min_acceptable_gap(self):
 823         # fixme: this assumes wallet is synchronized
 824         n = 0
 825         nmax = 0
 826
 827         for account in self.accounts.values():
 828             addresses = account.get_addresses(0)
 829             k = self.num_unused_trailing_addresses(addresses)
 830             for a in addresses[0:-k]:
 831                 if self.history.get(a):
 832                     n = 0
 833                 else:
 834                     n += 1
 835                     if n > nmax: nmax = n
 836         return nmax + 1
 837
 838
 839     def address_is_old(self, address):
 840         age = -1
 841         h = self.history.get(address, [])
 842         if h == ['*']:
 843             return True
 844         for tx_hash, tx_height in h:
 845             if tx_height == 0:
 846                 tx_age = 0
 847             else:
 848                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 849             if tx_age > age:
 850                 age = tx_age
 851         return age > 2
 852
 853
 854     def synchronize_sequence(self, account, for_change):
 855         limit = self.gap_limit_for_change if for_change else self.gap_limit
 856         new_addresses = []
 857         while True:
 858             addresses = account.get_addresses(for_change)
 859             if len(addresses) < limit:
 860                 address = account.create_new_address(for_change)
 861                 self.history[address] = []
 862                 new_addresses.append( address )
 863                 continue
 864
 865             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 866                 break
 867             else:
 868                 address = account.create_new_address(for_change)
 869                 self.history[address] = []
 870                 new_addresses.append( address )
 871
 872         return new_addresses
 873
 874
 875
 876     def create_pending_accounts(self):
 877         for account_type in ['1','2of2','2of3']:
 878             if not self.has_master_public_keys(account_type):
 879                 continue
 880             k, a = self.new_account_address(account_type)
 881             if self.address_is_old(a):
 882                 print_error( "creating account", a )
 883                 self.create_account(account_type)
 884                 self.next_addresses.pop(k)
 885
 886
 887     def synchronize_account(self, account):
 888         new = []
 889         new += self.synchronize_sequence(account, 0)
 890         new += self.synchronize_sequence(account, 1)
 891         return new
 892
 893
 894     def synchronize(self):
 895         if self.master_public_keys:
 896             self.create_pending_accounts()
 897         new = []
 898         for account in self.accounts.values():
 899             new += self.synchronize_account(account)
 900         if new:
 901             self.save_accounts()
 902             self.storage.put('addr_history', self.history, True)
 903         return new
 904
 905
 906     def is_found(self):
 907         return self.history.values() != [[]] * len(self.history)
 908
 909
 910     def add_contact(self, address, label=None):
 911         self.addressbook.append(address)
 912         self.storage.put('contacts', self.addressbook, True)
 913         if label:
 914             self.set_label(address, label)
 915
 916
 917     def delete_contact(self, addr):
 918         if addr in self.addressbook:
 919             self.addressbook.remove(addr)
 920             self.storage.put('addressbook', self.addressbook, True)
 921
 922
 923     def fill_addressbook(self):
 924         for tx_hash, tx in self.transactions.items():
 925             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 926             if is_send:
 927                 for addr, v in tx.outputs:
 928                     if not self.is_mine(addr) and addr not in self.addressbook:
 929                         self.addressbook.append(addr)
 930         # redo labels
 931         # self.update_tx_labels()
 932
 933     def get_num_tx(self, address):
 934         n = 0
 935         for tx in self.transactions.values():
 936             if address in map(lambda x:x[0], tx.outputs): n += 1
 937         return n
 938
 939
 940     def get_address_flags(self, addr):
 941         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 942         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 943         return flags
 944
 945
 946     def get_tx_value(self, tx, account=None):
 947         domain = self.get_account_addresses(account)
 948         return tx.get_value(domain, self.prevout_values)
 949
 950
 951     def update_tx_outputs(self, tx_hash):
 952         tx = self.transactions.get(tx_hash)
 953
 954         for i, (addr, value) in enumerate(tx.outputs):
 955             key = tx_hash+ ':%d'%i
 956             self.prevout_values[key] = value
 957
 958         for item in tx.inputs:
 959             if self.is_mine(item.get('address')):
 960                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 961                 self.spent_outputs.append(key)
 962
 963
 964     def get_addr_balance(self, address):
 965         assert self.is_mine(address)
 966         h = self.history.get(address,[])
 967         if h == ['*']: return 0,0
 968         c = u = 0
 969         received_coins = []   # list of coins received at address
 970
 971         for tx_hash, tx_height in h:
 972             tx = self.transactions.get(tx_hash)
 973             if not tx: continue
 974
 975             for i, (addr, value) in enumerate(tx.outputs):
 976                 if addr == address:
 977                     key = tx_hash + ':%d'%i
 978                     received_coins.append(key)
 979
 980         for tx_hash, tx_height in h:
 981             tx = self.transactions.get(tx_hash)
 982             if not tx: continue
 983             v = 0
 984
 985             for item in tx.inputs:
 986                 addr = item.get('address')
 987                 if addr == address:
 988                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 989                     value = self.prevout_values.get( key )
 990                     if key in received_coins:
 991                         v -= value
 992
 993             for i, (addr, value) in enumerate(tx.outputs):
 994                 key = tx_hash + ':%d'%i
 995                 if addr == address:
 996                     v += value
 997
 998             if tx_height:
 999                 c += v
1000             else:
1001                 u += v
1002         return c, u
1003
1004
1005     def get_account_name(self, k):
1006         if k == 0:
1007             if self.seed_version == 4:
1008                 name = 'Main account'
1009             else:
1010                 name = 'Old account'
1011         else:
1012             default = "Unnamed account"
1013             m = re.match("m/0'/(\d+)", k)
1014             if m:
1015                 num = m.group(1)
1016                 if num == '0':
1017                     default = "Main account"
1018                 else:
1019                     default = "Account %s"%num
1020
1021             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1022             if m:
1023                 num = m.group(1)
1024                 default = "2of2 account %s"%num
1025             name = self.labels.get(k, default)
1026
1027         return name
1028
1029     def get_account_names(self):
1030         accounts = {}
1031         for k, account in self.accounts.items():
1032             accounts[k] = self.get_account_name(k)
1033         if self.imported_keys:
1034             accounts[-1] = 'Imported keys'
1035         return accounts
1036
1037     def get_account_addresses(self, a, include_change=True):
1038         if a is None:
1039             o = self.addresses(True)
1040         elif a == -1:
1041             o = self.imported_keys.keys()
1042         else:
1043             ac = self.accounts[a]
1044             o = ac.get_addresses(0)
1045             if include_change: o += ac.get_addresses(1)
1046         return o
1047
1048     def get_imported_balance(self):
1049         return self.get_balance(self.imported_keys.keys())
1050
1051     def get_account_balance(self, account):
1052         return self.get_balance(self.get_account_addresses(account))
1053
1054     def get_frozen_balance(self):
1055         return self.get_balance(self.frozen_addresses)
1056
1057     def get_balance(self, domain=None):
1058         if domain is None: domain = self.addresses(True)
1059         cc = uu = 0
1060         for addr in domain:
1061             c, u = self.get_addr_balance(addr)
1062             cc += c
1063             uu += u
1064         return cc, uu
1065
1066
1067     def get_unspent_coins(self, domain=None):
1068         coins = []
1069         if domain is None: domain = self.addresses(True)
1070         for addr in domain:
1071             h = self.history.get(addr, [])
1072             if h == ['*']: continue
1073             for tx_hash, tx_height in h:
1074                 tx = self.transactions.get(tx_hash)
1075                 if tx is None: raise BaseException("Wallet not synchronized")
1076                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1077                 for output in tx.d.get('outputs'):
1078                     if output.get('address') != addr: continue
1079                     key = tx_hash + ":%d" % output.get('prevout_n')
1080                     if key in self.spent_outputs: continue
1081                     output['prevout_hash'] = tx_hash
1082                     output['height'] = tx_height
1083                     output['coinbase'] = is_coinbase
1084                     coins.append((tx_height, output))
1085
1086         # sort by age
1087         if coins:
1088             coins = sorted(coins)
1089             if coins[-1][0] != 0:
1090                 while coins[0][0] == 0:
1091                     coins = coins[1:] + [ coins[0] ]
1092         return [x[1] for x in coins]
1093
1094
1095
1096     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1097         domain = self.get_account_addresses(account) if account else None
1098         return self.choose_tx_inputs( amount, fixed_fee, domain )
1099
1100
1101     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1102         """ todo: minimize tx size """
1103         total = 0
1104         fee = self.fee if fixed_fee is None else fixed_fee
1105         if domain is None:
1106             domain = self.addresses(True)
1107
1108         for i in self.frozen_addresses:
1109             if i in domain: domain.remove(i)
1110
1111         prioritized = []
1112         for i in self.prioritized_addresses:
1113             if i in domain:
1114                 domain.remove(i)
1115                 prioritized.append(i)
1116
1117         coins = self.get_unspent_coins(domain)
1118         prioritized_coins = self.get_unspent_coins(prioritized)
1119
1120         inputs = []
1121         coins = prioritized_coins + coins
1122
1123         for item in coins:
1124             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1125                 continue
1126             addr = item.get('address')
1127             v = item.get('value')
1128             total += v
1129             inputs.append(item)
1130             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1131             if total >= amount + fee: break
1132         else:
1133             inputs = []
1134
1135         return inputs, total, fee
1136
1137
1138     def set_fee(self, fee):
1139         if self.fee != fee:
1140             self.fee = fee
1141             self.storage.put('fee_per_kb', self.fee, True)
1142
1143     def estimated_fee(self, inputs):
1144         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1145         fee = self.fee * int(round(estimated_size/1024.))
1146         if fee == 0: fee = self.fee
1147         return fee
1148
1149
1150     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1151         "add change to a transaction"
1152         change_amount = total - ( amount + fee )
1153         if change_amount != 0:
1154             if not change_addr:
1155
1156                 # send change to one of the accounts involved in the tx
1157                 address = inputs[0].get('address')
1158                 account, _ = self.get_address_index(address)
1159
1160                 if not self.use_change or account == -1:
1161                     change_addr = inputs[-1]['address']
1162                 else:
1163                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1164
1165             # Insert the change output at a random position in the outputs
1166             posn = random.randint(0, len(outputs))
1167             outputs[posn:posn] = [( change_addr,  change_amount)]
1168         return outputs
1169
1170
1171     def get_history(self, address):
1172         with self.lock:
1173             return self.history.get(address)
1174
1175
1176     def get_status(self, h):
1177         if not h: return None
1178         if h == ['*']: return '*'
1179         status = ''
1180         for tx_hash, height in h:
1181             status += tx_hash + ':%d:' % height
1182         return hashlib.sha256( status ).digest().encode('hex')
1183
1184
1185     def receive_tx_callback(self, tx_hash, tx, tx_height):
1186
1187         with self.transaction_lock:
1188             self.add_extra_addresses(tx)
1189             if not self.check_new_tx(tx_hash, tx):
1190                 # may happen due to pruning
1191                 print_error("received transaction that is no longer referenced in history", tx_hash)
1192                 return
1193             self.transactions[tx_hash] = tx
1194             self.network.interface.pending_transactions_for_notifications.append(tx)
1195             self.save_transactions()
1196             if self.verifier and tx_height>0:
1197                 self.verifier.add(tx_hash, tx_height)
1198             self.update_tx_outputs(tx_hash)
1199
1200
1201     def save_transactions(self):
1202         tx = {}
1203         for k,v in self.transactions.items():
1204             tx[k] = str(v)
1205         self.storage.put('transactions', tx, True)
1206
1207     def receive_history_callback(self, addr, hist):
1208
1209         if not self.check_new_history(addr, hist):
1210             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1211
1212         with self.lock:
1213             self.history[addr] = hist
1214             self.storage.put('addr_history', self.history, True)
1215
1216         if hist != ['*']:
1217             for tx_hash, tx_height in hist:
1218                 if tx_height>0:
1219                     # add it in case it was previously unconfirmed
1220                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1221
1222
1223     def get_tx_history(self, account=None):
1224         with self.transaction_lock:
1225             history = self.transactions.items()
1226             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1227             result = []
1228
1229             balance = 0
1230             for tx_hash, tx in history:
1231                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1232                 if v is not None: balance += v
1233
1234             c, u = self.get_account_balance(account)
1235
1236             if balance != c+u:
1237                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1238
1239             balance = c + u - balance
1240             for tx_hash, tx in history:
1241                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1242                 if not is_relevant:
1243                     continue
1244                 if value is not None:
1245                     balance += value
1246
1247                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1248                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1249
1250         return result
1251
1252
1253     def get_label(self, tx_hash):
1254         label = self.labels.get(tx_hash)
1255         is_default = (label == '') or (label is None)
1256         if is_default: label = self.get_default_label(tx_hash)
1257         return label, is_default
1258
1259
1260     def get_default_label(self, tx_hash):
1261         tx = self.transactions.get(tx_hash)
1262         default_label = ''
1263         if tx:
1264             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1265             if is_mine:
1266                 for o in tx.outputs:
1267                     o_addr, _ = o
1268                     if not self.is_mine(o_addr):
1269                         try:
1270                             default_label = self.labels[o_addr]
1271                         except KeyError:
1272                             default_label = o_addr
1273                         break
1274                 else:
1275                     default_label = '(internal)'
1276             else:
1277                 for o in tx.outputs:
1278                     o_addr, _ = o
1279                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1280                         break
1281                 else:
1282                     for o in tx.outputs:
1283                         o_addr, _ = o
1284                         if self.is_mine(o_addr):
1285                             break
1286                     else:
1287                         o_addr = None
1288
1289                 if o_addr:
1290                     dest_label = self.labels.get(o_addr)
1291                     try:
1292                         default_label = self.labels[o_addr]
1293                     except KeyError:
1294                         default_label = o_addr
1295
1296         return default_label
1297
1298
1299     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1300         for address, x in outputs:
1301             assert is_valid(address)
1302         amount = sum( map(lambda x:x[1], outputs) )
1303         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1304         if not inputs:
1305             raise ValueError("Not enough funds")
1306         self.add_input_info(inputs)
1307         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1308         return Transaction.from_io(inputs, outputs)
1309
1310
1311     def mktx_from_account(self, outputs, password, fee=None, account=None):
1312         domain = self.get_account_addresses(account) if account else None
1313         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1314
1315
1316     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1317         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1318         keypairs = {}
1319         self.add_keypairs_from_wallet(tx, keypairs, password)
1320         if keypairs:
1321             self.sign_transaction(tx, keypairs)
1322         return tx
1323
1324
1325     def add_input_info(self, inputs):
1326         for txin in inputs:
1327             address = txin['address']
1328             account, sequence = self.get_address_index(address)
1329             txin['KeyID'] = self.get_keyID(account, sequence)
1330             redeemScript = self.accounts[account].redeem_script(sequence)
1331             if redeemScript:
1332                 txin['redeemScript'] = redeemScript
1333             else:
1334                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1335
1336
1337     def sign_transaction(self, tx, keypairs):
1338         tx.sign(keypairs)
1339         run_hook('sign_transaction', tx)
1340
1341
1342     def sendtx(self, tx):
1343         # synchronous
1344         h = self.send_tx(tx)
1345         self.tx_event.wait()
1346         return self.receive_tx(h)
1347
1348     def send_tx(self, tx):
1349         # asynchronous
1350         self.tx_event.clear()
1351         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1352         return tx.hash()
1353
1354     def on_broadcast(self, i, r):
1355         self.tx_result = r.get('result')
1356         self.tx_event.set()
1357
1358     def receive_tx(self,tx_hash):
1359         out = self.tx_result
1360         if out != tx_hash:
1361             return False, "error: " + out
1362         return True, out
1363
1364
1365
1366     def update_password(self, old_password, new_password):
1367         if new_password == '': new_password = None
1368         # this will throw an exception if unicode cannot be converted
1369         decoded = pw_decode(self.seed, old_password)
1370         self.seed = pw_encode( decoded, new_password)
1371         self.storage.put('seed', self.seed, True)
1372         self.use_encryption = (new_password != None)
1373         self.storage.put('use_encryption', self.use_encryption,True)
1374         for k in self.imported_keys.keys():
1375             a = self.imported_keys[k]
1376             b = pw_decode(a, old_password)
1377             c = pw_encode(b, new_password)
1378             self.imported_keys[k] = c
1379         self.storage.put('imported_keys', self.imported_keys, True)
1380
1381         for k, v in self.master_private_keys.items():
1382             b = pw_decode(v, old_password)
1383             c = pw_encode(b, new_password)
1384             self.master_private_keys[k] = c
1385         self.storage.put('master_private_keys', self.master_private_keys, True)
1386
1387
1388     def freeze(self,addr):
1389         if self.is_mine(addr) and addr not in self.frozen_addresses:
1390             self.unprioritize(addr)
1391             self.frozen_addresses.append(addr)
1392             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1393             return True
1394         else:
1395             return False
1396
1397     def unfreeze(self,addr):
1398         if self.is_mine(addr) and addr in self.frozen_addresses:
1399             self.frozen_addresses.remove(addr)
1400             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1401             return True
1402         else:
1403             return False
1404
1405     def prioritize(self,addr):
1406         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1407             self.unfreeze(addr)
1408             self.prioritized_addresses.append(addr)
1409             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1410             return True
1411         else:
1412             return False
1413
1414     def unprioritize(self,addr):
1415         if self.is_mine(addr) and addr in self.prioritized_addresses:
1416             self.prioritized_addresses.remove(addr)
1417             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1418             return True
1419         else:
1420             return False
1421
1422
1423     def set_verifier(self, verifier):
1424         self.verifier = verifier
1425
1426         # review transactions that are in the history
1427         for addr, hist in self.history.items():
1428             if hist == ['*']: continue
1429             for tx_hash, tx_height in hist:
1430                 if tx_height>0:
1431                     # add it in case it was previously unconfirmed
1432                     self.verifier.add(tx_hash, tx_height)
1433
1434
1435         # if we are on a pruning server, remove unverified transactions
1436         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1437         for tx_hash in self.transactions.keys():
1438             if tx_hash not in vr:
1439                 self.transactions.pop(tx_hash)
1440
1441
1442
1443     def check_new_history(self, addr, hist):
1444
1445         # check that all tx in hist are relevant
1446         if hist != ['*']:
1447             for tx_hash, height in hist:
1448                 tx = self.transactions.get(tx_hash)
1449                 if not tx: continue
1450                 if not tx.has_address(addr):
1451                     return False
1452
1453         # check that we are not "orphaning" a transaction
1454         old_hist = self.history.get(addr,[])
1455         if old_hist == ['*']: return True
1456
1457         for tx_hash, height in old_hist:
1458             if tx_hash in map(lambda x:x[0], hist): continue
1459             found = False
1460             for _addr, _hist in self.history.items():
1461                 if _addr == addr: continue
1462                 if _hist == ['*']: continue
1463                 _tx_hist = map(lambda x:x[0], _hist)
1464                 if tx_hash in _tx_hist:
1465                     found = True
1466                     break
1467
1468             if not found:
1469                 tx = self.transactions.get(tx_hash)
1470                 # tx might not be there
1471                 if not tx: continue
1472
1473                 # already verified?
1474                 if self.verifier.get_height(tx_hash):
1475                     continue
1476                 # unconfirmed tx
1477                 print_error("new history is orphaning transaction:", tx_hash)
1478                 # check that all outputs are not mine, request histories
1479                 ext_requests = []
1480                 for _addr, _v in tx.outputs:
1481                     # assert not self.is_mine(_addr)
1482                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1483
1484                 ext_h = self.network.synchronous_get(ext_requests)
1485                 print_error("sync:", ext_requests, ext_h)
1486                 height = None
1487                 for h in ext_h:
1488                     if h == ['*']: continue
1489                     for item in h:
1490                         if item.get('tx_hash') == tx_hash:
1491                             height = item.get('height')
1492                 if height:
1493                     print_error("found height for", tx_hash, height)
1494                     self.verifier.add(tx_hash, height)
1495                 else:
1496                     print_error("removing orphaned tx from history", tx_hash)
1497                     self.transactions.pop(tx_hash)
1498
1499         return True
1500
1501
1502
1503     def check_new_tx(self, tx_hash, tx):
1504         # 1 check that tx is referenced in addr_history.
1505         addresses = []
1506         for addr, hist in self.history.items():
1507             if hist == ['*']:continue
1508             for txh, height in hist:
1509                 if txh == tx_hash:
1510                     addresses.append(addr)
1511
1512         if not addresses:
1513             return False
1514
1515         # 2 check that referencing addresses are in the tx
1516         for addr in addresses:
1517             if not tx.has_address(addr):
1518                 return False
1519
1520         return True
1521
1522
1523     def start_threads(self, network):
1524         from verifier import TxVerifier
1525         self.network = network
1526         self.verifier = TxVerifier(self.network, self.storage)
1527         self.verifier.start()
1528         self.set_verifier(self.verifier)
1529         self.synchronizer = WalletSynchronizer(self, network)
1530         self.synchronizer.start()
1531
1532     def stop_threads(self):
1533         self.verifier.stop()
1534         self.synchronizer.stop()
1535
1536
1537
1538     def restore(self, callback):
1539         from i18n import _
1540         def wait_for_wallet():
1541             self.set_up_to_date(False)
1542             while not self.is_up_to_date():
1543                 msg = "%s\n%s %d\n%s %.1f"%(
1544                     _("Please wait..."),
1545                     _("Addresses generated:"),
1546                     len(self.addresses(True)),_("Kilobytes received:"),
1547                     self.network.interface.bytes_received/1024.)
1548
1549                 apply(callback, (msg,))
1550                 time.sleep(0.1)
1551
1552         def wait_for_network():
1553             while not self.network.interface.is_connected:
1554                 msg = "%s \n" % (_("Connecting..."))
1555                 apply(callback, (msg,))
1556                 time.sleep(0.1)
1557
1558         # wait until we are connected, because the user might have selected another server
1559         wait_for_network()
1560
1561
1562         self.create_accounts()
1563         wait_for_wallet()
1564
1565
1566
1567
1568 class WalletSynchronizer(threading.Thread):
1569
1570
1571     def __init__(self, wallet, network):
1572         threading.Thread.__init__(self)
1573         self.daemon = True
1574         self.wallet = wallet
1575         self.network = network
1576         self.was_updated = True
1577         self.running = False
1578         self.lock = threading.Lock()
1579         self.queue = Queue.Queue()
1580
1581     def stop(self):
1582         with self.lock: self.running = False
1583
1584     def is_running(self):
1585         with self.lock: return self.running
1586
1587
1588     def subscribe_to_addresses(self, addresses):
1589         messages = []
1590         for addr in addresses:
1591             messages.append(('blockchain.address.subscribe', [addr]))
1592         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1593
1594
1595     def run(self):
1596         with self.lock:
1597             self.running = True
1598
1599         while self.is_running():
1600
1601             if not self.network.is_connected():
1602                 print_error("synchronizer: waiting for interface")
1603                 self.network.wait_until_connected()
1604
1605             self.run_interface(self.network.interface)
1606
1607
1608     def run_interface(self, interface):
1609
1610         print_error("synchronizer: connected to", interface.server)
1611
1612         requested_tx = []
1613         missing_tx = []
1614         requested_histories = {}
1615
1616         # request any missing transactions
1617         for history in self.wallet.history.values():
1618             if history == ['*']: continue
1619             for tx_hash, tx_height in history:
1620                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1621                     missing_tx.append( (tx_hash, tx_height) )
1622
1623         if missing_tx:
1624             print_error("missing tx", missing_tx)
1625
1626         # subscriptions
1627         self.subscribe_to_addresses(self.wallet.addresses(True))
1628
1629         while self.is_running():
1630             # 1. create new addresses
1631             new_addresses = self.wallet.synchronize()
1632
1633             # request missing addresses
1634             if new_addresses:
1635                 self.subscribe_to_addresses(new_addresses)
1636
1637             # request missing transactions
1638             for tx_hash, tx_height in missing_tx:
1639                 if (tx_hash, tx_height) not in requested_tx:
1640                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1641                     requested_tx.append( (tx_hash, tx_height) )
1642             missing_tx = []
1643
1644             # detect if situation has changed
1645             if interface.is_up_to_date() and self.queue.empty():
1646                 if not self.wallet.is_up_to_date():
1647                     self.wallet.set_up_to_date(True)
1648                     self.was_updated = True
1649             else:
1650                 if self.wallet.is_up_to_date():
1651                     self.wallet.set_up_to_date(False)
1652                     self.was_updated = True
1653
1654             if self.was_updated:
1655                 self.wallet.network.trigger_callback('updated')
1656                 self.was_updated = False
1657
1658             # 2. get a response
1659             try:
1660                 r = self.queue.get(block=True, timeout=1)
1661             except Queue.Empty:
1662                 continue
1663
1664             if interface != self.network.interface:
1665                 break
1666
1667             if not r:
1668                 continue
1669
1670             # 3. handle response
1671             method = r['method']
1672             params = r['params']
1673             result = r.get('result')
1674             error = r.get('error')
1675             if error:
1676                 print "error", r
1677                 continue
1678
1679             if method == 'blockchain.address.subscribe':
1680                 addr = params[0]
1681                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1682                     if requested_histories.get(addr) is None:
1683                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1684                         requested_histories[addr] = result
1685
1686             elif method == 'blockchain.address.get_history':
1687                 addr = params[0]
1688                 print_error("receiving history", addr, result)
1689                 if result == ['*']:
1690                     assert requested_histories.pop(addr) == '*'
1691                     self.wallet.receive_history_callback(addr, result)
1692                 else:
1693                     hist = []
1694                     # check that txids are unique
1695                     txids = []
1696                     for item in result:
1697                         tx_hash = item['tx_hash']
1698                         if tx_hash not in txids:
1699                             txids.append(tx_hash)
1700                             hist.append( (tx_hash, item['height']) )
1701
1702                     if len(hist) != len(result):
1703                         raise BaseException("error: server sent history with non-unique txid", result)
1704
1705                     # check that the status corresponds to what was announced
1706                     rs = requested_histories.pop(addr)
1707                     if self.wallet.get_status(hist) != rs:
1708                         raise BaseException("error: status mismatch: %s"%addr)
1709
1710                     # store received history
1711                     self.wallet.receive_history_callback(addr, hist)
1712
1713                     # request transactions that we don't have
1714                     for tx_hash, tx_height in hist:
1715                         if self.wallet.transactions.get(tx_hash) is None:
1716                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1717                                 missing_tx.append( (tx_hash, tx_height) )
1718
1719             elif method == 'blockchain.transaction.get':
1720                 tx_hash = params[0]
1721                 tx_height = params[1]
1722                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1723                 tx = Transaction(result)
1724                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1725                 self.was_updated = True
1726                 requested_tx.remove( (tx_hash, tx_height) )
1727                 print_error("received tx:", tx_hash, len(tx.raw))
1728
1729             else:
1730                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1731
1732             if self.was_updated and not requested_tx:
1733                 self.wallet.network.trigger_callback('updated')
1734                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1735
1736                 self.was_updated = False