lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32 import math
  33
  34 from util import print_msg, print_error, format_satoshis
  35 from bitcoin import *
  36 from account import *
  37 from transaction import Transaction
  38 from plugins import run_hook
  39
  40 COINBASE_MATURITY = 100
  41 DUST_THRESHOLD = 5430
  42
  43 # AES encryption
  44 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  45 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  46
  47 def pw_encode(s, password):
  48     if password:
  49         secret = Hash(password)
  50         return EncodeAES(secret, s)
  51     else:
  52         return s
  53
  54 def pw_decode(s, password):
  55     if password is not None:
  56         secret = Hash(password)
  57         try:
  58             d = DecodeAES(secret, s)
  59         except Exception:
  60             raise Exception('Invalid password')
  61         return d
  62     else:
  63         return s
  64
  65
  66
  67
  68
  69 from version import *
  70
  71
  72 class WalletStorage:
  73
  74     def __init__(self, config):
  75         self.lock = threading.Lock()
  76         self.data = {}
  77         self.file_exists = False
  78         self.path = self.init_path(config)
  79         print_error( "wallet path", self.path )
  80         if self.path:
  81             self.read(self.path)
  82
  83
  84     def init_path(self, config):
  85         """Set the path of the wallet."""
  86
  87         # command line -w option
  88         path = config.get('wallet_path')
  89         if path:
  90             return path
  91
  92         # path in config file
  93         path = config.get('default_wallet_path')
  94         if path:
  95             return path
  96
  97         # default path
  98         dirpath = os.path.join(config.path, "wallets")
  99         if not os.path.exists(dirpath):
 100             os.mkdir(dirpath)
 101
 102         new_path = os.path.join(config.path, "wallets", "default_wallet")
 103
 104         # default path in pre 1.9 versions
 105         old_path = os.path.join(config.path, "electrum.dat")
 106         if os.path.exists(old_path) and not os.path.exists(new_path):
 107             os.rename(old_path, new_path)
 108
 109         return new_path
 110
 111
 112     def read(self, path):
 113         """Read the contents of the wallet file."""
 114         try:
 115             with open(self.path, "r") as f:
 116                 data = f.read()
 117         except IOError:
 118             return
 119         try:
 120             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 121         except Exception:
 122             raise IOError("Cannot read wallet file.")
 123
 124         self.data = d
 125         self.file_exists = True
 126
 127
 128     def get(self, key, default=None):
 129         v = self.data.get(key)
 130         if v is None:
 131             v = default
 132         return v
 133
 134     def put(self, key, value, save = True):
 135
 136         with self.lock:
 137             if value is not None:
 138                 self.data[key] = value
 139             else:
 140                 self.data.pop(key)
 141             if save:
 142                 self.write()
 143
 144     def write(self):
 145         s = repr(self.data)
 146         f = open(self.path,"w")
 147         f.write( s )
 148         f.close()
 149         if 'ANDROID_DATA' not in os.environ:
 150             import stat
 151             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 152
 153
 154 class Wallet:
 155
 156     def __init__(self, storage):
 157
 158         self.storage = storage
 159         self.electrum_version = ELECTRUM_VERSION
 160         self.gap_limit_for_change = 3 # constant
 161
 162         # saved fields
 163         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 164
 165         self.gap_limit             = storage.get('gap_limit', 5)
 166         self.use_change            = storage.get('use_change',True)
 167         self.use_encryption        = storage.get('use_encryption', False)
 168         self.seed                  = storage.get('seed', '')               # encrypted
 169         self.labels                = storage.get('labels', {})
 170         self.frozen_addresses      = storage.get('frozen_addresses',[])
 171         self.addressbook           = storage.get('contacts', [])
 172
 173         self.imported_keys         = storage.get('imported_keys',{})
 174         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 175
 176         self.fee                   = int(storage.get('fee_per_kb',20000))
 177
 178         self.master_public_keys = storage.get('master_public_keys',{})
 179         self.master_private_keys = storage.get('master_private_keys', {})
 180
 181         self.next_addresses = storage.get('next_addresses',{})
 182
 183         if self.seed_version not in [4, 6]:
 184             msg = "This wallet seed is not supported."
 185             if self.seed_version in [5]:
 186                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 187                 print msg
 188                 sys.exit(1)
 189
 190         # This attribute is set when wallet.start_threads is called.
 191         self.synchronizer = None
 192
 193         self.load_accounts()
 194
 195         self.transactions = {}
 196         tx_list = self.storage.get('transactions',{})
 197         for k,v in tx_list.items():
 198             try:
 199                 tx = Transaction(v)
 200             except Exception:
 201                 print_msg("Warning: Cannot deserialize transactions. skipping")
 202                 continue
 203
 204             self.add_extra_addresses(tx)
 205             self.transactions[k] = tx
 206
 207         for h,tx in self.transactions.items():
 208             if not self.check_new_tx(h, tx):
 209                 print_error("removing unreferenced tx", h)
 210                 self.transactions.pop(h)
 211
 212
 213         # not saved
 214         self.prevout_values = {}     # my own transaction outputs
 215         self.spent_outputs = []
 216
 217         # spv
 218         self.verifier = None
 219
 220         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 221         # interface.is_up_to_date() returns true when all requests have been answered and processed
 222         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 223
 224         self.up_to_date = False
 225         self.lock = threading.Lock()
 226         self.transaction_lock = threading.Lock()
 227         self.tx_event = threading.Event()
 228
 229         for tx_hash, tx in self.transactions.items():
 230             self.update_tx_outputs(tx_hash)
 231
 232
 233     def add_extra_addresses(self, tx):
 234         h = tx.hash()
 235         # find the address corresponding to pay-to-pubkey inputs
 236         tx.add_extra_addresses(self.transactions)
 237         for o in tx.d.get('outputs'):
 238             if o.get('is_pubkey'):
 239                 for tx2 in self.transactions.values():
 240                     tx2.add_extra_addresses({h:tx})
 241
 242
 243
 244
 245     def set_up_to_date(self,b):
 246         with self.lock: self.up_to_date = b
 247
 248
 249     def is_up_to_date(self):
 250         with self.lock: return self.up_to_date
 251
 252
 253     def update(self):
 254         self.up_to_date = False
 255         while not self.is_up_to_date():
 256             time.sleep(0.1)
 257
 258
 259     def import_key(self, sec, password):
 260         # check password
 261         seed = self.get_seed(password)
 262         try:
 263             address = address_from_private_key(sec)
 264         except Exception:
 265             raise Exception('Invalid private key')
 266
 267         if self.is_mine(address):
 268             raise Exception('Address already in wallet')
 269
 270         # store the originally requested keypair into the imported keys table
 271         self.imported_keys[address] = pw_encode(sec, password )
 272         self.storage.put('imported_keys', self.imported_keys, True)
 273         if self.synchronizer:
 274             self.synchronizer.subscribe_to_addresses([address])
 275         return address
 276
 277     def delete_imported_key(self, addr):
 278         if addr in self.imported_keys:
 279             self.imported_keys.pop(addr)
 280             self.storage.put('imported_keys', self.imported_keys, True)
 281
 282
 283     def make_seed(self):
 284         import mnemonic, ecdsa
 285         entropy = ecdsa.util.randrange( pow(2,160) )
 286         nonce = 0
 287         while True:
 288             ss = "%040x"%(entropy+nonce)
 289             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 290             # we keep only 13 words, that's approximately 139 bits of entropy
 291             words = mnemonic.mn_encode(s)[0:13]
 292             seed = ' '.join(words)
 293             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 294                 break  # this removes 12 bits of entropy
 295             nonce += 1
 296
 297         return seed
 298
 299
 300     def init_seed(self, seed):
 301         import mnemonic
 302
 303         if self.seed:
 304             raise Exception("a seed exists")
 305
 306         if not seed:
 307             self.seed = random_seed(128)
 308             self.seed_version = 4
 309             return
 310
 311         #if not seed:
 312         #    self.seed = self.make_seed()
 313         #    self.seed_version = SEED_VERSION
 314         #    return
 315
 316         # find out what kind of wallet we are
 317         try:
 318             seed.strip().decode('hex')
 319             self.seed_version = 4
 320             self.seed = str(seed)
 321             return
 322         except Exception:
 323             pass
 324
 325         words = seed.split()
 326         self.seed_version = 4
 327         self.seed = mnemonic.mn_decode(words)
 328
 329         if not self.seed:
 330             raise Exception("Invalid seed")
 331
 332
 333
 334     def save_seed(self, password):
 335         if password:
 336             self.seed = pw_encode( self.seed, password)
 337             self.use_encryption = True
 338         self.storage.put('seed', self.seed, True)
 339         self.storage.put('seed_version', self.seed_version, True)
 340         self.storage.put('use_encryption', self.use_encryption,True)
 341         self.create_accounts(password)
 342
 343
 344     def create_watching_only_wallet(self, params):
 345         K0, c0 = params
 346         if not K0:
 347             return
 348
 349         if not c0:
 350             self.seed_version = 4
 351             self.storage.put('seed_version', self.seed_version, True)
 352             self.create_old_account(K0)
 353             return
 354
 355         cK0 = ""
 356         self.master_public_keys = {
 357             "m/0'/": (c0, K0, cK0),
 358             }
 359         self.storage.put('master_public_keys', self.master_public_keys, True)
 360         self.storage.put('seed_version', self.seed_version, True)
 361         self.create_account('1','Main account')
 362
 363
 364     def create_accounts(self, password):
 365         seed = pw_decode(self.seed, password)
 366
 367         if self.seed_version == 4:
 368             mpk = OldAccount.mpk_from_seed(seed)
 369             self.create_old_account(mpk)
 370         else:
 371             # create default account
 372             self.create_master_keys('1', password)
 373             self.create_account('1','Main account')
 374
 375
 376     def create_master_keys(self, account_type, password):
 377         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 378         if account_type == '1':
 379             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 380             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 381             self.master_private_keys["m/0'/"] = pw_encode(k0, password)
 382         elif account_type == '2of2':
 383             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 384             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 385             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 386             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 387             self.master_private_keys["m/1'/"] = pw_encode(k1, password)
 388             self.master_private_keys["m/2'/"] = pw_encode(k2, password)
 389         elif account_type == '2of3':
 390             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 391             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 392             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 393             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 394             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 395             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 396             self.master_private_keys["m/3'/"] = pw_encode(k3, password)
 397             self.master_private_keys["m/4'/"] = pw_encode(k4, password)
 398             self.master_private_keys["m/5'/"] = pw_encode(k5, password)
 399
 400         self.storage.put('master_public_keys', self.master_public_keys, True)
 401         self.storage.put('master_private_keys', self.master_private_keys, True)
 402
 403     def has_master_public_keys(self, account_type):
 404         if account_type == '1':
 405             return "m/0'/" in self.master_public_keys
 406         elif account_type == '2of2':
 407             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 408         elif account_type == '2of3':
 409             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 410
 411     def find_root_by_master_key(self, c, K):
 412         for key, v in self.master_public_keys.items():
 413             if key == "m/":continue
 414             cc, KK, _ = v
 415             if (c == cc) and (K == KK):
 416                 return key
 417
 418     def deseed_root(self, seed, password):
 419         # for safety, we ask the user to enter their seed
 420         assert seed == self.get_seed(password)
 421         self.seed = ''
 422         self.storage.put('seed', '', True)
 423
 424
 425     def deseed_branch(self, k):
 426         # check that parent has no seed
 427         assert self.seed == ''
 428         self.master_private_keys.pop(k)
 429         self.storage.put('master_private_keys', self.master_private_keys, True)
 430
 431     def is_watching_only(self):
 432         return (self.seed == '') and (self.master_private_keys == {})
 433
 434
 435
 436     def account_id(self, account_type, i):
 437         if account_type == '1':
 438             return "m/0'/%d"%i
 439         elif account_type == '2of2':
 440             return "m/1'/%d & m/2'/%d"%(i,i)
 441         elif account_type == '2of3':
 442             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 443         else:
 444             raise Exception('unknown account type')
 445
 446
 447     def num_accounts(self, account_type):
 448         keys = self.accounts.keys()
 449         i = 0
 450         while True:
 451             account_id = self.account_id(account_type, i)
 452             if account_id not in keys: break
 453             i += 1
 454         return i
 455
 456
 457     def new_account_address(self, account_type = '1'):
 458         i = self.num_accounts(account_type)
 459         k = self.account_id(account_type,i)
 460
 461         addr = self.next_addresses.get(k)
 462         if not addr:
 463             account_id, account = self.next_account(account_type)
 464             addr = account.first_address()
 465             self.next_addresses[k] = addr
 466             self.storage.put('next_addresses',self.next_addresses)
 467
 468         return k, addr
 469
 470
 471     def next_account(self, account_type = '1'):
 472
 473         i = self.num_accounts(account_type)
 474         account_id = self.account_id(account_type,i)
 475
 476         if account_type is '1':
 477             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 478             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 479             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 480
 481         elif account_type == '2of2':
 482             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 483             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 484             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 485             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 486             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 487
 488         elif account_type == '2of3':
 489             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 490             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 491             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 492             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 493             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 494             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 495             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 496
 497         return account_id, account
 498
 499
 500     def set_label(self, name, text = None):
 501         changed = False
 502         old_text = self.labels.get(name)
 503         if text:
 504             if old_text != text:
 505                 self.labels[name] = text
 506                 changed = True
 507         else:
 508             if old_text:
 509                 self.labels.pop(name)
 510                 changed = True
 511
 512         if changed:
 513             self.storage.put('labels', self.labels, True)
 514
 515         run_hook('set_label', name, text, changed)
 516         return changed
 517
 518
 519
 520     def create_account(self, account_type = '1', name = None):
 521         k, account = self.next_account(account_type)
 522         if k in self.pending_accounts:
 523             self.pending_accounts.pop(k)
 524             self.storage.put('pending_accounts', self.pending_accounts)
 525
 526         self.accounts[k] = account
 527         self.save_accounts()
 528         if name:
 529             self.set_label(k, name)
 530
 531
 532     def create_old_account(self, mpk):
 533         self.storage.put('master_public_key', mpk, True)
 534         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 535         self.save_accounts()
 536
 537
 538     def save_accounts(self):
 539         d = {}
 540         for k, v in self.accounts.items():
 541             d[k] = v.dump()
 542         self.storage.put('accounts', d, True)
 543
 544
 545
 546     def load_accounts(self):
 547         d = self.storage.get('accounts', {})
 548         self.accounts = {}
 549         for k, v in d.items():
 550             if k == 0:
 551                 v['mpk'] = self.storage.get('master_public_key')
 552                 self.accounts[k] = OldAccount(v)
 553             elif '&' in k:
 554                 self.accounts[k] = BIP32_Account_2of2(v)
 555             else:
 556                 self.accounts[k] = BIP32_Account(v)
 557
 558         self.pending_accounts = self.storage.get('pending_accounts',{})
 559
 560
 561     def delete_pending_account(self, k):
 562         self.pending_accounts.pop(k)
 563         self.storage.put('pending_accounts', self.pending_accounts)
 564
 565     def account_is_pending(self, k):
 566         return k in self.pending_accounts
 567
 568     def create_pending_account(self, acct_type, name):
 569         k, addr = self.new_account_address(acct_type)
 570         self.set_label(k, name)
 571         self.pending_accounts[k] = addr
 572         self.storage.put('pending_accounts', self.pending_accounts)
 573
 574     def get_pending_accounts(self):
 575         return self.pending_accounts.items()
 576
 577
 578     def addresses(self, include_change = True, _next=True):
 579         o = self.get_account_addresses(-1, include_change)
 580         for a in self.accounts.keys():
 581             o += self.get_account_addresses(a, include_change)
 582
 583         if _next:
 584             for addr in self.next_addresses.values():
 585                 if addr not in o:
 586                     o += [addr]
 587         return o
 588
 589
 590     def is_mine(self, address):
 591         return address in self.addresses(True)
 592
 593
 594     def is_change(self, address):
 595         if not self.is_mine(address): return False
 596         if address in self.imported_keys.keys(): return False
 597         acct, s = self.get_address_index(address)
 598         if s is None: return False
 599         return s[0] == 1
 600
 601     def get_master_public_key(self):
 602         if self.seed_version == 4:
 603             return self.storage.get("master_public_key")
 604         else:
 605             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 606             return repr((c, K))
 607
 608     def get_master_private_key(self, account, password):
 609         master_k = pw_decode( self.master_private_keys[account], password)
 610         master_c, master_K, master_Kc = self.master_public_keys[account]
 611         try:
 612             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 613             assert K.encode('hex') == master_K
 614         except Exception:
 615             raise Exception("Invalid password")
 616         return master_k
 617
 618
 619     def get_address_index(self, address):
 620         if address in self.imported_keys.keys():
 621             return -1, None
 622
 623         for account in self.accounts.keys():
 624             for for_change in [0,1]:
 625                 addresses = self.accounts[account].get_addresses(for_change)
 626                 for addr in addresses:
 627                     if address == addr:
 628                         return account, (for_change, addresses.index(addr))
 629
 630         for k,v in self.next_addresses.items():
 631             if v == address:
 632                 return k, (0,0)
 633
 634         raise Exception("Address not found", address)
 635
 636
 637     def get_roots(self, account):
 638         roots = []
 639         for a in account.split('&'):
 640             s = a.strip()
 641             m = re.match("(m/\d+'/)(\d+)", s)
 642             roots.append( m.group(1) )
 643         return roots
 644
 645     def is_seeded(self, account):
 646         if type(account) is int:
 647             return self.seed is not None
 648
 649         for root in self.get_roots(account):
 650             if root not in self.master_private_keys.keys():
 651                 return False
 652         return True
 653
 654     def rebase_sequence(self, account, sequence):
 655         c, i = sequence
 656         dd = []
 657         for a in account.split('&'):
 658             s = a.strip()
 659             m = re.match("(m/\d+'/)(\d+)", s)
 660             root = m.group(1)
 661             num = int(m.group(2))
 662             dd.append( (root, [num,c,i] ) )
 663         return dd
 664
 665
 666     def get_keyID(self, account, sequence):
 667         if account == 0:
 668             a, b = sequence
 669             mpk = self.storage.get('master_public_key')
 670             return 'old(%s,%d,%d)'%(mpk,a,b)
 671
 672         rs = self.rebase_sequence(account, sequence)
 673         dd = []
 674         for root, public_sequence in rs:
 675             c, K, _ = self.master_public_keys[root]
 676             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 677             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 678         return '&'.join(dd)
 679
 680
 681
 682     def get_seed(self, password):
 683         s = pw_decode(self.seed, password)
 684         if self.seed_version == 4:
 685             seed = s
 686             self.accounts[0].check_seed(seed)
 687         else:
 688             seed = mnemonic_hash(s)
 689         return seed
 690
 691
 692     def get_mnemonic(self, password):
 693         import mnemonic
 694         s = pw_decode(self.seed, password)
 695         if self.seed_version == 4:
 696             return ' '.join(mnemonic.mn_encode(s))
 697         else:
 698             return s
 699
 700
 701
 702     def get_private_key(self, address, password):
 703         if self.is_watching_only():
 704             return []
 705
 706         # first check the provided password
 707         seed = self.get_seed(password)
 708
 709         out = []
 710         if address in self.imported_keys.keys():
 711             out.append( pw_decode( self.imported_keys[address], password ) )
 712         else:
 713             account, sequence = self.get_address_index(address)
 714             if account == 0:
 715                 pk = self.accounts[account].get_private_key(seed, sequence)
 716                 out.append(pk)
 717                 return out
 718
 719             # assert address == self.accounts[account].get_address(*sequence)
 720             rs = self.rebase_sequence( account, sequence)
 721             for root, public_sequence in rs:
 722
 723                 if root not in self.master_private_keys.keys(): continue
 724                 master_k = self.get_master_private_key(root, password)
 725                 master_c, _, _ = self.master_public_keys[root]
 726                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 727                 out.append(pk)
 728
 729         return out
 730
 731
 732     def add_keypairs_from_wallet(self, tx, keypairs, password):
 733         for txin in tx.inputs:
 734             address = txin['address']
 735             if not self.is_mine(address):
 736                 continue
 737             private_keys = self.get_private_key(address, password)
 738             for sec in private_keys:
 739                 pubkey = public_key_from_private_key(sec)
 740                 keypairs[ pubkey ] = sec
 741                 if address in self.imported_keys.keys():
 742                     txin['redeemPubkey'] = pubkey
 743
 744
 745     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 746         # first check the provided password
 747         seed = self.get_seed(password)
 748
 749         for txin in tx.inputs:
 750             keyid = txin.get('KeyID')
 751             if keyid:
 752
 753                 if self.seed_version == 4:
 754                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 755                     if not m: continue
 756                     mpk = m.group(1)
 757                     if mpk != self.storage.get('master_public_key'): continue
 758                     for_change = int(m.group(2))
 759                     num = int(m.group(3))
 760                     account = self.accounts[0]
 761                     addr = account.get_address(for_change, num)
 762                     txin['address'] = addr # fixme: side effect
 763                     pk = account.get_private_key(seed, (for_change, num))
 764                     pubkey = public_key_from_private_key(pk)
 765                     keypairs[pubkey] = pk
 766                     continue
 767
 768
 769                 roots = []
 770                 for s in keyid.split('&'):
 771                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 772                     if not m: continue
 773                     c = m.group(1)
 774                     K = m.group(2)
 775                     sequence = m.group(3)
 776                     root = self.find_root_by_master_key(c,K)
 777                     if not root: continue
 778                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 779                     root = root + '%d'%sequence[0]
 780                     sequence = sequence[1:]
 781                     roots.append((root,sequence))
 782
 783                 account_id = " & ".join( map(lambda x:x[0], roots) )
 784                 account = self.accounts.get(account_id)
 785                 if not account: continue
 786                 addr = account.get_address(*sequence)
 787                 txin['address'] = addr # fixme: side effect
 788                 pk = self.get_private_key(addr, password)
 789                 for sec in pk:
 790                     pubkey = public_key_from_private_key(sec)
 791                     keypairs[pubkey] = sec
 792
 793
 794
 795     def signrawtransaction(self, tx, input_info, private_keys, password):
 796
 797         # check that the password is correct
 798         seed = self.get_seed(password)
 799
 800         # add input info
 801         tx.add_input_info(input_info)
 802
 803         # add redeem script for coins that are in the wallet
 804         # FIXME: add redeemPubkey too!
 805         unspent_coins = self.get_unspent_coins()
 806         for txin in tx.inputs:
 807             for item in unspent_coins:
 808                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 809                     print_error( "tx input is in unspent coins" )
 810                     txin['scriptPubKey'] = item['scriptPubKey']
 811                     account, sequence = self.get_address_index(item['address'])
 812                     if account != -1:
 813                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 814                         print_error("added redeemScript", txin['redeemScript'])
 815                     break
 816
 817
 818         # build a list of public/private keys
 819         keypairs = {}
 820
 821         # add private keys from parameter
 822         for sec in private_keys:
 823             pubkey = public_key_from_private_key(sec)
 824             keypairs[ pubkey ] = sec
 825
 826         # add private_keys from KeyID
 827         self.add_keypairs_from_KeyID(tx, keypairs, password)
 828
 829         # add private keys from wallet
 830         self.add_keypairs_from_wallet(tx, keypairs, password)
 831         self.sign_transaction(tx, keypairs)
 832
 833
 834     def sign_message(self, address, message, password):
 835         keys = self.get_private_key(address, password)
 836         assert len(keys) == 1
 837         sec = keys[0]
 838         key = regenerate_key(sec)
 839         compressed = is_compressed(sec)
 840         return key.sign_message(message, compressed, address)
 841
 842
 843     def change_gap_limit(self, value):
 844         if value >= self.gap_limit:
 845             self.gap_limit = value
 846             self.storage.put('gap_limit', self.gap_limit, True)
 847             #self.interface.poke('synchronizer')
 848             return True
 849
 850         elif value >= self.min_acceptable_gap():
 851             for key, account in self.accounts.items():
 852                 addresses = account[0]
 853                 k = self.num_unused_trailing_addresses(addresses)
 854                 n = len(addresses) - k + value
 855                 addresses = addresses[0:n]
 856                 self.accounts[key][0] = addresses
 857
 858             self.gap_limit = value
 859             self.storage.put('gap_limit', self.gap_limit, True)
 860             self.save_accounts()
 861             return True
 862         else:
 863             return False
 864
 865     def num_unused_trailing_addresses(self, addresses):
 866         k = 0
 867         for a in addresses[::-1]:
 868             if self.history.get(a):break
 869             k = k + 1
 870         return k
 871
 872     def min_acceptable_gap(self):
 873         # fixme: this assumes wallet is synchronized
 874         n = 0
 875         nmax = 0
 876
 877         for account in self.accounts.values():
 878             addresses = account.get_addresses(0)
 879             k = self.num_unused_trailing_addresses(addresses)
 880             for a in addresses[0:-k]:
 881                 if self.history.get(a):
 882                     n = 0
 883                 else:
 884                     n += 1
 885                     if n > nmax: nmax = n
 886         return nmax + 1
 887
 888
 889     def address_is_old(self, address):
 890         age = -1
 891         h = self.history.get(address, [])
 892         if h == ['*']:
 893             return True
 894         for tx_hash, tx_height in h:
 895             if tx_height == 0:
 896                 tx_age = 0
 897             else:
 898                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 899             if tx_age > age:
 900                 age = tx_age
 901         return age > 2
 902
 903
 904     def synchronize_sequence(self, account, for_change):
 905         limit = self.gap_limit_for_change if for_change else self.gap_limit
 906         new_addresses = []
 907         while True:
 908             addresses = account.get_addresses(for_change)
 909             if len(addresses) < limit:
 910                 address = account.create_new_address(for_change)
 911                 self.history[address] = []
 912                 new_addresses.append( address )
 913                 continue
 914
 915             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 916                 break
 917             else:
 918                 address = account.create_new_address(for_change)
 919                 self.history[address] = []
 920                 new_addresses.append( address )
 921
 922         return new_addresses
 923
 924
 925
 926     def create_pending_accounts(self):
 927         for account_type in ['1','2of2','2of3']:
 928             if not self.has_master_public_keys(account_type):
 929                 continue
 930             k, a = self.new_account_address(account_type)
 931             if self.address_is_old(a):
 932                 print_error( "creating account", a )
 933                 self.create_account(account_type)
 934                 self.next_addresses.pop(k)
 935
 936
 937     def synchronize_account(self, account):
 938         new = []
 939         new += self.synchronize_sequence(account, 0)
 940         new += self.synchronize_sequence(account, 1)
 941         return new
 942
 943
 944     def synchronize(self):
 945         if self.master_public_keys:
 946             self.create_pending_accounts()
 947         new = []
 948         for account in self.accounts.values():
 949             new += self.synchronize_account(account)
 950         if new:
 951             self.save_accounts()
 952             self.storage.put('addr_history', self.history, True)
 953         return new
 954
 955
 956     def is_found(self):
 957         return self.history.values() != [[]] * len(self.history)
 958
 959
 960     def add_contact(self, address, label=None):
 961         self.addressbook.append(address)
 962         self.storage.put('contacts', self.addressbook, True)
 963         if label:
 964             self.set_label(address, label)
 965
 966
 967     def delete_contact(self, addr):
 968         if addr in self.addressbook:
 969             self.addressbook.remove(addr)
 970             self.storage.put('addressbook', self.addressbook, True)
 971
 972
 973     def fill_addressbook(self):
 974         for tx_hash, tx in self.transactions.items():
 975             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 976             if is_send:
 977                 for addr, v in tx.outputs:
 978                     if not self.is_mine(addr) and addr not in self.addressbook:
 979                         self.addressbook.append(addr)
 980         # redo labels
 981         # self.update_tx_labels()
 982
 983     def get_num_tx(self, address):
 984         n = 0
 985         for tx in self.transactions.values():
 986             if address in map(lambda x:x[0], tx.outputs): n += 1
 987         return n
 988
 989
 990     def get_address_flags(self, addr):
 991         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 992         flags += "F" if addr in self.frozen_addresses else "-"
 993         return flags
 994
 995
 996     def get_tx_value(self, tx, account=None):
 997         domain = self.get_account_addresses(account)
 998         return tx.get_value(domain, self.prevout_values)
 999
1000
1001     def update_tx_outputs(self, tx_hash):
1002         tx = self.transactions.get(tx_hash)
1003
1004         for i, (addr, value) in enumerate(tx.outputs):
1005             key = tx_hash+ ':%d'%i
1006             self.prevout_values[key] = value
1007
1008         for item in tx.inputs:
1009             if self.is_mine(item.get('address')):
1010                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1011                 self.spent_outputs.append(key)
1012
1013
1014     def get_addr_balance(self, address):
1015         assert self.is_mine(address)
1016         h = self.history.get(address,[])
1017         if h == ['*']: return 0,0
1018         c = u = 0
1019         received_coins = []   # list of coins received at address
1020
1021         for tx_hash, tx_height in h:
1022             tx = self.transactions.get(tx_hash)
1023             if not tx: continue
1024
1025             for i, (addr, value) in enumerate(tx.outputs):
1026                 if addr == address:
1027                     key = tx_hash + ':%d'%i
1028                     received_coins.append(key)
1029
1030         for tx_hash, tx_height in h:
1031             tx = self.transactions.get(tx_hash)
1032             if not tx: continue
1033             v = 0
1034
1035             for item in tx.inputs:
1036                 addr = item.get('address')
1037                 if addr == address:
1038                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1039                     value = self.prevout_values.get( key )
1040                     if key in received_coins:
1041                         v -= value
1042
1043             for i, (addr, value) in enumerate(tx.outputs):
1044                 key = tx_hash + ':%d'%i
1045                 if addr == address:
1046                     v += value
1047
1048             if tx_height:
1049                 c += v
1050             else:
1051                 u += v
1052         return c, u
1053
1054
1055     def get_account_name(self, k):
1056         if k == 0:
1057             if self.seed_version == 4:
1058                 name = 'Main account'
1059             else:
1060                 name = 'Old account'
1061         else:
1062             default = "Unnamed account"
1063             m = re.match("m/0'/(\d+)", k)
1064             if m:
1065                 num = m.group(1)
1066                 if num == '0':
1067                     default = "Main account"
1068                 else:
1069                     default = "Account %s"%num
1070
1071             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1072             if m:
1073                 num = m.group(1)
1074                 default = "2of2 account %s"%num
1075             name = self.labels.get(k, default)
1076
1077         return name
1078
1079     def get_account_names(self):
1080         accounts = {}
1081         for k, account in self.accounts.items():
1082             accounts[k] = self.get_account_name(k)
1083         if self.imported_keys:
1084             accounts[-1] = 'Imported keys'
1085         return accounts
1086
1087     def get_account_addresses(self, a, include_change=True):
1088         if a is None:
1089             o = self.addresses(True)
1090         elif a == -1:
1091             o = self.imported_keys.keys()
1092         else:
1093             ac = self.accounts[a]
1094             o = ac.get_addresses(0)
1095             if include_change: o += ac.get_addresses(1)
1096         return o
1097
1098     def get_imported_balance(self):
1099         return self.get_balance(self.imported_keys.keys())
1100
1101     def get_account_balance(self, account):
1102         return self.get_balance(self.get_account_addresses(account))
1103
1104     def get_frozen_balance(self):
1105         return self.get_balance(self.frozen_addresses)
1106
1107     def get_balance(self, domain=None):
1108         if domain is None: domain = self.addresses(True)
1109         cc = uu = 0
1110         for addr in domain:
1111             c, u = self.get_addr_balance(addr)
1112             cc += c
1113             uu += u
1114         return cc, uu
1115
1116
1117     def get_unspent_coins(self, domain=None):
1118         coins = []
1119         if domain is None: domain = self.addresses(True)
1120         for addr in domain:
1121             h = self.history.get(addr, [])
1122             if h == ['*']: continue
1123             for tx_hash, tx_height in h:
1124                 tx = self.transactions.get(tx_hash)
1125                 if tx is None: raise Exception("Wallet not synchronized")
1126                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1127                 for o in tx.d.get('outputs'):
1128                     output = o.copy()
1129                     if output.get('address') != addr: continue
1130                     key = tx_hash + ":%d" % output.get('prevout_n')
1131                     if key in self.spent_outputs: continue
1132                     output['prevout_hash'] = tx_hash
1133                     output['height'] = tx_height
1134                     output['coinbase'] = is_coinbase
1135                     coins.append((tx_height, output))
1136
1137         # sort by age
1138         if coins:
1139             coins = sorted(coins)
1140             if coins[-1][0] != 0:
1141                 while coins[0][0] == 0:
1142                     coins = coins[1:] + [ coins[0] ]
1143         return [x[1] for x in coins]
1144
1145
1146     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1147         """ todo: minimize tx size """
1148         total = 0
1149         fee = self.fee if fixed_fee is None else fixed_fee
1150         if domain is None:
1151             domain = self.addresses(True)
1152
1153         for i in self.frozen_addresses:
1154             if i in domain: domain.remove(i)
1155
1156         coins = self.get_unspent_coins(domain)
1157         inputs = []
1158
1159         for item in coins:
1160             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1161                 continue
1162             addr = item.get('address')
1163             v = item.get('value')
1164             total += v
1165             inputs.append(item)
1166             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1167             if total >= amount + fee: break
1168         else:
1169             inputs = []
1170
1171         return inputs, total, fee
1172
1173
1174     def set_fee(self, fee):
1175         if self.fee != fee:
1176             self.fee = fee
1177             self.storage.put('fee_per_kb', self.fee, True)
1178
1179     def estimated_fee(self, inputs):
1180         estimated_size =  len(inputs) * 180 + 80    # this assumes non-compressed keys
1181         fee = self.fee * int(math.ceil(estimated_size/1000.))
1182         return fee
1183
1184
1185     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1186         "add change to a transaction"
1187         change_amount = total - ( amount + fee )
1188         if change_amount > DUST_THRESHOLD:
1189             if not change_addr:
1190
1191                 # send change to one of the accounts involved in the tx
1192                 address = inputs[0].get('address')
1193                 account, _ = self.get_address_index(address)
1194
1195                 if not self.use_change or account == -1:
1196                     change_addr = inputs[-1]['address']
1197                 else:
1198                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1199
1200             # Insert the change output at a random position in the outputs
1201             posn = random.randint(0, len(outputs))
1202             outputs[posn:posn] = [( change_addr,  change_amount)]
1203         return outputs
1204
1205
1206     def get_history(self, address):
1207         with self.lock:
1208             return self.history.get(address)
1209
1210
1211     def get_status(self, h):
1212         if not h: return None
1213         if h == ['*']: return '*'
1214         status = ''
1215         for tx_hash, height in h:
1216             status += tx_hash + ':%d:' % height
1217         return hashlib.sha256( status ).digest().encode('hex')
1218
1219
1220     def receive_tx_callback(self, tx_hash, tx, tx_height):
1221
1222         with self.transaction_lock:
1223             self.add_extra_addresses(tx)
1224             if not self.check_new_tx(tx_hash, tx):
1225                 # may happen due to pruning
1226                 print_error("received transaction that is no longer referenced in history", tx_hash)
1227                 return
1228             self.transactions[tx_hash] = tx
1229             self.network.interface.pending_transactions_for_notifications.append(tx)
1230             self.save_transactions()
1231             if self.verifier and tx_height>0:
1232                 self.verifier.add(tx_hash, tx_height)
1233             self.update_tx_outputs(tx_hash)
1234
1235
1236     def save_transactions(self):
1237         tx = {}
1238         for k,v in self.transactions.items():
1239             tx[k] = str(v)
1240         self.storage.put('transactions', tx, True)
1241
1242     def receive_history_callback(self, addr, hist):
1243
1244         if not self.check_new_history(addr, hist):
1245             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1246
1247         with self.lock:
1248             self.history[addr] = hist
1249             self.storage.put('addr_history', self.history, True)
1250
1251         if hist != ['*']:
1252             for tx_hash, tx_height in hist:
1253                 if tx_height>0:
1254                     # add it in case it was previously unconfirmed
1255                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1256
1257
1258     def get_tx_history(self, account=None):
1259         if not self.verifier:
1260             return []
1261
1262         with self.transaction_lock:
1263             history = self.transactions.items()
1264             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1265             result = []
1266
1267             balance = 0
1268             for tx_hash, tx in history:
1269                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1270                 if v is not None: balance += v
1271
1272             c, u = self.get_account_balance(account)
1273
1274             if balance != c+u:
1275                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1276
1277             balance = c + u - balance
1278             for tx_hash, tx in history:
1279                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1280                 if not is_relevant:
1281                     continue
1282                 if value is not None:
1283                     balance += value
1284
1285                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1286                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1287
1288         return result
1289
1290
1291     def get_label(self, tx_hash):
1292         label = self.labels.get(tx_hash)
1293         is_default = (label == '') or (label is None)
1294         if is_default: label = self.get_default_label(tx_hash)
1295         return label, is_default
1296
1297
1298     def get_default_label(self, tx_hash):
1299         tx = self.transactions.get(tx_hash)
1300         default_label = ''
1301         if tx:
1302             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1303             if is_mine:
1304                 for o in tx.outputs:
1305                     o_addr, _ = o
1306                     if not self.is_mine(o_addr):
1307                         try:
1308                             default_label = self.labels[o_addr]
1309                         except KeyError:
1310                             default_label = o_addr
1311                         break
1312                 else:
1313                     default_label = '(internal)'
1314             else:
1315                 for o in tx.outputs:
1316                     o_addr, _ = o
1317                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1318                         break
1319                 else:
1320                     for o in tx.outputs:
1321                         o_addr, _ = o
1322                         if self.is_mine(o_addr):
1323                             break
1324                     else:
1325                         o_addr = None
1326
1327                 if o_addr:
1328                     dest_label = self.labels.get(o_addr)
1329                     try:
1330                         default_label = self.labels[o_addr]
1331                     except KeyError:
1332                         default_label = o_addr
1333
1334         return default_label
1335
1336
1337     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1338         for address, x in outputs:
1339             assert is_valid(address)
1340         amount = sum( map(lambda x:x[1], outputs) )
1341         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1342         if not inputs:
1343             raise ValueError("Not enough funds")
1344         self.add_input_info(inputs)
1345         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1346         return Transaction.from_io(inputs, outputs)
1347
1348
1349     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1350         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1351         keypairs = {}
1352         self.add_keypairs_from_wallet(tx, keypairs, password)
1353         if keypairs:
1354             self.sign_transaction(tx, keypairs)
1355         return tx
1356
1357
1358     def add_input_info(self, inputs):
1359         for txin in inputs:
1360             address = txin['address']
1361             if address in self.imported_keys.keys():
1362                 continue
1363             account, sequence = self.get_address_index(address)
1364             txin['KeyID'] = self.get_keyID(account, sequence)
1365             redeemScript = self.accounts[account].redeem_script(sequence)
1366             if redeemScript:
1367                 txin['redeemScript'] = redeemScript
1368             else:
1369                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1370
1371
1372     def sign_transaction(self, tx, keypairs):
1373         tx.sign(keypairs)
1374         run_hook('sign_transaction', tx)
1375
1376
1377     def sendtx(self, tx):
1378         # synchronous
1379         h = self.send_tx(tx)
1380         self.tx_event.wait()
1381         return self.receive_tx(h, tx)
1382
1383     def send_tx(self, tx):
1384         # asynchronous
1385         self.tx_event.clear()
1386         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1387         return tx.hash()
1388
1389     def on_broadcast(self, i, r):
1390         self.tx_result = r.get('result')
1391         self.tx_event.set()
1392
1393     def receive_tx(self, tx_hash, tx):
1394         out = self.tx_result
1395         if out != tx_hash:
1396             return False, "error: " + out
1397         run_hook('receive_tx', tx, self)
1398         return True, out
1399
1400
1401
1402     def update_password(self, old_password, new_password):
1403         if new_password == '': new_password = None
1404         decoded = self.get_seed(old_password)
1405         self.seed = pw_encode( decoded, new_password)
1406         self.storage.put('seed', self.seed, True)
1407         self.use_encryption = (new_password != None)
1408         self.storage.put('use_encryption', self.use_encryption,True)
1409         for k in self.imported_keys.keys():
1410             a = self.imported_keys[k]
1411             b = pw_decode(a, old_password)
1412             c = pw_encode(b, new_password)
1413             self.imported_keys[k] = c
1414         self.storage.put('imported_keys', self.imported_keys, True)
1415
1416         for k, v in self.master_private_keys.items():
1417             b = pw_decode(v, old_password)
1418             c = pw_encode(b, new_password)
1419             self.master_private_keys[k] = c
1420         self.storage.put('master_private_keys', self.master_private_keys, True)
1421
1422
1423     def freeze(self,addr):
1424         if self.is_mine(addr) and addr not in self.frozen_addresses:
1425             self.frozen_addresses.append(addr)
1426             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1427             return True
1428         else:
1429             return False
1430
1431     def unfreeze(self,addr):
1432         if self.is_mine(addr) and addr in self.frozen_addresses:
1433             self.frozen_addresses.remove(addr)
1434             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1435             return True
1436         else:
1437             return False
1438
1439
1440     def set_verifier(self, verifier):
1441         self.verifier = verifier
1442
1443         # review transactions that are in the history
1444         for addr, hist in self.history.items():
1445             if hist == ['*']: continue
1446             for tx_hash, tx_height in hist:
1447                 if tx_height>0:
1448                     # add it in case it was previously unconfirmed
1449                     self.verifier.add(tx_hash, tx_height)
1450
1451
1452         # if we are on a pruning server, remove unverified transactions
1453         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1454         for tx_hash in self.transactions.keys():
1455             if tx_hash not in vr:
1456                 self.transactions.pop(tx_hash)
1457
1458
1459
1460     def check_new_history(self, addr, hist):
1461
1462         # check that all tx in hist are relevant
1463         if hist != ['*']:
1464             for tx_hash, height in hist:
1465                 tx = self.transactions.get(tx_hash)
1466                 if not tx: continue
1467                 if not tx.has_address(addr):
1468                     return False
1469
1470         # check that we are not "orphaning" a transaction
1471         old_hist = self.history.get(addr,[])
1472         if old_hist == ['*']: return True
1473
1474         for tx_hash, height in old_hist:
1475             if tx_hash in map(lambda x:x[0], hist): continue
1476             found = False
1477             for _addr, _hist in self.history.items():
1478                 if _addr == addr: continue
1479                 if _hist == ['*']: continue
1480                 _tx_hist = map(lambda x:x[0], _hist)
1481                 if tx_hash in _tx_hist:
1482                     found = True
1483                     break
1484
1485             if not found:
1486                 tx = self.transactions.get(tx_hash)
1487                 # tx might not be there
1488                 if not tx: continue
1489
1490                 # already verified?
1491                 if self.verifier.get_height(tx_hash):
1492                     continue
1493                 # unconfirmed tx
1494                 print_error("new history is orphaning transaction:", tx_hash)
1495                 # check that all outputs are not mine, request histories
1496                 ext_requests = []
1497                 for _addr, _v in tx.outputs:
1498                     # assert not self.is_mine(_addr)
1499                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1500
1501                 ext_h = self.network.synchronous_get(ext_requests)
1502                 print_error("sync:", ext_requests, ext_h)
1503                 height = None
1504                 for h in ext_h:
1505                     if h == ['*']: continue
1506                     for item in h:
1507                         if item.get('tx_hash') == tx_hash:
1508                             height = item.get('height')
1509                 if height:
1510                     print_error("found height for", tx_hash, height)
1511                     self.verifier.add(tx_hash, height)
1512                 else:
1513                     print_error("removing orphaned tx from history", tx_hash)
1514                     self.transactions.pop(tx_hash)
1515
1516         return True
1517
1518
1519
1520     def check_new_tx(self, tx_hash, tx):
1521         # 1 check that tx is referenced in addr_history.
1522         addresses = []
1523         for addr, hist in self.history.items():
1524             if hist == ['*']:continue
1525             for txh, height in hist:
1526                 if txh == tx_hash:
1527                     addresses.append(addr)
1528
1529         if not addresses:
1530             return False
1531
1532         # 2 check that referencing addresses are in the tx
1533         for addr in addresses:
1534             if not tx.has_address(addr):
1535                 return False
1536
1537         return True
1538
1539
1540     def start_threads(self, network):
1541         from verifier import TxVerifier
1542         self.network = network
1543         if self.network:
1544             self.verifier = TxVerifier(self.network, self.storage)
1545             self.verifier.start()
1546             self.set_verifier(self.verifier)
1547             self.synchronizer = WalletSynchronizer(self, network)
1548             self.synchronizer.start()
1549         else:
1550             self.verifier = None
1551             self.synchronizer =None
1552
1553     def stop_threads(self):
1554         if self.network:
1555             self.verifier.stop()
1556             self.synchronizer.stop()
1557
1558
1559     def restore(self, callback):
1560         from i18n import _
1561         def wait_for_wallet():
1562             self.set_up_to_date(False)
1563             while not self.is_up_to_date():
1564                 msg = "%s\n%s %d\n%s %.1f"%(
1565                     _("Please wait..."),
1566                     _("Addresses generated:"),
1567                     len(self.addresses(True)),
1568                     _("Kilobytes received:"),
1569                     self.network.interface.bytes_received/1024.)
1570
1571                 apply(callback, (msg,))
1572                 time.sleep(0.1)
1573
1574         def wait_for_network():
1575             while not self.network.is_connected():
1576                 msg = "%s \n" % (_("Connecting..."))
1577                 apply(callback, (msg,))
1578                 time.sleep(0.1)
1579
1580         # wait until we are connected, because the user might have selected another server
1581         if self.network:
1582             wait_for_network()
1583             wait_for_wallet()
1584         else:
1585             self.synchronize()
1586
1587         self.fill_addressbook()
1588
1589
1590
1591
1592 class WalletSynchronizer(threading.Thread):
1593
1594
1595     def __init__(self, wallet, network):
1596         threading.Thread.__init__(self)
1597         self.daemon = True
1598         self.wallet = wallet
1599         self.network = network
1600         self.was_updated = True
1601         self.running = False
1602         self.lock = threading.Lock()
1603         self.queue = Queue.Queue()
1604
1605     def stop(self):
1606         with self.lock: self.running = False
1607
1608     def is_running(self):
1609         with self.lock: return self.running
1610
1611
1612     def subscribe_to_addresses(self, addresses):
1613         messages = []
1614         for addr in addresses:
1615             messages.append(('blockchain.address.subscribe', [addr]))
1616         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1617
1618
1619     def run(self):
1620         with self.lock:
1621             self.running = True
1622
1623         while self.is_running():
1624
1625             if not self.network.is_connected():
1626                 self.network.wait_until_connected()
1627
1628             self.run_interface(self.network.interface)
1629
1630
1631     def run_interface(self, interface):
1632
1633         print_error("synchronizer: connected to", interface.server)
1634
1635         requested_tx = []
1636         missing_tx = []
1637         requested_histories = {}
1638
1639         # request any missing transactions
1640         for history in self.wallet.history.values():
1641             if history == ['*']: continue
1642             for tx_hash, tx_height in history:
1643                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1644                     missing_tx.append( (tx_hash, tx_height) )
1645
1646         if missing_tx:
1647             print_error("missing tx", missing_tx)
1648
1649         # subscriptions
1650         self.subscribe_to_addresses(self.wallet.addresses(True))
1651
1652         while self.is_running():
1653             # 1. create new addresses
1654             new_addresses = self.wallet.synchronize()
1655
1656             # request missing addresses
1657             if new_addresses:
1658                 self.subscribe_to_addresses(new_addresses)
1659
1660             # request missing transactions
1661             for tx_hash, tx_height in missing_tx:
1662                 if (tx_hash, tx_height) not in requested_tx:
1663                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1664                     requested_tx.append( (tx_hash, tx_height) )
1665             missing_tx = []
1666
1667             # detect if situation has changed
1668             if interface.is_up_to_date() and self.queue.empty():
1669                 if not self.wallet.is_up_to_date():
1670                     self.wallet.set_up_to_date(True)
1671                     self.was_updated = True
1672             else:
1673                 if self.wallet.is_up_to_date():
1674                     self.wallet.set_up_to_date(False)
1675                     self.was_updated = True
1676
1677             if self.was_updated:
1678                 self.wallet.network.trigger_callback('updated')
1679                 self.was_updated = False
1680
1681             # 2. get a response
1682             try:
1683                 r = self.queue.get(block=True, timeout=1)
1684             except Queue.Empty:
1685                 continue
1686
1687             if interface != self.network.interface:
1688                 break
1689
1690             if not r:
1691                 continue
1692
1693             # 3. handle response
1694             method = r['method']
1695             params = r['params']
1696             result = r.get('result')
1697             error = r.get('error')
1698             if error:
1699                 print "error", r
1700                 continue
1701
1702             if method == 'blockchain.address.subscribe':
1703                 addr = params[0]
1704                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1705                     if requested_histories.get(addr) is None:
1706                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1707                         requested_histories[addr] = result
1708
1709             elif method == 'blockchain.address.get_history':
1710                 addr = params[0]
1711                 print_error("receiving history", addr, result)
1712                 if result == ['*']:
1713                     assert requested_histories.pop(addr) == '*'
1714                     self.wallet.receive_history_callback(addr, result)
1715                 else:
1716                     hist = []
1717                     # check that txids are unique
1718                     txids = []
1719                     for item in result:
1720                         tx_hash = item['tx_hash']
1721                         if tx_hash not in txids:
1722                             txids.append(tx_hash)
1723                             hist.append( (tx_hash, item['height']) )
1724
1725                     if len(hist) != len(result):
1726                         raise Exception("error: server sent history with non-unique txid", result)
1727
1728                     # check that the status corresponds to what was announced
1729                     rs = requested_histories.pop(addr)
1730                     if self.wallet.get_status(hist) != rs:
1731                         raise Exception("error: status mismatch: %s"%addr)
1732
1733                     # store received history
1734                     self.wallet.receive_history_callback(addr, hist)
1735
1736                     # request transactions that we don't have
1737                     for tx_hash, tx_height in hist:
1738                         if self.wallet.transactions.get(tx_hash) is None:
1739                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1740                                 missing_tx.append( (tx_hash, tx_height) )
1741
1742             elif method == 'blockchain.transaction.get':
1743                 tx_hash = params[0]
1744                 tx_height = params[1]
1745                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1746                 tx = Transaction(result)
1747                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1748                 self.was_updated = True
1749                 requested_tx.remove( (tx_hash, tx_height) )
1750                 print_error("received tx:", tx_hash, len(tx.raw))
1751
1752             else:
1753                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1754
1755             if self.was_updated and not requested_tx:
1756                 self.wallet.network.trigger_callback('updated')
1757                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1758
1759                 self.was_updated = False