lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         v = self.data.get(key)
 129         if v is None:
 130             v = default
 131         return v
 132
 133     def put(self, key, value, save = True):
 134
 135         with self.lock:
 136             if value is not None:
 137                 self.data[key] = value
 138             else:
 139                 self.data.pop(key)
 140             if save:
 141                 self.write()
 142
 143     def write(self):
 144         s = repr(self.data)
 145         f = open(self.path,"w")
 146         f.write( s )
 147         f.close()
 148         if 'ANDROID_DATA' not in os.environ:
 149             import stat
 150             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 151
 152
 153 class Wallet:
 154
 155     def __init__(self, storage):
 156
 157         self.storage = storage
 158         self.electrum_version = ELECTRUM_VERSION
 159         self.gap_limit_for_change = 3 # constant
 160
 161         # saved fields
 162         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 163
 164         self.gap_limit             = storage.get('gap_limit', 5)
 165         self.use_change            = storage.get('use_change',True)
 166         self.use_encryption        = storage.get('use_encryption', False)
 167         self.seed                  = storage.get('seed', '')               # encrypted
 168         self.labels                = storage.get('labels', {})
 169         self.frozen_addresses      = storage.get('frozen_addresses',[])
 170         self.addressbook           = storage.get('contacts', [])
 171
 172         self.imported_keys         = storage.get('imported_keys',{})
 173         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 174
 175         self.fee                   = int(storage.get('fee_per_kb',20000))
 176
 177         self.master_public_keys = storage.get('master_public_keys',{})
 178         self.master_private_keys = storage.get('master_private_keys', {})
 179
 180         self.next_addresses = storage.get('next_addresses',{})
 181
 182         if self.seed_version not in [4, 6]:
 183             msg = "This wallet seed is not supported."
 184             if self.seed_version in [5]:
 185                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 186                 print msg
 187                 sys.exit(1)
 188
 189
 190         self.load_accounts()
 191
 192         self.transactions = {}
 193         tx_list = self.storage.get('transactions',{})
 194         for k,v in tx_list.items():
 195             try:
 196                 tx = Transaction(v)
 197             except Exception:
 198                 print_msg("Warning: Cannot deserialize transactions. skipping")
 199                 continue
 200
 201             self.add_extra_addresses(tx)
 202             self.transactions[k] = tx
 203
 204         for h,tx in self.transactions.items():
 205             if not self.check_new_tx(h, tx):
 206                 print_error("removing unreferenced tx", h)
 207                 self.transactions.pop(h)
 208
 209
 210         # not saved
 211         self.prevout_values = {}     # my own transaction outputs
 212         self.spent_outputs = []
 213
 214         # spv
 215         self.verifier = None
 216
 217         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 218         # interface.is_up_to_date() returns true when all requests have been answered and processed
 219         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 220
 221         self.up_to_date = False
 222         self.lock = threading.Lock()
 223         self.transaction_lock = threading.Lock()
 224         self.tx_event = threading.Event()
 225
 226         for tx_hash, tx in self.transactions.items():
 227             self.update_tx_outputs(tx_hash)
 228
 229
 230     def add_extra_addresses(self, tx):
 231         h = tx.hash()
 232         # find the address corresponding to pay-to-pubkey inputs
 233         tx.add_extra_addresses(self.transactions)
 234         for o in tx.d.get('outputs'):
 235             if o.get('is_pubkey'):
 236                 for tx2 in self.transactions.values():
 237                     tx2.add_extra_addresses({h:tx})
 238
 239
 240
 241
 242     def set_up_to_date(self,b):
 243         with self.lock: self.up_to_date = b
 244
 245
 246     def is_up_to_date(self):
 247         with self.lock: return self.up_to_date
 248
 249
 250     def update(self):
 251         self.up_to_date = False
 252         while not self.is_up_to_date():
 253             time.sleep(0.1)
 254
 255
 256     def import_key(self, sec, password):
 257         # check password
 258         seed = self.get_seed(password)
 259         try:
 260             address = address_from_private_key(sec)
 261         except Exception:
 262             raise Exception('Invalid private key')
 263
 264         if self.is_mine(address):
 265             raise Exception('Address already in wallet')
 266
 267         # store the originally requested keypair into the imported keys table
 268         self.imported_keys[address] = pw_encode(sec, password )
 269         self.storage.put('imported_keys', self.imported_keys, True)
 270         if self.synchronizer:
 271             self.synchronizer.subscribe_to_addresses([address])
 272         return address
 273
 274     def delete_imported_key(self, addr):
 275         if addr in self.imported_keys:
 276             self.imported_keys.pop(addr)
 277             self.storage.put('imported_keys', self.imported_keys, True)
 278
 279
 280     def make_seed(self):
 281         import mnemonic, ecdsa
 282         entropy = ecdsa.util.randrange( pow(2,160) )
 283         nonce = 0
 284         while True:
 285             ss = "%040x"%(entropy+nonce)
 286             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 287             # we keep only 13 words, that's approximately 139 bits of entropy
 288             words = mnemonic.mn_encode(s)[0:13]
 289             seed = ' '.join(words)
 290             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 291                 break  # this removes 12 bits of entropy
 292             nonce += 1
 293
 294         return seed
 295
 296
 297     def init_seed(self, seed):
 298         import mnemonic
 299
 300         if self.seed:
 301             raise Exception("a seed exists")
 302
 303         if not seed:
 304             self.seed = random_seed(128)
 305             self.seed_version = 4
 306             return
 307
 308         #if not seed:
 309         #    self.seed = self.make_seed()
 310         #    self.seed_version = SEED_VERSION
 311         #    return
 312
 313         # find out what kind of wallet we are
 314         try:
 315             seed.strip().decode('hex')
 316             self.seed_version = 4
 317             self.seed = str(seed)
 318             return
 319         except Exception:
 320             pass
 321
 322         words = seed.split()
 323         self.seed_version = 4
 324         self.seed = mnemonic.mn_decode(words)
 325
 326         #try:
 327         #    mnemonic.mn_decode(words)
 328         #    uses_electrum_words = True
 329         #except Exception:
 330         #    uses_electrum_words = False
 331         #
 332         #if uses_electrum_words and len(words) != 13:
 333         #    self.seed_version = 4
 334         #    self.seed = mnemonic.mn_decode(words)
 335         #else:
 336         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 337         #    self.seed_version = SEED_VERSION
 338         #    self.seed = seed
 339
 340
 341     def save_seed(self):
 342         self.storage.put('seed', self.seed, True)
 343         self.storage.put('seed_version', self.seed_version, True)
 344         self.create_accounts()
 345
 346
 347     def create_watching_only_wallet(self, params):
 348         K0, c0 = params
 349         if not K0:
 350             return
 351
 352         if not c0:
 353             self.seed_version = 4
 354             self.storage.put('seed_version', self.seed_version, True)
 355             self.create_old_account(K0)
 356             return
 357
 358         cK0 = ""
 359         self.master_public_keys = {
 360             "m/0'/": (c0, K0, cK0),
 361             }
 362         self.storage.put('master_public_keys', self.master_public_keys, True)
 363         self.storage.put('seed_version', self.seed_version, True)
 364         self.create_account('1','Main account')
 365
 366
 367     def create_accounts(self):
 368         if self.seed_version == 4:
 369             mpk = OldAccount.mpk_from_seed(self.seed)
 370             self.create_old_account(mpk)
 371         else:
 372             # create default account
 373             self.create_master_keys('1')
 374             self.create_account('1','Main account')
 375
 376
 377     def create_master_keys(self, account_type):
 378         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 379         if account_type == '1':
 380             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 381             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 382             self.master_private_keys["m/0'/"] = k0
 383         elif account_type == '2of2':
 384             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 385             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 386             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 387             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 388             self.master_private_keys["m/1'/"] = k1
 389             self.master_private_keys["m/2'/"] = k2
 390         elif account_type == '2of3':
 391             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 392             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 393             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 394             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 395             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 396             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 397             self.master_private_keys["m/3'/"] = k3
 398             self.master_private_keys["m/4'/"] = k4
 399             self.master_private_keys["m/5'/"] = k5
 400
 401         self.storage.put('master_public_keys', self.master_public_keys, True)
 402         self.storage.put('master_private_keys', self.master_private_keys, True)
 403
 404     def has_master_public_keys(self, account_type):
 405         if account_type == '1':
 406             return "m/0'/" in self.master_public_keys
 407         elif account_type == '2of2':
 408             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 409         elif account_type == '2of3':
 410             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 411
 412     def find_root_by_master_key(self, c, K):
 413         for key, v in self.master_public_keys.items():
 414             if key == "m/":continue
 415             cc, KK, _ = v
 416             if (c == cc) and (K == KK):
 417                 return key
 418
 419     def deseed_root(self, seed, password):
 420         # for safety, we ask the user to enter their seed
 421         assert seed == self.get_seed(password)
 422         self.seed = ''
 423         self.storage.put('seed', '', True)
 424
 425
 426     def deseed_branch(self, k):
 427         # check that parent has no seed
 428         assert self.seed == ''
 429         self.master_private_keys.pop(k)
 430         self.storage.put('master_private_keys', self.master_private_keys, True)
 431
 432     def is_watching_only(self):
 433         return (self.seed == '') and (self.master_private_keys == {})
 434
 435
 436
 437     def account_id(self, account_type, i):
 438         if account_type == '1':
 439             return "m/0'/%d"%i
 440         elif account_type == '2of2':
 441             return "m/1'/%d & m/2'/%d"%(i,i)
 442         elif account_type == '2of3':
 443             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 444         else:
 445             raise Exception('unknown account type')
 446
 447
 448     def num_accounts(self, account_type):
 449         keys = self.accounts.keys()
 450         i = 0
 451         while True:
 452             account_id = self.account_id(account_type, i)
 453             if account_id not in keys: break
 454             i += 1
 455         return i
 456
 457
 458     def new_account_address(self, account_type = '1'):
 459         i = self.num_accounts(account_type)
 460         k = self.account_id(account_type,i)
 461
 462         addr = self.next_addresses.get(k)
 463         if not addr:
 464             account_id, account = self.next_account(account_type)
 465             addr = account.first_address()
 466             self.next_addresses[k] = addr
 467             self.storage.put('next_addresses',self.next_addresses)
 468
 469         return k, addr
 470
 471
 472     def next_account(self, account_type = '1'):
 473
 474         i = self.num_accounts(account_type)
 475         account_id = self.account_id(account_type,i)
 476
 477         if account_type is '1':
 478             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 479             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 480             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 481
 482         elif account_type == '2of2':
 483             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 484             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 485             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 486             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 487             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 488
 489         elif account_type == '2of3':
 490             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 491             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 492             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 493             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 494             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 495             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 496             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 497
 498         return account_id, account
 499
 500
 501     def set_label(self, name, text = None):
 502         changed = False
 503         old_text = self.labels.get(name)
 504         if text:
 505             if old_text != text:
 506                 self.labels[name] = text
 507                 changed = True
 508         else:
 509             if old_text:
 510                 self.labels.pop(name)
 511                 changed = True
 512
 513         if changed:
 514             self.storage.put('labels', self.labels, True)
 515
 516         run_hook('set_label', name, text, changed)
 517         return changed
 518
 519
 520
 521     def create_account(self, account_type = '1', name = None):
 522         k, account = self.next_account(account_type)
 523         if k in self.pending_accounts:
 524             self.pending_accounts.pop(k)
 525             self.storage.put('pending_accounts', self.pending_accounts)
 526
 527         self.accounts[k] = account
 528         self.save_accounts()
 529         if name:
 530             self.set_label(k, name)
 531
 532
 533     def create_old_account(self, mpk):
 534         self.storage.put('master_public_key', mpk, True)
 535         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 536         self.save_accounts()
 537
 538
 539     def save_accounts(self):
 540         d = {}
 541         for k, v in self.accounts.items():
 542             d[k] = v.dump()
 543         self.storage.put('accounts', d, True)
 544
 545
 546
 547     def load_accounts(self):
 548         d = self.storage.get('accounts', {})
 549         self.accounts = {}
 550         for k, v in d.items():
 551             if k == 0:
 552                 v['mpk'] = self.storage.get('master_public_key')
 553                 self.accounts[k] = OldAccount(v)
 554             elif '&' in k:
 555                 self.accounts[k] = BIP32_Account_2of2(v)
 556             else:
 557                 self.accounts[k] = BIP32_Account(v)
 558
 559         self.pending_accounts = self.storage.get('pending_accounts',{})
 560
 561
 562     def delete_pending_account(self, k):
 563         self.pending_accounts.pop(k)
 564         self.storage.put('pending_accounts', self.pending_accounts)
 565
 566     def account_is_pending(self, k):
 567         return k in self.pending_accounts
 568
 569     def create_pending_account(self, acct_type, name):
 570         k, addr = self.new_account_address(acct_type)
 571         self.set_label(k, name)
 572         self.pending_accounts[k] = addr
 573         self.storage.put('pending_accounts', self.pending_accounts)
 574
 575     def get_pending_accounts(self):
 576         return self.pending_accounts.items()
 577
 578
 579     def addresses(self, include_change = True, _next=True):
 580         o = self.get_account_addresses(-1, include_change)
 581         for a in self.accounts.keys():
 582             o += self.get_account_addresses(a, include_change)
 583
 584         if _next:
 585             for addr in self.next_addresses.values():
 586                 if addr not in o:
 587                     o += [addr]
 588         return o
 589
 590
 591     def is_mine(self, address):
 592         return address in self.addresses(True)
 593
 594
 595     def is_change(self, address):
 596         if not self.is_mine(address): return False
 597         if address in self.imported_keys.keys(): return False
 598         acct, s = self.get_address_index(address)
 599         if s is None: return False
 600         return s[0] == 1
 601
 602     def get_master_public_key(self):
 603         if self.seed_version == 4:
 604             return self.storage.get("master_public_key")
 605         else:
 606             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 607             return repr((c, K))
 608
 609     def get_master_private_key(self, account, password):
 610         master_k = pw_decode( self.master_private_keys[account], password)
 611         master_c, master_K, master_Kc = self.master_public_keys[account]
 612         try:
 613             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 614             assert K.encode('hex') == master_K
 615         except Exception:
 616             raise Exception("Invalid password")
 617         return master_k
 618
 619
 620     def get_address_index(self, address):
 621         if address in self.imported_keys.keys():
 622             return -1, None
 623
 624         for account in self.accounts.keys():
 625             for for_change in [0,1]:
 626                 addresses = self.accounts[account].get_addresses(for_change)
 627                 for addr in addresses:
 628                     if address == addr:
 629                         return account, (for_change, addresses.index(addr))
 630
 631         for k,v in self.next_addresses.items():
 632             if v == address:
 633                 return k, (0,0)
 634
 635         raise Exception("Address not found", address)
 636
 637
 638     def get_roots(self, account):
 639         roots = []
 640         for a in account.split('&'):
 641             s = a.strip()
 642             m = re.match("(m/\d+'/)(\d+)", s)
 643             roots.append( m.group(1) )
 644         return roots
 645
 646     def is_seeded(self, account):
 647         if type(account) is int:
 648             return self.seed is not None
 649
 650         for root in self.get_roots(account):
 651             if root not in self.master_private_keys.keys():
 652                 return False
 653         return True
 654
 655     def rebase_sequence(self, account, sequence):
 656         c, i = sequence
 657         dd = []
 658         for a in account.split('&'):
 659             s = a.strip()
 660             m = re.match("(m/\d+'/)(\d+)", s)
 661             root = m.group(1)
 662             num = int(m.group(2))
 663             dd.append( (root, [num,c,i] ) )
 664         return dd
 665
 666
 667     def get_keyID(self, account, sequence):
 668         if account == 0:
 669             a, b = sequence
 670             mpk = self.storage.get('master_public_key')
 671             return 'old(%s,%d,%d)'%(mpk,a,b)
 672
 673         rs = self.rebase_sequence(account, sequence)
 674         dd = []
 675         for root, public_sequence in rs:
 676             c, K, _ = self.master_public_keys[root]
 677             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 678             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 679         return '&'.join(dd)
 680
 681
 682
 683     def get_seed(self, password):
 684         s = pw_decode(self.seed, password)
 685         if self.seed_version == 4:
 686             seed = s
 687             self.accounts[0].check_seed(seed)
 688         else:
 689             seed = mnemonic_hash(s)
 690         return seed
 691
 692
 693     def get_mnemonic(self, password):
 694         import mnemonic
 695         s = pw_decode(self.seed, password)
 696         if self.seed_version == 4:
 697             return ' '.join(mnemonic.mn_encode(s))
 698         else:
 699             return s
 700
 701
 702
 703     def get_private_key(self, address, password):
 704         if self.is_watching_only():
 705             return []
 706
 707         # first check the provided password
 708         seed = self.get_seed(password)
 709
 710         out = []
 711         if address in self.imported_keys.keys():
 712             out.append( pw_decode( self.imported_keys[address], password ) )
 713         else:
 714             account, sequence = self.get_address_index(address)
 715             if account == 0:
 716                 pk = self.accounts[account].get_private_key(seed, sequence)
 717                 out.append(pk)
 718                 return out
 719
 720             # assert address == self.accounts[account].get_address(*sequence)
 721             rs = self.rebase_sequence( account, sequence)
 722             for root, public_sequence in rs:
 723
 724                 if root not in self.master_private_keys.keys(): continue
 725                 master_k = self.get_master_private_key(root, password)
 726                 master_c, _, _ = self.master_public_keys[root]
 727                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 728                 out.append(pk)
 729
 730         return out
 731
 732
 733     def add_keypairs_from_wallet(self, tx, keypairs, password):
 734         for txin in tx.inputs:
 735             address = txin['address']
 736             private_keys = self.get_private_key(address, password)
 737             for sec in private_keys:
 738                 pubkey = public_key_from_private_key(sec)
 739                 keypairs[ pubkey ] = sec
 740                 if address in self.imported_keys.keys():
 741                     txin['redeemPubkey'] = pubkey
 742
 743
 744     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 745         # first check the provided password
 746         seed = self.get_seed(password)
 747
 748         for txin in tx.inputs:
 749             keyid = txin.get('KeyID')
 750             if keyid:
 751
 752                 if self.seed_version == 4:
 753                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 754                     if not m: continue
 755                     mpk = m.group(1)
 756                     if mpk != self.storage.get('master_public_key'): continue
 757                     for_change = int(m.group(2))
 758                     num = int(m.group(3))
 759                     account = self.accounts[0]
 760                     addr = account.get_address(for_change, num)
 761                     txin['address'] = addr # fixme: side effect
 762                     pk = account.get_private_key(seed, (for_change, num))
 763                     pubkey = public_key_from_private_key(pk)
 764                     keypairs[pubkey] = pk
 765                     continue
 766
 767
 768                 roots = []
 769                 for s in keyid.split('&'):
 770                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 771                     if not m: continue
 772                     c = m.group(1)
 773                     K = m.group(2)
 774                     sequence = m.group(3)
 775                     root = self.find_root_by_master_key(c,K)
 776                     if not root: continue
 777                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 778                     root = root + '%d'%sequence[0]
 779                     sequence = sequence[1:]
 780                     roots.append((root,sequence))
 781
 782                 account_id = " & ".join( map(lambda x:x[0], roots) )
 783                 account = self.accounts.get(account_id)
 784                 if not account: continue
 785                 addr = account.get_address(*sequence)
 786                 txin['address'] = addr # fixme: side effect
 787                 pk = self.get_private_key(addr, password)
 788                 for sec in pk:
 789                     pubkey = public_key_from_private_key(sec)
 790                     keypairs[pubkey] = sec
 791
 792
 793
 794     def signrawtransaction(self, tx, input_info, private_keys, password):
 795
 796         # check that the password is correct
 797         seed = self.get_seed(password)
 798
 799         # add input info
 800         tx.add_input_info(input_info)
 801
 802         # add redeem script for coins that are in the wallet
 803         # FIXME: add redeemPubkey too!
 804         unspent_coins = self.get_unspent_coins()
 805         for txin in tx.inputs:
 806             for item in unspent_coins:
 807                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 808                     print_error( "tx input is in unspent coins" )
 809                     txin['scriptPubKey'] = item['scriptPubKey']
 810                     account, sequence = self.get_address_index(item['address'])
 811                     if account != -1:
 812                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 813                         print_error("added redeemScript", txin['redeemScript'])
 814                     break
 815
 816
 817         # build a list of public/private keys
 818         keypairs = {}
 819
 820         # add private keys from parameter
 821         for sec in private_keys:
 822             pubkey = public_key_from_private_key(sec)
 823             keypairs[ pubkey ] = sec
 824
 825         # add private_keys from KeyID
 826         self.add_keypairs_from_KeyID(tx, keypairs, password)
 827
 828         # add private keys from wallet
 829         self.add_keypairs_from_wallet(tx, keypairs, password)
 830         self.sign_transaction(tx, keypairs)
 831
 832
 833     def sign_message(self, address, message, password):
 834         keys = self.get_private_key(address, password)
 835         assert len(keys) == 1
 836         sec = keys[0]
 837         key = regenerate_key(sec)
 838         compressed = is_compressed(sec)
 839         return key.sign_message(message, compressed, address)
 840
 841
 842     def change_gap_limit(self, value):
 843         if value >= self.gap_limit:
 844             self.gap_limit = value
 845             self.storage.put('gap_limit', self.gap_limit, True)
 846             #self.interface.poke('synchronizer')
 847             return True
 848
 849         elif value >= self.min_acceptable_gap():
 850             for key, account in self.accounts.items():
 851                 addresses = account[0]
 852                 k = self.num_unused_trailing_addresses(addresses)
 853                 n = len(addresses) - k + value
 854                 addresses = addresses[0:n]
 855                 self.accounts[key][0] = addresses
 856
 857             self.gap_limit = value
 858             self.storage.put('gap_limit', self.gap_limit, True)
 859             self.save_accounts()
 860             return True
 861         else:
 862             return False
 863
 864     def num_unused_trailing_addresses(self, addresses):
 865         k = 0
 866         for a in addresses[::-1]:
 867             if self.history.get(a):break
 868             k = k + 1
 869         return k
 870
 871     def min_acceptable_gap(self):
 872         # fixme: this assumes wallet is synchronized
 873         n = 0
 874         nmax = 0
 875
 876         for account in self.accounts.values():
 877             addresses = account.get_addresses(0)
 878             k = self.num_unused_trailing_addresses(addresses)
 879             for a in addresses[0:-k]:
 880                 if self.history.get(a):
 881                     n = 0
 882                 else:
 883                     n += 1
 884                     if n > nmax: nmax = n
 885         return nmax + 1
 886
 887
 888     def address_is_old(self, address):
 889         age = -1
 890         h = self.history.get(address, [])
 891         if h == ['*']:
 892             return True
 893         for tx_hash, tx_height in h:
 894             if tx_height == 0:
 895                 tx_age = 0
 896             else:
 897                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 898             if tx_age > age:
 899                 age = tx_age
 900         return age > 2
 901
 902
 903     def synchronize_sequence(self, account, for_change):
 904         limit = self.gap_limit_for_change if for_change else self.gap_limit
 905         new_addresses = []
 906         while True:
 907             addresses = account.get_addresses(for_change)
 908             if len(addresses) < limit:
 909                 address = account.create_new_address(for_change)
 910                 self.history[address] = []
 911                 new_addresses.append( address )
 912                 continue
 913
 914             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 915                 break
 916             else:
 917                 address = account.create_new_address(for_change)
 918                 self.history[address] = []
 919                 new_addresses.append( address )
 920
 921         return new_addresses
 922
 923
 924
 925     def create_pending_accounts(self):
 926         for account_type in ['1','2of2','2of3']:
 927             if not self.has_master_public_keys(account_type):
 928                 continue
 929             k, a = self.new_account_address(account_type)
 930             if self.address_is_old(a):
 931                 print_error( "creating account", a )
 932                 self.create_account(account_type)
 933                 self.next_addresses.pop(k)
 934
 935
 936     def synchronize_account(self, account):
 937         new = []
 938         new += self.synchronize_sequence(account, 0)
 939         new += self.synchronize_sequence(account, 1)
 940         return new
 941
 942
 943     def synchronize(self):
 944         if self.master_public_keys:
 945             self.create_pending_accounts()
 946         new = []
 947         for account in self.accounts.values():
 948             new += self.synchronize_account(account)
 949         if new:
 950             self.save_accounts()
 951             self.storage.put('addr_history', self.history, True)
 952         return new
 953
 954
 955     def is_found(self):
 956         return self.history.values() != [[]] * len(self.history)
 957
 958
 959     def add_contact(self, address, label=None):
 960         self.addressbook.append(address)
 961         self.storage.put('contacts', self.addressbook, True)
 962         if label:
 963             self.set_label(address, label)
 964
 965
 966     def delete_contact(self, addr):
 967         if addr in self.addressbook:
 968             self.addressbook.remove(addr)
 969             self.storage.put('addressbook', self.addressbook, True)
 970
 971
 972     def fill_addressbook(self):
 973         for tx_hash, tx in self.transactions.items():
 974             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 975             if is_send:
 976                 for addr, v in tx.outputs:
 977                     if not self.is_mine(addr) and addr not in self.addressbook:
 978                         self.addressbook.append(addr)
 979         # redo labels
 980         # self.update_tx_labels()
 981
 982     def get_num_tx(self, address):
 983         n = 0
 984         for tx in self.transactions.values():
 985             if address in map(lambda x:x[0], tx.outputs): n += 1
 986         return n
 987
 988
 989     def get_address_flags(self, addr):
 990         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 991         flags += "F" if addr in self.frozen_addresses else "-"
 992         return flags
 993
 994
 995     def get_tx_value(self, tx, account=None):
 996         domain = self.get_account_addresses(account)
 997         return tx.get_value(domain, self.prevout_values)
 998
 999
1000     def update_tx_outputs(self, tx_hash):
1001         tx = self.transactions.get(tx_hash)
1002
1003         for i, (addr, value) in enumerate(tx.outputs):
1004             key = tx_hash+ ':%d'%i
1005             self.prevout_values[key] = value
1006
1007         for item in tx.inputs:
1008             if self.is_mine(item.get('address')):
1009                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1010                 self.spent_outputs.append(key)
1011
1012
1013     def get_addr_balance(self, address):
1014         assert self.is_mine(address)
1015         h = self.history.get(address,[])
1016         if h == ['*']: return 0,0
1017         c = u = 0
1018         received_coins = []   # list of coins received at address
1019
1020         for tx_hash, tx_height in h:
1021             tx = self.transactions.get(tx_hash)
1022             if not tx: continue
1023
1024             for i, (addr, value) in enumerate(tx.outputs):
1025                 if addr == address:
1026                     key = tx_hash + ':%d'%i
1027                     received_coins.append(key)
1028
1029         for tx_hash, tx_height in h:
1030             tx = self.transactions.get(tx_hash)
1031             if not tx: continue
1032             v = 0
1033
1034             for item in tx.inputs:
1035                 addr = item.get('address')
1036                 if addr == address:
1037                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1038                     value = self.prevout_values.get( key )
1039                     if key in received_coins:
1040                         v -= value
1041
1042             for i, (addr, value) in enumerate(tx.outputs):
1043                 key = tx_hash + ':%d'%i
1044                 if addr == address:
1045                     v += value
1046
1047             if tx_height:
1048                 c += v
1049             else:
1050                 u += v
1051         return c, u
1052
1053
1054     def get_account_name(self, k):
1055         if k == 0:
1056             if self.seed_version == 4:
1057                 name = 'Main account'
1058             else:
1059                 name = 'Old account'
1060         else:
1061             default = "Unnamed account"
1062             m = re.match("m/0'/(\d+)", k)
1063             if m:
1064                 num = m.group(1)
1065                 if num == '0':
1066                     default = "Main account"
1067                 else:
1068                     default = "Account %s"%num
1069
1070             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1071             if m:
1072                 num = m.group(1)
1073                 default = "2of2 account %s"%num
1074             name = self.labels.get(k, default)
1075
1076         return name
1077
1078     def get_account_names(self):
1079         accounts = {}
1080         for k, account in self.accounts.items():
1081             accounts[k] = self.get_account_name(k)
1082         if self.imported_keys:
1083             accounts[-1] = 'Imported keys'
1084         return accounts
1085
1086     def get_account_addresses(self, a, include_change=True):
1087         if a is None:
1088             o = self.addresses(True)
1089         elif a == -1:
1090             o = self.imported_keys.keys()
1091         else:
1092             ac = self.accounts[a]
1093             o = ac.get_addresses(0)
1094             if include_change: o += ac.get_addresses(1)
1095         return o
1096
1097     def get_imported_balance(self):
1098         return self.get_balance(self.imported_keys.keys())
1099
1100     def get_account_balance(self, account):
1101         return self.get_balance(self.get_account_addresses(account))
1102
1103     def get_frozen_balance(self):
1104         return self.get_balance(self.frozen_addresses)
1105
1106     def get_balance(self, domain=None):
1107         if domain is None: domain = self.addresses(True)
1108         cc = uu = 0
1109         for addr in domain:
1110             c, u = self.get_addr_balance(addr)
1111             cc += c
1112             uu += u
1113         return cc, uu
1114
1115
1116     def get_unspent_coins(self, domain=None):
1117         coins = []
1118         if domain is None: domain = self.addresses(True)
1119         for addr in domain:
1120             h = self.history.get(addr, [])
1121             if h == ['*']: continue
1122             for tx_hash, tx_height in h:
1123                 tx = self.transactions.get(tx_hash)
1124                 if tx is None: raise Exception("Wallet not synchronized")
1125                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1126                 for o in tx.d.get('outputs'):
1127                     output = o.copy()
1128                     if output.get('address') != addr: continue
1129                     key = tx_hash + ":%d" % output.get('prevout_n')
1130                     if key in self.spent_outputs: continue
1131                     output['prevout_hash'] = tx_hash
1132                     output['height'] = tx_height
1133                     output['coinbase'] = is_coinbase
1134                     coins.append((tx_height, output))
1135
1136         # sort by age
1137         if coins:
1138             coins = sorted(coins)
1139             if coins[-1][0] != 0:
1140                 while coins[0][0] == 0:
1141                     coins = coins[1:] + [ coins[0] ]
1142         return [x[1] for x in coins]
1143
1144
1145     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1146         """ todo: minimize tx size """
1147         total = 0
1148         fee = self.fee if fixed_fee is None else fixed_fee
1149         if domain is None:
1150             domain = self.addresses(True)
1151
1152         for i in self.frozen_addresses:
1153             if i in domain: domain.remove(i)
1154
1155         coins = self.get_unspent_coins(domain)
1156         inputs = []
1157
1158         for item in coins:
1159             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1160                 continue
1161             addr = item.get('address')
1162             v = item.get('value')
1163             total += v
1164             inputs.append(item)
1165             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1166             if total >= amount + fee: break
1167         else:
1168             inputs = []
1169
1170         return inputs, total, fee
1171
1172
1173     def set_fee(self, fee):
1174         if self.fee != fee:
1175             self.fee = fee
1176             self.storage.put('fee_per_kb', self.fee, True)
1177
1178     def estimated_fee(self, inputs):
1179         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1180         fee = self.fee * int(round(estimated_size/1024.))
1181         if fee == 0: fee = self.fee
1182         return fee
1183
1184
1185     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1186         "add change to a transaction"
1187         change_amount = total - ( amount + fee )
1188         if change_amount > DUST_THRESHOLD:
1189             if not change_addr:
1190
1191                 # send change to one of the accounts involved in the tx
1192                 address = inputs[0].get('address')
1193                 account, _ = self.get_address_index(address)
1194
1195                 if not self.use_change or account == -1:
1196                     change_addr = inputs[-1]['address']
1197                 else:
1198                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1199
1200             # Insert the change output at a random position in the outputs
1201             posn = random.randint(0, len(outputs))
1202             outputs[posn:posn] = [( change_addr,  change_amount)]
1203         return outputs
1204
1205
1206     def get_history(self, address):
1207         with self.lock:
1208             return self.history.get(address)
1209
1210
1211     def get_status(self, h):
1212         if not h: return None
1213         if h == ['*']: return '*'
1214         status = ''
1215         for tx_hash, height in h:
1216             status += tx_hash + ':%d:' % height
1217         return hashlib.sha256( status ).digest().encode('hex')
1218
1219
1220     def receive_tx_callback(self, tx_hash, tx, tx_height):
1221
1222         with self.transaction_lock:
1223             self.add_extra_addresses(tx)
1224             if not self.check_new_tx(tx_hash, tx):
1225                 # may happen due to pruning
1226                 print_error("received transaction that is no longer referenced in history", tx_hash)
1227                 return
1228             self.transactions[tx_hash] = tx
1229             self.network.interface.pending_transactions_for_notifications.append(tx)
1230             self.save_transactions()
1231             if self.verifier and tx_height>0:
1232                 self.verifier.add(tx_hash, tx_height)
1233             self.update_tx_outputs(tx_hash)
1234
1235
1236     def save_transactions(self):
1237         tx = {}
1238         for k,v in self.transactions.items():
1239             tx[k] = str(v)
1240         self.storage.put('transactions', tx, True)
1241
1242     def receive_history_callback(self, addr, hist):
1243
1244         if not self.check_new_history(addr, hist):
1245             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1246
1247         with self.lock:
1248             self.history[addr] = hist
1249             self.storage.put('addr_history', self.history, True)
1250
1251         if hist != ['*']:
1252             for tx_hash, tx_height in hist:
1253                 if tx_height>0:
1254                     # add it in case it was previously unconfirmed
1255                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1256
1257
1258     def get_tx_history(self, account=None):
1259         if not self.verifier:
1260             return []
1261
1262         with self.transaction_lock:
1263             history = self.transactions.items()
1264             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1265             result = []
1266
1267             balance = 0
1268             for tx_hash, tx in history:
1269                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1270                 if v is not None: balance += v
1271
1272             c, u = self.get_account_balance(account)
1273
1274             if balance != c+u:
1275                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1276
1277             balance = c + u - balance
1278             for tx_hash, tx in history:
1279                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1280                 if not is_relevant:
1281                     continue
1282                 if value is not None:
1283                     balance += value
1284
1285                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1286                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1287
1288         return result
1289
1290
1291     def get_label(self, tx_hash):
1292         label = self.labels.get(tx_hash)
1293         is_default = (label == '') or (label is None)
1294         if is_default: label = self.get_default_label(tx_hash)
1295         return label, is_default
1296
1297
1298     def get_default_label(self, tx_hash):
1299         tx = self.transactions.get(tx_hash)
1300         default_label = ''
1301         if tx:
1302             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1303             if is_mine:
1304                 for o in tx.outputs:
1305                     o_addr, _ = o
1306                     if not self.is_mine(o_addr):
1307                         try:
1308                             default_label = self.labels[o_addr]
1309                         except KeyError:
1310                             default_label = o_addr
1311                         break
1312                 else:
1313                     default_label = '(internal)'
1314             else:
1315                 for o in tx.outputs:
1316                     o_addr, _ = o
1317                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1318                         break
1319                 else:
1320                     for o in tx.outputs:
1321                         o_addr, _ = o
1322                         if self.is_mine(o_addr):
1323                             break
1324                     else:
1325                         o_addr = None
1326
1327                 if o_addr:
1328                     dest_label = self.labels.get(o_addr)
1329                     try:
1330                         default_label = self.labels[o_addr]
1331                     except KeyError:
1332                         default_label = o_addr
1333
1334         return default_label
1335
1336
1337     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1338         for address, x in outputs:
1339             assert is_valid(address)
1340         amount = sum( map(lambda x:x[1], outputs) )
1341         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1342         if not inputs:
1343             raise ValueError("Not enough funds")
1344         self.add_input_info(inputs)
1345         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1346         return Transaction.from_io(inputs, outputs)
1347
1348
1349     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1350         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1351         keypairs = {}
1352         self.add_keypairs_from_wallet(tx, keypairs, password)
1353         if keypairs:
1354             self.sign_transaction(tx, keypairs)
1355         return tx
1356
1357
1358     def add_input_info(self, inputs):
1359         for txin in inputs:
1360             address = txin['address']
1361             if address in self.imported_keys.keys():
1362                 continue
1363             account, sequence = self.get_address_index(address)
1364             txin['KeyID'] = self.get_keyID(account, sequence)
1365             redeemScript = self.accounts[account].redeem_script(sequence)
1366             if redeemScript:
1367                 txin['redeemScript'] = redeemScript
1368             else:
1369                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1370
1371
1372     def sign_transaction(self, tx, keypairs):
1373         tx.sign(keypairs)
1374         run_hook('sign_transaction', tx)
1375
1376
1377     def sendtx(self, tx):
1378         # synchronous
1379         h = self.send_tx(tx)
1380         self.tx_event.wait()
1381         return self.receive_tx(h)
1382
1383     def send_tx(self, tx):
1384         # asynchronous
1385         self.tx_event.clear()
1386         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1387         return tx.hash()
1388
1389     def on_broadcast(self, i, r):
1390         self.tx_result = r.get('result')
1391         self.tx_event.set()
1392
1393     def receive_tx(self,tx_hash):
1394         out = self.tx_result
1395         if out != tx_hash:
1396             return False, "error: " + out
1397         return True, out
1398
1399
1400
1401     def update_password(self, old_password, new_password):
1402         if new_password == '': new_password = None
1403         decoded = self.get_seed(old_password)
1404         self.seed = pw_encode( decoded, new_password)
1405         self.storage.put('seed', self.seed, True)
1406         self.use_encryption = (new_password != None)
1407         self.storage.put('use_encryption', self.use_encryption,True)
1408         for k in self.imported_keys.keys():
1409             a = self.imported_keys[k]
1410             b = pw_decode(a, old_password)
1411             c = pw_encode(b, new_password)
1412             self.imported_keys[k] = c
1413         self.storage.put('imported_keys', self.imported_keys, True)
1414
1415         for k, v in self.master_private_keys.items():
1416             b = pw_decode(v, old_password)
1417             c = pw_encode(b, new_password)
1418             self.master_private_keys[k] = c
1419         self.storage.put('master_private_keys', self.master_private_keys, True)
1420
1421
1422     def freeze(self,addr):
1423         if self.is_mine(addr) and addr not in self.frozen_addresses:
1424             self.frozen_addresses.append(addr)
1425             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1426             return True
1427         else:
1428             return False
1429
1430     def unfreeze(self,addr):
1431         if self.is_mine(addr) and addr in self.frozen_addresses:
1432             self.frozen_addresses.remove(addr)
1433             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1434             return True
1435         else:
1436             return False
1437
1438
1439     def set_verifier(self, verifier):
1440         self.verifier = verifier
1441
1442         # review transactions that are in the history
1443         for addr, hist in self.history.items():
1444             if hist == ['*']: continue
1445             for tx_hash, tx_height in hist:
1446                 if tx_height>0:
1447                     # add it in case it was previously unconfirmed
1448                     self.verifier.add(tx_hash, tx_height)
1449
1450
1451         # if we are on a pruning server, remove unverified transactions
1452         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1453         for tx_hash in self.transactions.keys():
1454             if tx_hash not in vr:
1455                 self.transactions.pop(tx_hash)
1456
1457
1458
1459     def check_new_history(self, addr, hist):
1460
1461         # check that all tx in hist are relevant
1462         if hist != ['*']:
1463             for tx_hash, height in hist:
1464                 tx = self.transactions.get(tx_hash)
1465                 if not tx: continue
1466                 if not tx.has_address(addr):
1467                     return False
1468
1469         # check that we are not "orphaning" a transaction
1470         old_hist = self.history.get(addr,[])
1471         if old_hist == ['*']: return True
1472
1473         for tx_hash, height in old_hist:
1474             if tx_hash in map(lambda x:x[0], hist): continue
1475             found = False
1476             for _addr, _hist in self.history.items():
1477                 if _addr == addr: continue
1478                 if _hist == ['*']: continue
1479                 _tx_hist = map(lambda x:x[0], _hist)
1480                 if tx_hash in _tx_hist:
1481                     found = True
1482                     break
1483
1484             if not found:
1485                 tx = self.transactions.get(tx_hash)
1486                 # tx might not be there
1487                 if not tx: continue
1488
1489                 # already verified?
1490                 if self.verifier.get_height(tx_hash):
1491                     continue
1492                 # unconfirmed tx
1493                 print_error("new history is orphaning transaction:", tx_hash)
1494                 # check that all outputs are not mine, request histories
1495                 ext_requests = []
1496                 for _addr, _v in tx.outputs:
1497                     # assert not self.is_mine(_addr)
1498                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1499
1500                 ext_h = self.network.synchronous_get(ext_requests)
1501                 print_error("sync:", ext_requests, ext_h)
1502                 height = None
1503                 for h in ext_h:
1504                     if h == ['*']: continue
1505                     for item in h:
1506                         if item.get('tx_hash') == tx_hash:
1507                             height = item.get('height')
1508                 if height:
1509                     print_error("found height for", tx_hash, height)
1510                     self.verifier.add(tx_hash, height)
1511                 else:
1512                     print_error("removing orphaned tx from history", tx_hash)
1513                     self.transactions.pop(tx_hash)
1514
1515         return True
1516
1517
1518
1519     def check_new_tx(self, tx_hash, tx):
1520         # 1 check that tx is referenced in addr_history.
1521         addresses = []
1522         for addr, hist in self.history.items():
1523             if hist == ['*']:continue
1524             for txh, height in hist:
1525                 if txh == tx_hash:
1526                     addresses.append(addr)
1527
1528         if not addresses:
1529             return False
1530
1531         # 2 check that referencing addresses are in the tx
1532         for addr in addresses:
1533             if not tx.has_address(addr):
1534                 return False
1535
1536         return True
1537
1538
1539     def start_threads(self, network):
1540         from verifier import TxVerifier
1541         self.network = network
1542         if self.network:
1543             self.verifier = TxVerifier(self.network, self.storage)
1544             self.verifier.start()
1545             self.set_verifier(self.verifier)
1546             self.synchronizer = WalletSynchronizer(self, network)
1547             self.synchronizer.start()
1548         else:
1549             self.verifier = None
1550             self.synchronizer =None
1551
1552     def stop_threads(self):
1553         if self.network:
1554             self.verifier.stop()
1555             self.synchronizer.stop()
1556
1557
1558     def restore(self, callback):
1559         from i18n import _
1560         def wait_for_wallet():
1561             self.set_up_to_date(False)
1562             while not self.is_up_to_date():
1563                 msg = "%s\n%s %d\n%s %.1f"%(
1564                     _("Please wait..."),
1565                     _("Addresses generated:"),
1566                     len(self.addresses(True)),
1567                     _("Kilobytes received:"),
1568                     self.network.interface.bytes_received/1024.)
1569
1570                 apply(callback, (msg,))
1571                 time.sleep(0.1)
1572
1573         def wait_for_network():
1574             while not self.network.is_connected():
1575                 msg = "%s \n" % (_("Connecting..."))
1576                 apply(callback, (msg,))
1577                 time.sleep(0.1)
1578
1579         # wait until we are connected, because the user might have selected another server
1580         if self.network:
1581             wait_for_network()
1582             wait_for_wallet()
1583         else:
1584             self.synchronize()
1585
1586         self.fill_addressbook()
1587
1588
1589
1590
1591 class WalletSynchronizer(threading.Thread):
1592
1593
1594     def __init__(self, wallet, network):
1595         threading.Thread.__init__(self)
1596         self.daemon = True
1597         self.wallet = wallet
1598         self.network = network
1599         self.was_updated = True
1600         self.running = False
1601         self.lock = threading.Lock()
1602         self.queue = Queue.Queue()
1603
1604     def stop(self):
1605         with self.lock: self.running = False
1606
1607     def is_running(self):
1608         with self.lock: return self.running
1609
1610
1611     def subscribe_to_addresses(self, addresses):
1612         messages = []
1613         for addr in addresses:
1614             messages.append(('blockchain.address.subscribe', [addr]))
1615         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1616
1617
1618     def run(self):
1619         with self.lock:
1620             self.running = True
1621
1622         while self.is_running():
1623
1624             if not self.network.is_connected():
1625                 print_error("synchronizer: waiting for interface")
1626                 self.network.wait_until_connected()
1627
1628             self.run_interface(self.network.interface)
1629
1630
1631     def run_interface(self, interface):
1632
1633         print_error("synchronizer: connected to", interface.server)
1634
1635         requested_tx = []
1636         missing_tx = []
1637         requested_histories = {}
1638
1639         # request any missing transactions
1640         for history in self.wallet.history.values():
1641             if history == ['*']: continue
1642             for tx_hash, tx_height in history:
1643                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1644                     missing_tx.append( (tx_hash, tx_height) )
1645
1646         if missing_tx:
1647             print_error("missing tx", missing_tx)
1648
1649         # subscriptions
1650         self.subscribe_to_addresses(self.wallet.addresses(True))
1651
1652         while self.is_running():
1653             # 1. create new addresses
1654             new_addresses = self.wallet.synchronize()
1655
1656             # request missing addresses
1657             if new_addresses:
1658                 self.subscribe_to_addresses(new_addresses)
1659
1660             # request missing transactions
1661             for tx_hash, tx_height in missing_tx:
1662                 if (tx_hash, tx_height) not in requested_tx:
1663                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1664                     requested_tx.append( (tx_hash, tx_height) )
1665             missing_tx = []
1666
1667             # detect if situation has changed
1668             if interface.is_up_to_date() and self.queue.empty():
1669                 if not self.wallet.is_up_to_date():
1670                     self.wallet.set_up_to_date(True)
1671                     self.was_updated = True
1672             else:
1673                 if self.wallet.is_up_to_date():
1674                     self.wallet.set_up_to_date(False)
1675                     self.was_updated = True
1676
1677             if self.was_updated:
1678                 self.wallet.network.trigger_callback('updated')
1679                 self.was_updated = False
1680
1681             # 2. get a response
1682             try:
1683                 r = self.queue.get(block=True, timeout=1)
1684             except Queue.Empty:
1685                 continue
1686
1687             if interface != self.network.interface:
1688                 break
1689
1690             if not r:
1691                 continue
1692
1693             # 3. handle response
1694             method = r['method']
1695             params = r['params']
1696             result = r.get('result')
1697             error = r.get('error')
1698             if error:
1699                 print "error", r
1700                 continue
1701
1702             if method == 'blockchain.address.subscribe':
1703                 addr = params[0]
1704                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1705                     if requested_histories.get(addr) is None:
1706                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1707                         requested_histories[addr] = result
1708
1709             elif method == 'blockchain.address.get_history':
1710                 addr = params[0]
1711                 print_error("receiving history", addr, result)
1712                 if result == ['*']:
1713                     assert requested_histories.pop(addr) == '*'
1714                     self.wallet.receive_history_callback(addr, result)
1715                 else:
1716                     hist = []
1717                     # check that txids are unique
1718                     txids = []
1719                     for item in result:
1720                         tx_hash = item['tx_hash']
1721                         if tx_hash not in txids:
1722                             txids.append(tx_hash)
1723                             hist.append( (tx_hash, item['height']) )
1724
1725                     if len(hist) != len(result):
1726                         raise Exception("error: server sent history with non-unique txid", result)
1727
1728                     # check that the status corresponds to what was announced
1729                     rs = requested_histories.pop(addr)
1730                     if self.wallet.get_status(hist) != rs:
1731                         raise Exception("error: status mismatch: %s"%addr)
1732
1733                     # store received history
1734                     self.wallet.receive_history_callback(addr, hist)
1735
1736                     # request transactions that we don't have
1737                     for tx_hash, tx_height in hist:
1738                         if self.wallet.transactions.get(tx_hash) is None:
1739                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1740                                 missing_tx.append( (tx_hash, tx_height) )
1741
1742             elif method == 'blockchain.transaction.get':
1743                 tx_hash = params[0]
1744                 tx_height = params[1]
1745                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1746                 tx = Transaction(result)
1747                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1748                 self.was_updated = True
1749                 requested_tx.remove( (tx_hash, tx_height) )
1750                 print_error("received tx:", tx_hash, len(tx.raw))
1751
1752             else:
1753                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1754
1755             if self.was_updated and not requested_tx:
1756                 self.wallet.network.trigger_callback('updated')
1757                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1758
1759                 self.was_updated = False