lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40
  41 # AES encryption
  42 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  43 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  44
  45 def pw_encode(s, password):
  46     if password:
  47         secret = Hash(password)
  48         return EncodeAES(secret, s)
  49     else:
  50         return s
  51
  52 def pw_decode(s, password):
  53     if password is not None:
  54         secret = Hash(password)
  55         try:
  56             d = DecodeAES(secret, s)
  57         except:
  58             raise BaseException('Invalid password')
  59         return d
  60     else:
  61         return s
  62
  63
  64
  65
  66
  67 from version import *
  68
  69
  70 class WalletStorage:
  71
  72     def __init__(self, config):
  73         self.lock = threading.Lock()
  74         self.data = {}
  75         self.file_exists = False
  76         self.path = self.init_path(config)
  77         print_error( "wallet path", self.path )
  78         if self.path:
  79             self.read(self.path)
  80
  81
  82     def init_path(self, config):
  83         """Set the path of the wallet."""
  84
  85         # command line -w option
  86         path = config.get('wallet_path')
  87         if path:
  88             return path
  89
  90         # path in config file
  91         path = config.get('default_wallet_path')
  92         if path:
  93             return path
  94
  95         # default path
  96         dirpath = os.path.join(config.path, "wallets")
  97         if not os.path.exists(dirpath):
  98             os.mkdir(dirpath)
  99
 100         new_path = os.path.join(config.path, "wallets", "default_wallet")
 101
 102         # default path in pre 1.9 versions
 103         old_path = os.path.join(config.path, "electrum.dat")
 104         if os.path.exists(old_path) and not os.path.exists(new_path):
 105             os.rename(old_path, new_path)
 106
 107         return new_path
 108
 109
 110     def read(self, path):
 111         """Read the contents of the wallet file."""
 112         try:
 113             with open(self.path, "r") as f:
 114                 data = f.read()
 115         except IOError:
 116             return
 117         try:
 118             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 119         except:
 120             raise IOError("Cannot read wallet file.")
 121
 122         self.data = d
 123         self.file_exists = True
 124
 125
 126     def get(self, key, default=None):
 127         return self.data.get(key, default)
 128
 129     def put(self, key, value, save = True):
 130
 131         with self.lock:
 132             if value is not None:
 133                 self.data[key] = value
 134             else:
 135                 self.data.pop[key]
 136             if save:
 137                 self.write()
 138
 139     def write(self):
 140         s = repr(self.data)
 141         f = open(self.path,"w")
 142         f.write( s )
 143         f.close()
 144         if 'ANDROID_DATA' not in os.environ:
 145             import stat
 146             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 147
 148
 149 class Wallet:
 150
 151     def __init__(self, storage):
 152
 153         self.storage = storage
 154         self.electrum_version = ELECTRUM_VERSION
 155         self.gap_limit_for_change = 3 # constant
 156
 157         # saved fields
 158         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 159
 160         self.gap_limit             = storage.get('gap_limit', 5)
 161         self.use_change            = storage.get('use_change',True)
 162         self.use_encryption        = storage.get('use_encryption', False)
 163         self.seed                  = storage.get('seed', '')               # encrypted
 164         self.labels                = storage.get('labels', {})
 165         self.frozen_addresses      = storage.get('frozen_addresses',[])
 166         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 167         self.addressbook           = storage.get('contacts', [])
 168
 169         self.imported_keys         = storage.get('imported_keys',{})
 170         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 171
 172         self.fee                   = int(storage.get('fee_per_kb',20000))
 173
 174         self.master_public_keys = storage.get('master_public_keys',{})
 175         self.master_private_keys = storage.get('master_private_keys', {})
 176
 177         self.next_addresses = storage.get('next_addresses',{})
 178
 179         if self.seed_version not in [4, 6]:
 180             msg = "This wallet seed is not supported."
 181             if self.seed_version in [5]:
 182                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 183                 print msg
 184                 sys.exit(1)
 185
 186
 187         self.load_accounts()
 188
 189         self.transactions = {}
 190         tx_list = self.storage.get('transactions',{})
 191         for k,v in tx_list.items():
 192             try:
 193                 tx = Transaction(v)
 194             except:
 195                 print_msg("Warning: Cannot deserialize transactions. skipping")
 196                 continue
 197
 198             self.add_extra_addresses(tx)
 199             self.transactions[k] = tx
 200
 201         for h,tx in self.transactions.items():
 202             if not self.check_new_tx(h, tx):
 203                 print_error("removing unreferenced tx", h)
 204                 self.transactions.pop(h)
 205
 206
 207         # not saved
 208         self.prevout_values = {}     # my own transaction outputs
 209         self.spent_outputs = []
 210
 211         # spv
 212         self.verifier = None
 213
 214         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 215         # interface.is_up_to_date() returns true when all requests have been answered and processed
 216         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 217
 218         self.up_to_date = False
 219         self.lock = threading.Lock()
 220         self.transaction_lock = threading.Lock()
 221         self.tx_event = threading.Event()
 222
 223         for tx_hash, tx in self.transactions.items():
 224             self.update_tx_outputs(tx_hash)
 225
 226
 227     def add_extra_addresses(self, tx):
 228         h = tx.hash()
 229         # find the address corresponding to pay-to-pubkey inputs
 230         tx.add_extra_addresses(self.transactions)
 231         for o in tx.d.get('outputs'):
 232             if o.get('is_pubkey'):
 233                 for tx2 in self.transactions.values():
 234                     tx2.add_extra_addresses({h:tx})
 235
 236
 237
 238
 239     def set_up_to_date(self,b):
 240         with self.lock: self.up_to_date = b
 241
 242
 243     def is_up_to_date(self):
 244         with self.lock: return self.up_to_date
 245
 246
 247     def update(self):
 248         self.up_to_date = False
 249         while not self.is_up_to_date():
 250             time.sleep(0.1)
 251
 252
 253     def import_key(self, sec, password):
 254         # check password
 255         seed = self.get_seed(password)
 256         try:
 257             address = address_from_private_key(sec)
 258         except:
 259             raise BaseException('Invalid private key')
 260
 261         if self.is_mine(address):
 262             raise BaseException('Address already in wallet')
 263
 264         # store the originally requested keypair into the imported keys table
 265         self.imported_keys[address] = pw_encode(sec, password )
 266         self.storage.put('imported_keys', self.imported_keys, True)
 267         if self.synchronizer:
 268             self.synchronizer.subscribe_to_addresses([address])
 269         return address
 270
 271     def delete_imported_key(self, addr):
 272         if addr in self.imported_keys:
 273             self.imported_keys.pop(addr)
 274             self.storage.put('imported_keys', self.imported_keys, True)
 275
 276
 277     def make_seed(self):
 278         import mnemonic, ecdsa
 279         entropy = ecdsa.util.randrange( pow(2,160) )
 280         nonce = 0
 281         while True:
 282             ss = "%040x"%(entropy+nonce)
 283             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 284             # we keep only 13 words, that's approximately 139 bits of entropy
 285             words = mnemonic.mn_encode(s)[0:13]
 286             seed = ' '.join(words)
 287             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 288                 break  # this removes 12 bits of entropy
 289             nonce += 1
 290
 291         return seed
 292
 293
 294     def init_seed(self, seed):
 295         import mnemonic
 296
 297         if self.seed:
 298             raise BaseException("a seed exists")
 299
 300         if not seed:
 301             self.seed = random_seed(128)
 302             self.seed_version = 4
 303             return
 304
 305         #if not seed:
 306         #    self.seed = self.make_seed()
 307         #    self.seed_version = SEED_VERSION
 308         #    return
 309
 310         # find out what kind of wallet we are
 311         try:
 312             seed.decode('hex')
 313             self.seed_version = 4
 314             self.seed = str(seed)
 315             return
 316         except:
 317             pass
 318
 319         words = seed.split()
 320         self.seed_version = 4
 321         self.seed = mnemonic.mn_decode(words)
 322
 323         #try:
 324         #    mnemonic.mn_decode(words)
 325         #    uses_electrum_words = True
 326         #except:
 327         #    uses_electrum_words = False
 328         #
 329         #if uses_electrum_words and len(words) != 13:
 330         #    self.seed_version = 4
 331         #    self.seed = mnemonic.mn_decode(words)
 332         #else:
 333         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 334         #    self.seed_version = SEED_VERSION
 335         #    self.seed = seed
 336
 337
 338     def save_seed(self):
 339         self.storage.put('seed', self.seed, True)
 340         self.storage.put('seed_version', self.seed_version, True)
 341
 342     def create_watching_only_wallet(self, c0, K0):
 343         cK0 = ""
 344         self.master_public_keys = {
 345             "m/0'/": (c0, K0, cK0),
 346             }
 347         self.storage.put('master_public_keys', self.master_public_keys, True)
 348         self.create_account('1','Main account')
 349
 350
 351     def create_accounts(self):
 352         if self.seed_version == 4:
 353             self.create_old_account()
 354         else:
 355             # create default account
 356             self.create_master_keys('1')
 357             self.create_account('1','Main account')
 358
 359
 360     def create_master_keys(self, account_type):
 361         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 362         if account_type == '1':
 363             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 364             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 365             self.master_private_keys["m/0'/"] = k0
 366         elif account_type == '2of2':
 367             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 368             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 369             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 370             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 371             self.master_private_keys["m/1'/"] = k1
 372             self.master_private_keys["m/2'/"] = k2
 373         elif account_type == '2of3':
 374             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 375             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 376             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 377             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 378             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 379             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 380             self.master_private_keys["m/3'/"] = k3
 381             self.master_private_keys["m/4'/"] = k4
 382             self.master_private_keys["m/5'/"] = k5
 383
 384         self.storage.put('master_public_keys', self.master_public_keys, True)
 385         self.storage.put('master_private_keys', self.master_private_keys, True)
 386
 387     def has_master_public_keys(self, account_type):
 388         if account_type == '1':
 389             return "m/0'/" in self.master_public_keys
 390         elif account_type == '2of2':
 391             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 392         elif account_type == '2of3':
 393             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 394
 395     def find_root_by_master_key(self, c, K):
 396         for key, v in self.master_public_keys.items():
 397             if key == "m/":continue
 398             cc, KK, _ = v
 399             if (c == cc) and (K == KK):
 400                 return key
 401
 402     def deseed_root(self, seed, password):
 403         # for safety, we ask the user to enter their seed
 404         assert seed == self.get_seed(password)
 405         self.seed = ''
 406         self.storage.put('seed', '', True)
 407
 408
 409     def deseed_branch(self, k):
 410         # check that parent has no seed
 411         assert self.seed == ''
 412         self.master_private_keys.pop(k)
 413         self.storage.put('master_private_keys', self.master_private_keys, True)
 414
 415     def is_watching_only(self):
 416         return (self.seed == '') and (self.master_private_keys == {})
 417
 418
 419
 420     def account_id(self, account_type, i):
 421         if account_type == '1':
 422             return "m/0'/%d"%i
 423         elif account_type == '2of2':
 424             return "m/1'/%d & m/2'/%d"%(i,i)
 425         elif account_type == '2of3':
 426             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 427         else:
 428             raise BaseException('unknown account type')
 429
 430
 431     def num_accounts(self, account_type):
 432         keys = self.accounts.keys()
 433         i = 0
 434         while True:
 435             account_id = self.account_id(account_type, i)
 436             if account_id not in keys: break
 437             i += 1
 438         return i
 439
 440
 441     def new_account_address(self, account_type = '1'):
 442         i = self.num_accounts(account_type)
 443         k = self.account_id(account_type,i)
 444
 445         addr = self.next_addresses.get(k)
 446         if not addr:
 447             account_id, account = self.next_account(account_type)
 448             addr = account.first_address()
 449             self.next_addresses[k] = addr
 450             self.storage.put('next_addresses',self.next_addresses)
 451
 452         return k, addr
 453
 454
 455     def next_account(self, account_type = '1'):
 456
 457         i = self.num_accounts(account_type)
 458         account_id = self.account_id(account_type,i)
 459
 460         if account_type is '1':
 461             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 462             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 463             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 464
 465         elif account_type == '2of2':
 466             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 467             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 468             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 469             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 470             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 471
 472         elif account_type == '2of3':
 473             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 474             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 475             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 476             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 477             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 478             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 479             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 480
 481         return account_id, account
 482
 483
 484     def set_label(self, name, text = None):
 485         changed = False
 486         old_text = self.labels.get(name)
 487         if text:
 488             if old_text != text:
 489                 self.labels[name] = text
 490                 changed = True
 491         else:
 492             if old_text:
 493                 self.labels.pop(name)
 494                 changed = True
 495
 496         if changed:
 497             self.storage.put('labels', self.labels, True)
 498
 499         run_hook('set_label', name, text, changed)
 500         return changed
 501
 502
 503
 504     def create_account(self, account_type = '1', name = None):
 505         k, account = self.next_account(account_type)
 506         if k in self.pending_accounts:
 507             self.pending_accounts.pop(k)
 508             self.storage.put('pending_accounts', self.pending_accounts)
 509
 510         self.accounts[k] = account
 511         self.save_accounts()
 512         if name:
 513             self.set_label(k, name)
 514
 515
 516     def create_old_account(self):
 517         mpk = OldAccount.mpk_from_seed(self.seed)
 518         self.storage.put('master_public_key', mpk, True)
 519         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 520         self.save_accounts()
 521
 522
 523     def save_accounts(self):
 524         d = {}
 525         for k, v in self.accounts.items():
 526             d[k] = v.dump()
 527         self.storage.put('accounts', d, True)
 528
 529
 530
 531     def load_accounts(self):
 532         d = self.storage.get('accounts', {})
 533         self.accounts = {}
 534         for k, v in d.items():
 535             if k == 0:
 536                 v['mpk'] = self.storage.get('master_public_key')
 537                 self.accounts[k] = OldAccount(v)
 538             elif '&' in k:
 539                 self.accounts[k] = BIP32_Account_2of2(v)
 540             else:
 541                 self.accounts[k] = BIP32_Account(v)
 542
 543         self.pending_accounts = self.storage.get('pending_accounts',{})
 544
 545
 546     def delete_pending_account(self, k):
 547         self.pending_accounts.pop(k)
 548         self.storage.put('pending_accounts', self.pending_accounts)
 549
 550     def account_is_pending(self, k):
 551         return k in self.pending_accounts
 552
 553     def create_pending_account(self, acct_type, name):
 554         k, addr = self.new_account_address(acct_type)
 555         self.set_label(k, name)
 556         self.pending_accounts[k] = addr
 557         self.storage.put('pending_accounts', self.pending_accounts)
 558
 559     def get_pending_accounts(self):
 560         return self.pending_accounts.items()
 561
 562
 563     def addresses(self, include_change = True, _next=True):
 564         o = self.get_account_addresses(-1, include_change)
 565         for a in self.accounts.keys():
 566             o += self.get_account_addresses(a, include_change)
 567
 568         if _next:
 569             for addr in self.next_addresses.values():
 570                 if addr not in o:
 571                     o += [addr]
 572         return o
 573
 574
 575     def is_mine(self, address):
 576         return address in self.addresses(True)
 577
 578
 579     def is_change(self, address):
 580         if not self.is_mine(address): return False
 581         if address in self.imported_keys.keys(): return False
 582         acct, s = self.get_address_index(address)
 583         if s is None: return False
 584         return s[0] == 1
 585
 586     def get_master_public_key(self):
 587         if self.seed_version == 4:
 588             return self.storage.get("master_public_key")
 589         else:
 590             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 591             return repr((c, K))
 592
 593     def get_master_private_key(self, account, password):
 594         master_k = pw_decode( self.master_private_keys[account], password)
 595         master_c, master_K, master_Kc = self.master_public_keys[account]
 596         try:
 597             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 598             assert K.encode('hex') == master_K
 599         except:
 600             raise BaseException("Invalid password")
 601         return master_k
 602
 603
 604     def get_address_index(self, address):
 605         if address in self.imported_keys.keys():
 606             return -1, None
 607
 608         for account in self.accounts.keys():
 609             for for_change in [0,1]:
 610                 addresses = self.accounts[account].get_addresses(for_change)
 611                 for addr in addresses:
 612                     if address == addr:
 613                         return account, (for_change, addresses.index(addr))
 614
 615         for k,v in self.next_addresses.items():
 616             if v == address:
 617                 return k, (0,0)
 618
 619         raise BaseException("Address not found", address)
 620
 621
 622     def get_roots(self, account):
 623         roots = []
 624         for a in account.split('&'):
 625             s = a.strip()
 626             m = re.match("(m/\d+'/)(\d+)", s)
 627             roots.append( m.group(1) )
 628         return roots
 629
 630     def is_seeded(self, account):
 631         if type(account) is int:
 632             return self.seed is not None
 633
 634         for root in self.get_roots(account):
 635             if root not in self.master_private_keys.keys():
 636                 return False
 637         return True
 638
 639     def rebase_sequence(self, account, sequence):
 640         c, i = sequence
 641         dd = []
 642         for a in account.split('&'):
 643             s = a.strip()
 644             m = re.match("(m/\d+'/)(\d+)", s)
 645             root = m.group(1)
 646             num = int(m.group(2))
 647             dd.append( (root, [num,c,i] ) )
 648         return dd
 649
 650
 651     def get_keyID(self, account, sequence):
 652         if account == 0:
 653             return 'old'
 654
 655         rs = self.rebase_sequence(account, sequence)
 656         dd = []
 657         for root, public_sequence in rs:
 658             c, K, _ = self.master_public_keys[root]
 659             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 660             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 661         return '&'.join(dd)
 662
 663
 664
 665     def get_seed(self, password):
 666         s = pw_decode(self.seed, password)
 667         if self.seed_version == 4:
 668             seed = s
 669         else:
 670             seed = mnemonic_hash(s)
 671         #todo:  #self.sequences[0].check_seed(seed)
 672         return seed
 673
 674
 675     def get_mnemonic(self, password):
 676         import mnemonic
 677         s = pw_decode(self.seed, password)
 678         if self.seed_version == 4:
 679             return ' '.join(mnemonic.mn_encode(s))
 680         else:
 681             return s
 682
 683
 684
 685     def get_private_key(self, address, password):
 686         out = []
 687         if address in self.imported_keys.keys():
 688             out.append( pw_decode( self.imported_keys[address], password ) )
 689         else:
 690             account, sequence = self.get_address_index(address)
 691             if account == 0:
 692                 seed = self.get_seed(password)
 693                 pk = self.accounts[account].get_private_key(seed, sequence)
 694                 out.append(pk)
 695                 return out
 696
 697             # assert address == self.accounts[account].get_address(*sequence)
 698             rs = self.rebase_sequence( account, sequence)
 699             for root, public_sequence in rs:
 700
 701                 if root not in self.master_private_keys.keys(): continue
 702                 master_k = self.get_master_private_key(root, password)
 703                 master_c, _, _ = self.master_public_keys[root]
 704                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 705                 out.append(pk)
 706
 707         return out
 708
 709
 710     def add_keypairs_from_wallet(self, tx, keypairs, password):
 711         for txin in tx.inputs:
 712             address = txin['address']
 713             private_keys = self.get_private_key(address, password)
 714             for sec in private_keys:
 715                 pubkey = public_key_from_private_key(sec)
 716                 keypairs[ pubkey ] = sec
 717
 718
 719     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 720         for txin in tx.inputs:
 721             keyid = txin.get('KeyID')
 722             if keyid:
 723                 roots = []
 724                 for s in keyid.split('&'):
 725                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 726                     if not m: continue
 727                     c = m.group(1)
 728                     K = m.group(2)
 729                     sequence = m.group(3)
 730                     root = self.find_root_by_master_key(c,K)
 731                     if not root: continue
 732                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 733                     root = root + '%d'%sequence[0]
 734                     sequence = sequence[1:]
 735                     roots.append((root,sequence))
 736
 737                 account_id = " & ".join( map(lambda x:x[0], roots) )
 738                 account = self.accounts.get(account_id)
 739                 if not account: continue
 740                 addr = account.get_address(*sequence)
 741                 txin['address'] = addr # fixme: side effect
 742                 pk = self.get_private_key(addr, password)
 743                 for sec in pk:
 744                     pubkey = public_key_from_private_key(sec)
 745                     keypairs[pubkey] = sec
 746
 747
 748
 749     def signrawtransaction(self, tx, input_info, private_keys, password):
 750
 751         # check that the password is correct
 752         seed = self.get_seed(password)
 753
 754         # add input info
 755         tx.add_input_info(input_info)
 756
 757         # add redeem script for coins that are in the wallet
 758         # FIXME: add redeemPubkey too!
 759         unspent_coins = self.get_unspent_coins()
 760         for txin in tx.inputs:
 761             for item in unspent_coins:
 762                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 763                     print_error( "tx input is in unspent coins" )
 764                     txin['scriptPubKey'] = item['scriptPubKey']
 765                     account, sequence = self.get_address_index(item['address'])
 766                     if account != -1:
 767                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 768                         print_error("added redeemScript", txin['redeemScript'])
 769                     break
 770
 771
 772         # build a list of public/private keys
 773         keypairs = {}
 774
 775         # add private keys from parameter
 776         for sec in private_keys:
 777             pubkey = public_key_from_private_key(sec)
 778             keypairs[ pubkey ] = sec
 779
 780         # add private_keys from KeyID
 781         self.add_keypairs_from_KeyID(tx, keypairs, password)
 782
 783         # add private keys from wallet
 784         self.add_keypairs_from_wallet(tx, keypairs, password)
 785         self.sign_transaction(tx, keypairs)
 786
 787
 788     def sign_message(self, address, message, password):
 789         keys = self.get_private_key(address, password)
 790         assert len(keys) == 1
 791         sec = keys[0]
 792         key = regenerate_key(sec)
 793         compressed = is_compressed(sec)
 794         return key.sign_message(message, compressed, address)
 795
 796
 797     def change_gap_limit(self, value):
 798         if value >= self.gap_limit:
 799             self.gap_limit = value
 800             self.storage.put('gap_limit', self.gap_limit, True)
 801             #self.interface.poke('synchronizer')
 802             return True
 803
 804         elif value >= self.min_acceptable_gap():
 805             for key, account in self.accounts.items():
 806                 addresses = account[0]
 807                 k = self.num_unused_trailing_addresses(addresses)
 808                 n = len(addresses) - k + value
 809                 addresses = addresses[0:n]
 810                 self.accounts[key][0] = addresses
 811
 812             self.gap_limit = value
 813             self.storage.put('gap_limit', self.gap_limit, True)
 814             self.save_accounts()
 815             return True
 816         else:
 817             return False
 818
 819     def num_unused_trailing_addresses(self, addresses):
 820         k = 0
 821         for a in addresses[::-1]:
 822             if self.history.get(a):break
 823             k = k + 1
 824         return k
 825
 826     def min_acceptable_gap(self):
 827         # fixme: this assumes wallet is synchronized
 828         n = 0
 829         nmax = 0
 830
 831         for account in self.accounts.values():
 832             addresses = account.get_addresses(0)
 833             k = self.num_unused_trailing_addresses(addresses)
 834             for a in addresses[0:-k]:
 835                 if self.history.get(a):
 836                     n = 0
 837                 else:
 838                     n += 1
 839                     if n > nmax: nmax = n
 840         return nmax + 1
 841
 842
 843     def address_is_old(self, address):
 844         age = -1
 845         h = self.history.get(address, [])
 846         if h == ['*']:
 847             return True
 848         for tx_hash, tx_height in h:
 849             if tx_height == 0:
 850                 tx_age = 0
 851             else:
 852                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 853             if tx_age > age:
 854                 age = tx_age
 855         return age > 2
 856
 857
 858     def synchronize_sequence(self, account, for_change):
 859         limit = self.gap_limit_for_change if for_change else self.gap_limit
 860         new_addresses = []
 861         while True:
 862             addresses = account.get_addresses(for_change)
 863             if len(addresses) < limit:
 864                 address = account.create_new_address(for_change)
 865                 self.history[address] = []
 866                 new_addresses.append( address )
 867                 continue
 868
 869             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 870                 break
 871             else:
 872                 address = account.create_new_address(for_change)
 873                 self.history[address] = []
 874                 new_addresses.append( address )
 875
 876         return new_addresses
 877
 878
 879
 880     def create_pending_accounts(self):
 881         for account_type in ['1','2of2','2of3']:
 882             if not self.has_master_public_keys(account_type):
 883                 continue
 884             k, a = self.new_account_address(account_type)
 885             if self.address_is_old(a):
 886                 print_error( "creating account", a )
 887                 self.create_account(account_type)
 888                 self.next_addresses.pop(k)
 889
 890
 891     def synchronize_account(self, account):
 892         new = []
 893         new += self.synchronize_sequence(account, 0)
 894         new += self.synchronize_sequence(account, 1)
 895         return new
 896
 897
 898     def synchronize(self):
 899         if self.master_public_keys:
 900             self.create_pending_accounts()
 901         new = []
 902         for account in self.accounts.values():
 903             new += self.synchronize_account(account)
 904         if new:
 905             self.save_accounts()
 906             self.storage.put('addr_history', self.history, True)
 907         return new
 908
 909
 910     def is_found(self):
 911         return self.history.values() != [[]] * len(self.history)
 912
 913
 914     def add_contact(self, address, label=None):
 915         self.addressbook.append(address)
 916         self.storage.put('contacts', self.addressbook, True)
 917         if label:
 918             self.set_label(address, label)
 919
 920
 921     def delete_contact(self, addr):
 922         if addr in self.addressbook:
 923             self.addressbook.remove(addr)
 924             self.storage.put('addressbook', self.addressbook, True)
 925
 926
 927     def fill_addressbook(self):
 928         for tx_hash, tx in self.transactions.items():
 929             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 930             if is_send:
 931                 for addr, v in tx.outputs:
 932                     if not self.is_mine(addr) and addr not in self.addressbook:
 933                         self.addressbook.append(addr)
 934         # redo labels
 935         # self.update_tx_labels()
 936
 937     def get_num_tx(self, address):
 938         n = 0
 939         for tx in self.transactions.values():
 940             if address in map(lambda x:x[0], tx.outputs): n += 1
 941         return n
 942
 943
 944     def get_address_flags(self, addr):
 945         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 946         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 947         return flags
 948
 949
 950     def get_tx_value(self, tx, account=None):
 951         domain = self.get_account_addresses(account)
 952         return tx.get_value(domain, self.prevout_values)
 953
 954
 955     def update_tx_outputs(self, tx_hash):
 956         tx = self.transactions.get(tx_hash)
 957
 958         for i, (addr, value) in enumerate(tx.outputs):
 959             key = tx_hash+ ':%d'%i
 960             self.prevout_values[key] = value
 961
 962         for item in tx.inputs:
 963             if self.is_mine(item.get('address')):
 964                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 965                 self.spent_outputs.append(key)
 966
 967
 968     def get_addr_balance(self, address):
 969         assert self.is_mine(address)
 970         h = self.history.get(address,[])
 971         if h == ['*']: return 0,0
 972         c = u = 0
 973         received_coins = []   # list of coins received at address
 974
 975         for tx_hash, tx_height in h:
 976             tx = self.transactions.get(tx_hash)
 977             if not tx: continue
 978
 979             for i, (addr, value) in enumerate(tx.outputs):
 980                 if addr == address:
 981                     key = tx_hash + ':%d'%i
 982                     received_coins.append(key)
 983
 984         for tx_hash, tx_height in h:
 985             tx = self.transactions.get(tx_hash)
 986             if not tx: continue
 987             v = 0
 988
 989             for item in tx.inputs:
 990                 addr = item.get('address')
 991                 if addr == address:
 992                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 993                     value = self.prevout_values.get( key )
 994                     if key in received_coins:
 995                         v -= value
 996
 997             for i, (addr, value) in enumerate(tx.outputs):
 998                 key = tx_hash + ':%d'%i
 999                 if addr == address:
1000                     v += value
1001
1002             if tx_height:
1003                 c += v
1004             else:
1005                 u += v
1006         return c, u
1007
1008
1009     def get_account_name(self, k):
1010         if k == 0:
1011             if self.seed_version == 4:
1012                 name = 'Main account'
1013             else:
1014                 name = 'Old account'
1015         else:
1016             default = "Unnamed account"
1017             m = re.match("m/0'/(\d+)", k)
1018             if m:
1019                 num = m.group(1)
1020                 if num == '0':
1021                     default = "Main account"
1022                 else:
1023                     default = "Account %s"%num
1024
1025             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1026             if m:
1027                 num = m.group(1)
1028                 default = "2of2 account %s"%num
1029             name = self.labels.get(k, default)
1030
1031         return name
1032
1033     def get_account_names(self):
1034         accounts = {}
1035         for k, account in self.accounts.items():
1036             accounts[k] = self.get_account_name(k)
1037         if self.imported_keys:
1038             accounts[-1] = 'Imported keys'
1039         return accounts
1040
1041     def get_account_addresses(self, a, include_change=True):
1042         if a is None:
1043             o = self.addresses(True)
1044         elif a == -1:
1045             o = self.imported_keys.keys()
1046         else:
1047             ac = self.accounts[a]
1048             o = ac.get_addresses(0)
1049             if include_change: o += ac.get_addresses(1)
1050         return o
1051
1052     def get_imported_balance(self):
1053         return self.get_balance(self.imported_keys.keys())
1054
1055     def get_account_balance(self, account):
1056         return self.get_balance(self.get_account_addresses(account))
1057
1058     def get_frozen_balance(self):
1059         return self.get_balance(self.frozen_addresses)
1060
1061     def get_balance(self, domain=None):
1062         if domain is None: domain = self.addresses(True)
1063         cc = uu = 0
1064         for addr in domain:
1065             c, u = self.get_addr_balance(addr)
1066             cc += c
1067             uu += u
1068         return cc, uu
1069
1070
1071     def get_unspent_coins(self, domain=None):
1072         coins = []
1073         if domain is None: domain = self.addresses(True)
1074         for addr in domain:
1075             h = self.history.get(addr, [])
1076             if h == ['*']: continue
1077             for tx_hash, tx_height in h:
1078                 tx = self.transactions.get(tx_hash)
1079                 if tx is None: raise BaseException("Wallet not synchronized")
1080                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1081                 for output in tx.d.get('outputs'):
1082                     if output.get('address') != addr: continue
1083                     key = tx_hash + ":%d" % output.get('prevout_n')
1084                     if key in self.spent_outputs: continue
1085                     output['prevout_hash'] = tx_hash
1086                     output['height'] = tx_height
1087                     output['coinbase'] = is_coinbase
1088                     coins.append((tx_height, output))
1089
1090         # sort by age
1091         if coins:
1092             coins = sorted(coins)
1093             if coins[-1][0] != 0:
1094                 while coins[0][0] == 0:
1095                     coins = coins[1:] + [ coins[0] ]
1096         return [x[1] for x in coins]
1097
1098
1099
1100     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1101         domain = self.get_account_addresses(account) if account else None
1102         return self.choose_tx_inputs( amount, fixed_fee, domain )
1103
1104
1105     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1106         """ todo: minimize tx size """
1107         total = 0
1108         fee = self.fee if fixed_fee is None else fixed_fee
1109         if domain is None:
1110             domain = self.addresses(True)
1111
1112         for i in self.frozen_addresses:
1113             if i in domain: domain.remove(i)
1114
1115         prioritized = []
1116         for i in self.prioritized_addresses:
1117             if i in domain:
1118                 domain.remove(i)
1119                 prioritized.append(i)
1120
1121         coins = self.get_unspent_coins(domain)
1122         prioritized_coins = self.get_unspent_coins(prioritized)
1123
1124         inputs = []
1125         coins = prioritized_coins + coins
1126
1127         for item in coins:
1128             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1129                 continue
1130             addr = item.get('address')
1131             v = item.get('value')
1132             total += v
1133             inputs.append(item)
1134             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1135             if total >= amount + fee: break
1136         else:
1137             inputs = []
1138
1139         return inputs, total, fee
1140
1141
1142     def set_fee(self, fee):
1143         if self.fee != fee:
1144             self.fee = fee
1145             self.storage.put('fee_per_kb', self.fee, True)
1146
1147     def estimated_fee(self, inputs):
1148         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1149         fee = self.fee * int(round(estimated_size/1024.))
1150         if fee == 0: fee = self.fee
1151         return fee
1152
1153
1154     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1155         "add change to a transaction"
1156         change_amount = total - ( amount + fee )
1157         if change_amount != 0:
1158             if not change_addr:
1159
1160                 # send change to one of the accounts involved in the tx
1161                 address = inputs[0].get('address')
1162                 account, _ = self.get_address_index(address)
1163
1164                 if not self.use_change or account == -1:
1165                     change_addr = inputs[-1]['address']
1166                 else:
1167                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1168
1169             # Insert the change output at a random position in the outputs
1170             posn = random.randint(0, len(outputs))
1171             outputs[posn:posn] = [( change_addr,  change_amount)]
1172         return outputs
1173
1174
1175     def get_history(self, address):
1176         with self.lock:
1177             return self.history.get(address)
1178
1179
1180     def get_status(self, h):
1181         if not h: return None
1182         if h == ['*']: return '*'
1183         status = ''
1184         for tx_hash, height in h:
1185             status += tx_hash + ':%d:' % height
1186         return hashlib.sha256( status ).digest().encode('hex')
1187
1188
1189     def receive_tx_callback(self, tx_hash, tx, tx_height):
1190
1191         with self.transaction_lock:
1192             self.add_extra_addresses(tx)
1193             if not self.check_new_tx(tx_hash, tx):
1194                 # may happen due to pruning
1195                 print_error("received transaction that is no longer referenced in history", tx_hash)
1196                 return
1197             self.transactions[tx_hash] = tx
1198             self.network.interface.pending_transactions_for_notifications.append(tx)
1199             self.save_transactions()
1200             if self.verifier and tx_height>0:
1201                 self.verifier.add(tx_hash, tx_height)
1202             self.update_tx_outputs(tx_hash)
1203
1204
1205     def save_transactions(self):
1206         tx = {}
1207         for k,v in self.transactions.items():
1208             tx[k] = str(v)
1209         self.storage.put('transactions', tx, True)
1210
1211     def receive_history_callback(self, addr, hist):
1212
1213         if not self.check_new_history(addr, hist):
1214             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1215
1216         with self.lock:
1217             self.history[addr] = hist
1218             self.storage.put('addr_history', self.history, True)
1219
1220         if hist != ['*']:
1221             for tx_hash, tx_height in hist:
1222                 if tx_height>0:
1223                     # add it in case it was previously unconfirmed
1224                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1225
1226
1227     def get_tx_history(self, account=None):
1228         with self.transaction_lock:
1229             history = self.transactions.items()
1230             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1231             result = []
1232
1233             balance = 0
1234             for tx_hash, tx in history:
1235                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1236                 if v is not None: balance += v
1237
1238             c, u = self.get_account_balance(account)
1239
1240             if balance != c+u:
1241                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1242
1243             balance = c + u - balance
1244             for tx_hash, tx in history:
1245                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1246                 if not is_relevant:
1247                     continue
1248                 if value is not None:
1249                     balance += value
1250
1251                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1252                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1253
1254         return result
1255
1256
1257     def get_label(self, tx_hash):
1258         label = self.labels.get(tx_hash)
1259         is_default = (label == '') or (label is None)
1260         if is_default: label = self.get_default_label(tx_hash)
1261         return label, is_default
1262
1263
1264     def get_default_label(self, tx_hash):
1265         tx = self.transactions.get(tx_hash)
1266         default_label = ''
1267         if tx:
1268             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1269             if is_mine:
1270                 for o in tx.outputs:
1271                     o_addr, _ = o
1272                     if not self.is_mine(o_addr):
1273                         try:
1274                             default_label = self.labels[o_addr]
1275                         except KeyError:
1276                             default_label = o_addr
1277                         break
1278                 else:
1279                     default_label = '(internal)'
1280             else:
1281                 for o in tx.outputs:
1282                     o_addr, _ = o
1283                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1284                         break
1285                 else:
1286                     for o in tx.outputs:
1287                         o_addr, _ = o
1288                         if self.is_mine(o_addr):
1289                             break
1290                     else:
1291                         o_addr = None
1292
1293                 if o_addr:
1294                     dest_label = self.labels.get(o_addr)
1295                     try:
1296                         default_label = self.labels[o_addr]
1297                     except KeyError:
1298                         default_label = o_addr
1299
1300         return default_label
1301
1302
1303     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1304         for address, x in outputs:
1305             assert is_valid(address)
1306         amount = sum( map(lambda x:x[1], outputs) )
1307         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1308         if not inputs:
1309             raise ValueError("Not enough funds")
1310         self.add_input_info(inputs)
1311         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1312         return Transaction.from_io(inputs, outputs)
1313
1314
1315     def mktx_from_account(self, outputs, password, fee=None, account=None):
1316         domain = self.get_account_addresses(account) if account else None
1317         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1318
1319
1320     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1321         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1322         keypairs = {}
1323         self.add_keypairs_from_wallet(tx, keypairs, password)
1324         if keypairs:
1325             self.sign_transaction(tx, keypairs)
1326         return tx
1327
1328
1329     def add_input_info(self, inputs):
1330         for txin in inputs:
1331             address = txin['address']
1332             if address in self.imported_keys.keys():
1333                 continue
1334             account, sequence = self.get_address_index(address)
1335             txin['KeyID'] = self.get_keyID(account, sequence)
1336             redeemScript = self.accounts[account].redeem_script(sequence)
1337             if redeemScript:
1338                 txin['redeemScript'] = redeemScript
1339             else:
1340                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1341
1342
1343     def sign_transaction(self, tx, keypairs):
1344         tx.sign(keypairs)
1345         run_hook('sign_transaction', tx)
1346
1347
1348     def sendtx(self, tx):
1349         # synchronous
1350         h = self.send_tx(tx)
1351         self.tx_event.wait()
1352         return self.receive_tx(h)
1353
1354     def send_tx(self, tx):
1355         # asynchronous
1356         self.tx_event.clear()
1357         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1358         return tx.hash()
1359
1360     def on_broadcast(self, i, r):
1361         self.tx_result = r.get('result')
1362         self.tx_event.set()
1363
1364     def receive_tx(self,tx_hash):
1365         out = self.tx_result
1366         if out != tx_hash:
1367             return False, "error: " + out
1368         return True, out
1369
1370
1371
1372     def update_password(self, old_password, new_password):
1373         if new_password == '': new_password = None
1374         # this will throw an exception if unicode cannot be converted
1375         decoded = pw_decode(self.seed, old_password)
1376         self.seed = pw_encode( decoded, new_password)
1377         self.storage.put('seed', self.seed, True)
1378         self.use_encryption = (new_password != None)
1379         self.storage.put('use_encryption', self.use_encryption,True)
1380         for k in self.imported_keys.keys():
1381             a = self.imported_keys[k]
1382             b = pw_decode(a, old_password)
1383             c = pw_encode(b, new_password)
1384             self.imported_keys[k] = c
1385         self.storage.put('imported_keys', self.imported_keys, True)
1386
1387         for k, v in self.master_private_keys.items():
1388             b = pw_decode(v, old_password)
1389             c = pw_encode(b, new_password)
1390             self.master_private_keys[k] = c
1391         self.storage.put('master_private_keys', self.master_private_keys, True)
1392
1393
1394     def freeze(self,addr):
1395         if self.is_mine(addr) and addr not in self.frozen_addresses:
1396             self.unprioritize(addr)
1397             self.frozen_addresses.append(addr)
1398             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1399             return True
1400         else:
1401             return False
1402
1403     def unfreeze(self,addr):
1404         if self.is_mine(addr) and addr in self.frozen_addresses:
1405             self.frozen_addresses.remove(addr)
1406             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1407             return True
1408         else:
1409             return False
1410
1411     def prioritize(self,addr):
1412         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1413             self.unfreeze(addr)
1414             self.prioritized_addresses.append(addr)
1415             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1416             return True
1417         else:
1418             return False
1419
1420     def unprioritize(self,addr):
1421         if self.is_mine(addr) and addr in self.prioritized_addresses:
1422             self.prioritized_addresses.remove(addr)
1423             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1424             return True
1425         else:
1426             return False
1427
1428
1429     def set_verifier(self, verifier):
1430         self.verifier = verifier
1431
1432         # review transactions that are in the history
1433         for addr, hist in self.history.items():
1434             if hist == ['*']: continue
1435             for tx_hash, tx_height in hist:
1436                 if tx_height>0:
1437                     # add it in case it was previously unconfirmed
1438                     self.verifier.add(tx_hash, tx_height)
1439
1440
1441         # if we are on a pruning server, remove unverified transactions
1442         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1443         for tx_hash in self.transactions.keys():
1444             if tx_hash not in vr:
1445                 self.transactions.pop(tx_hash)
1446
1447
1448
1449     def check_new_history(self, addr, hist):
1450
1451         # check that all tx in hist are relevant
1452         if hist != ['*']:
1453             for tx_hash, height in hist:
1454                 tx = self.transactions.get(tx_hash)
1455                 if not tx: continue
1456                 if not tx.has_address(addr):
1457                     return False
1458
1459         # check that we are not "orphaning" a transaction
1460         old_hist = self.history.get(addr,[])
1461         if old_hist == ['*']: return True
1462
1463         for tx_hash, height in old_hist:
1464             if tx_hash in map(lambda x:x[0], hist): continue
1465             found = False
1466             for _addr, _hist in self.history.items():
1467                 if _addr == addr: continue
1468                 if _hist == ['*']: continue
1469                 _tx_hist = map(lambda x:x[0], _hist)
1470                 if tx_hash in _tx_hist:
1471                     found = True
1472                     break
1473
1474             if not found:
1475                 tx = self.transactions.get(tx_hash)
1476                 # tx might not be there
1477                 if not tx: continue
1478
1479                 # already verified?
1480                 if self.verifier.get_height(tx_hash):
1481                     continue
1482                 # unconfirmed tx
1483                 print_error("new history is orphaning transaction:", tx_hash)
1484                 # check that all outputs are not mine, request histories
1485                 ext_requests = []
1486                 for _addr, _v in tx.outputs:
1487                     # assert not self.is_mine(_addr)
1488                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1489
1490                 ext_h = self.network.synchronous_get(ext_requests)
1491                 print_error("sync:", ext_requests, ext_h)
1492                 height = None
1493                 for h in ext_h:
1494                     if h == ['*']: continue
1495                     for item in h:
1496                         if item.get('tx_hash') == tx_hash:
1497                             height = item.get('height')
1498                 if height:
1499                     print_error("found height for", tx_hash, height)
1500                     self.verifier.add(tx_hash, height)
1501                 else:
1502                     print_error("removing orphaned tx from history", tx_hash)
1503                     self.transactions.pop(tx_hash)
1504
1505         return True
1506
1507
1508
1509     def check_new_tx(self, tx_hash, tx):
1510         # 1 check that tx is referenced in addr_history.
1511         addresses = []
1512         for addr, hist in self.history.items():
1513             if hist == ['*']:continue
1514             for txh, height in hist:
1515                 if txh == tx_hash:
1516                     addresses.append(addr)
1517
1518         if not addresses:
1519             return False
1520
1521         # 2 check that referencing addresses are in the tx
1522         for addr in addresses:
1523             if not tx.has_address(addr):
1524                 return False
1525
1526         return True
1527
1528
1529     def start_threads(self, network):
1530         from verifier import TxVerifier
1531         self.network = network
1532         self.verifier = TxVerifier(self.network, self.storage)
1533         self.verifier.start()
1534         self.set_verifier(self.verifier)
1535         self.synchronizer = WalletSynchronizer(self, network)
1536         self.synchronizer.start()
1537
1538     def stop_threads(self):
1539         self.verifier.stop()
1540         self.synchronizer.stop()
1541
1542
1543
1544     def restore(self, callback):
1545         from i18n import _
1546         def wait_for_wallet():
1547             self.set_up_to_date(False)
1548             while not self.is_up_to_date():
1549                 msg = "%s\n%s %d\n%s %.1f"%(
1550                     _("Please wait..."),
1551                     _("Addresses generated:"),
1552                     len(self.addresses(True)),_("Kilobytes received:"),
1553                     self.network.interface.bytes_received/1024.)
1554
1555                 apply(callback, (msg,))
1556                 time.sleep(0.1)
1557
1558         def wait_for_network():
1559             while not self.network.interface.is_connected:
1560                 msg = "%s \n" % (_("Connecting..."))
1561                 apply(callback, (msg,))
1562                 time.sleep(0.1)
1563
1564         # wait until we are connected, because the user might have selected another server
1565         wait_for_network()
1566
1567
1568         self.create_accounts()
1569         wait_for_wallet()
1570
1571
1572
1573
1574 class WalletSynchronizer(threading.Thread):
1575
1576
1577     def __init__(self, wallet, network):
1578         threading.Thread.__init__(self)
1579         self.daemon = True
1580         self.wallet = wallet
1581         self.network = network
1582         self.was_updated = True
1583         self.running = False
1584         self.lock = threading.Lock()
1585         self.queue = Queue.Queue()
1586
1587     def stop(self):
1588         with self.lock: self.running = False
1589
1590     def is_running(self):
1591         with self.lock: return self.running
1592
1593
1594     def subscribe_to_addresses(self, addresses):
1595         messages = []
1596         for addr in addresses:
1597             messages.append(('blockchain.address.subscribe', [addr]))
1598         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1599
1600
1601     def run(self):
1602         with self.lock:
1603             self.running = True
1604
1605         while self.is_running():
1606
1607             if not self.network.is_connected():
1608                 print_error("synchronizer: waiting for interface")
1609                 self.network.wait_until_connected()
1610
1611             self.run_interface(self.network.interface)
1612
1613
1614     def run_interface(self, interface):
1615
1616         print_error("synchronizer: connected to", interface.server)
1617
1618         requested_tx = []
1619         missing_tx = []
1620         requested_histories = {}
1621
1622         # request any missing transactions
1623         for history in self.wallet.history.values():
1624             if history == ['*']: continue
1625             for tx_hash, tx_height in history:
1626                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1627                     missing_tx.append( (tx_hash, tx_height) )
1628
1629         if missing_tx:
1630             print_error("missing tx", missing_tx)
1631
1632         # subscriptions
1633         self.subscribe_to_addresses(self.wallet.addresses(True))
1634
1635         while self.is_running():
1636             # 1. create new addresses
1637             new_addresses = self.wallet.synchronize()
1638
1639             # request missing addresses
1640             if new_addresses:
1641                 self.subscribe_to_addresses(new_addresses)
1642
1643             # request missing transactions
1644             for tx_hash, tx_height in missing_tx:
1645                 if (tx_hash, tx_height) not in requested_tx:
1646                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1647                     requested_tx.append( (tx_hash, tx_height) )
1648             missing_tx = []
1649
1650             # detect if situation has changed
1651             if interface.is_up_to_date() and self.queue.empty():
1652                 if not self.wallet.is_up_to_date():
1653                     self.wallet.set_up_to_date(True)
1654                     self.was_updated = True
1655             else:
1656                 if self.wallet.is_up_to_date():
1657                     self.wallet.set_up_to_date(False)
1658                     self.was_updated = True
1659
1660             if self.was_updated:
1661                 self.wallet.network.trigger_callback('updated')
1662                 self.was_updated = False
1663
1664             # 2. get a response
1665             try:
1666                 r = self.queue.get(block=True, timeout=1)
1667             except Queue.Empty:
1668                 continue
1669
1670             if interface != self.network.interface:
1671                 break
1672
1673             if not r:
1674                 continue
1675
1676             # 3. handle response
1677             method = r['method']
1678             params = r['params']
1679             result = r.get('result')
1680             error = r.get('error')
1681             if error:
1682                 print "error", r
1683                 continue
1684
1685             if method == 'blockchain.address.subscribe':
1686                 addr = params[0]
1687                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1688                     if requested_histories.get(addr) is None:
1689                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1690                         requested_histories[addr] = result
1691
1692             elif method == 'blockchain.address.get_history':
1693                 addr = params[0]
1694                 print_error("receiving history", addr, result)
1695                 if result == ['*']:
1696                     assert requested_histories.pop(addr) == '*'
1697                     self.wallet.receive_history_callback(addr, result)
1698                 else:
1699                     hist = []
1700                     # check that txids are unique
1701                     txids = []
1702                     for item in result:
1703                         tx_hash = item['tx_hash']
1704                         if tx_hash not in txids:
1705                             txids.append(tx_hash)
1706                             hist.append( (tx_hash, item['height']) )
1707
1708                     if len(hist) != len(result):
1709                         raise BaseException("error: server sent history with non-unique txid", result)
1710
1711                     # check that the status corresponds to what was announced
1712                     rs = requested_histories.pop(addr)
1713                     if self.wallet.get_status(hist) != rs:
1714                         raise BaseException("error: status mismatch: %s"%addr)
1715
1716                     # store received history
1717                     self.wallet.receive_history_callback(addr, hist)
1718
1719                     # request transactions that we don't have
1720                     for tx_hash, tx_height in hist:
1721                         if self.wallet.transactions.get(tx_hash) is None:
1722                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1723                                 missing_tx.append( (tx_hash, tx_height) )
1724
1725             elif method == 'blockchain.transaction.get':
1726                 tx_hash = params[0]
1727                 tx_height = params[1]
1728                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1729                 tx = Transaction(result)
1730                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1731                 self.was_updated = True
1732                 requested_tx.remove( (tx_hash, tx_height) )
1733                 print_error("received tx:", tx_hash, len(tx.raw))
1734
1735             else:
1736                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1737
1738             if self.was_updated and not requested_tx:
1739                 self.wallet.network.trigger_callback('updated')
1740                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1741
1742                 self.was_updated = False