lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40
  41 # AES encryption
  42 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  43 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  44
  45 def pw_encode(s, password):
  46     if password:
  47         secret = Hash(password)
  48         return EncodeAES(secret, s)
  49     else:
  50         return s
  51
  52 def pw_decode(s, password):
  53     if password is not None:
  54         secret = Hash(password)
  55         try:
  56             d = DecodeAES(secret, s)
  57         except:
  58             raise BaseException('Invalid password')
  59         return d
  60     else:
  61         return s
  62
  63
  64
  65
  66
  67 from version import *
  68
  69
  70 class WalletStorage:
  71
  72     def __init__(self, config):
  73         self.lock = threading.Lock()
  74         self.data = {}
  75         self.file_exists = False
  76         self.path = self.init_path(config)
  77         print_error( "wallet path", self.path )
  78         if self.path:
  79             self.read(self.path)
  80
  81
  82     def init_path(self, config):
  83         """Set the path of the wallet."""
  84
  85         # command line -w option
  86         path = config.get('wallet_path')
  87         if path:
  88             return path
  89
  90         # path in config file
  91         path = config.get('default_wallet_path')
  92         if path:
  93             return path
  94
  95         # default path
  96         dirpath = os.path.join(config.path, "wallets")
  97         if not os.path.exists(dirpath):
  98             os.mkdir(dirpath)
  99
 100         new_path = os.path.join(config.path, "wallets", "default_wallet")
 101
 102         # default path in pre 1.9 versions
 103         old_path = os.path.join(config.path, "electrum.dat")
 104         if os.path.exists(old_path) and not os.path.exists(new_path):
 105             os.rename(old_path, new_path)
 106
 107         return new_path
 108
 109
 110     def read(self, path):
 111         """Read the contents of the wallet file."""
 112         try:
 113             with open(self.path, "r") as f:
 114                 data = f.read()
 115         except IOError:
 116             return
 117         try:
 118             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 119         except:
 120             raise IOError("Cannot read wallet file.")
 121
 122         self.data = d
 123         self.file_exists = True
 124
 125
 126     def get(self, key, default=None):
 127         return self.data.get(key, default)
 128
 129     def put(self, key, value, save = True):
 130
 131         with self.lock:
 132             if value is not None:
 133                 self.data[key] = value
 134             else:
 135                 self.data.pop[key]
 136             if save:
 137                 self.write()
 138
 139     def write(self):
 140         s = repr(self.data)
 141         f = open(self.path,"w")
 142         f.write( s )
 143         f.close()
 144         if 'ANDROID_DATA' not in os.environ:
 145             import stat
 146             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 147
 148
 149 class Wallet:
 150
 151     def __init__(self, storage):
 152
 153         self.storage = storage
 154         self.electrum_version = ELECTRUM_VERSION
 155         self.gap_limit_for_change = 3 # constant
 156
 157         # saved fields
 158         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 159
 160         self.gap_limit             = storage.get('gap_limit', 5)
 161         self.use_change            = storage.get('use_change',True)
 162         self.use_encryption        = storage.get('use_encryption', False)
 163         self.seed                  = storage.get('seed', '')               # encrypted
 164         self.labels                = storage.get('labels', {})
 165         self.frozen_addresses      = storage.get('frozen_addresses',[])
 166         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 167         self.addressbook           = storage.get('contacts', [])
 168
 169         self.imported_keys         = storage.get('imported_keys',{})
 170         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 171
 172         self.fee                   = int(storage.get('fee_per_kb',20000))
 173
 174         self.master_public_keys = storage.get('master_public_keys',{})
 175         self.master_private_keys = storage.get('master_private_keys', {})
 176
 177         self.next_addresses = storage.get('next_addresses',{})
 178
 179         if self.seed_version not in [4, 6]:
 180             msg = "This wallet seed is not supported."
 181             if self.seed_version in [5]:
 182                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 183                 print msg
 184                 sys.exit(1)
 185
 186
 187         self.load_accounts()
 188
 189         self.transactions = {}
 190         tx_list = self.storage.get('transactions',{})
 191         for k,v in tx_list.items():
 192             try:
 193                 tx = Transaction(v)
 194             except:
 195                 print_msg("Warning: Cannot deserialize transactions. skipping")
 196                 continue
 197
 198             self.add_extra_addresses(tx)
 199             self.transactions[k] = tx
 200
 201         for h,tx in self.transactions.items():
 202             if not self.check_new_tx(h, tx):
 203                 print_error("removing unreferenced tx", h)
 204                 self.transactions.pop(h)
 205
 206
 207         # not saved
 208         self.prevout_values = {}     # my own transaction outputs
 209         self.spent_outputs = []
 210
 211         # spv
 212         self.verifier = None
 213
 214         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 215         # interface.is_up_to_date() returns true when all requests have been answered and processed
 216         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 217
 218         self.up_to_date = False
 219         self.lock = threading.Lock()
 220         self.transaction_lock = threading.Lock()
 221         self.tx_event = threading.Event()
 222
 223         for tx_hash, tx in self.transactions.items():
 224             self.update_tx_outputs(tx_hash)
 225
 226
 227     def add_extra_addresses(self, tx):
 228         h = tx.hash()
 229         # find the address corresponding to pay-to-pubkey inputs
 230         tx.add_extra_addresses(self.transactions)
 231         for o in tx.d.get('outputs'):
 232             if o.get('is_pubkey'):
 233                 for tx2 in self.transactions.values():
 234                     tx2.add_extra_addresses({h:tx})
 235
 236
 237
 238
 239     def set_up_to_date(self,b):
 240         with self.lock: self.up_to_date = b
 241
 242
 243     def is_up_to_date(self):
 244         with self.lock: return self.up_to_date
 245
 246
 247     def update(self):
 248         self.up_to_date = False
 249         while not self.is_up_to_date():
 250             time.sleep(0.1)
 251
 252
 253     def import_key(self, sec, password):
 254         # check password
 255         seed = self.get_seed(password)
 256         try:
 257             address = address_from_private_key(sec)
 258         except:
 259             raise BaseException('Invalid private key')
 260
 261         if self.is_mine(address):
 262             raise BaseException('Address already in wallet')
 263
 264         # store the originally requested keypair into the imported keys table
 265         self.imported_keys[address] = pw_encode(sec, password )
 266         self.storage.put('imported_keys', self.imported_keys, True)
 267         if self.synchronizer:
 268             self.synchronizer.subscribe_to_addresses([address])
 269         return address
 270
 271     def delete_imported_key(self, addr):
 272         if addr in self.imported_keys:
 273             self.imported_keys.pop(addr)
 274             self.storage.put('imported_keys', self.imported_keys, True)
 275
 276
 277     def make_seed(self):
 278         import mnemonic, ecdsa
 279         entropy = ecdsa.util.randrange( pow(2,160) )
 280         nonce = 0
 281         while True:
 282             ss = "%040x"%(entropy+nonce)
 283             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 284             # we keep only 13 words, that's approximately 139 bits of entropy
 285             words = mnemonic.mn_encode(s)[0:13]
 286             seed = ' '.join(words)
 287             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 288                 break  # this removes 12 bits of entropy
 289             nonce += 1
 290
 291         return seed
 292
 293
 294     def init_seed(self, seed):
 295         import mnemonic
 296
 297         if self.seed:
 298             raise BaseException("a seed exists")
 299
 300         if not seed:
 301             self.seed = random_seed(128)
 302             self.seed_version = 4
 303             return
 304
 305         #if not seed:
 306         #    self.seed = self.make_seed()
 307         #    self.seed_version = SEED_VERSION
 308         #    return
 309
 310         # find out what kind of wallet we are
 311         try:
 312             seed.decode('hex')
 313             self.seed_version = 4
 314             return
 315         except:
 316             pass
 317
 318         words = seed.split()
 319         self.seed_version = 4
 320         self.seed = mnemonic.mn_decode(words)
 321
 322         #try:
 323         #    mnemonic.mn_decode(words)
 324         #    uses_electrum_words = True
 325         #except:
 326         #    uses_electrum_words = False
 327         #
 328         #if uses_electrum_words and len(words) != 13:
 329         #    self.seed_version = 4
 330         #    self.seed = mnemonic.mn_decode(words)
 331         #else:
 332         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 333         #    self.seed_version = SEED_VERSION
 334         #    self.seed = seed
 335
 336
 337     def save_seed(self):
 338         self.storage.put('seed', self.seed, True)
 339         self.storage.put('seed_version', self.seed_version, True)
 340
 341     def create_watching_only_wallet(self, c0, K0):
 342         cK0 = ""
 343         self.master_public_keys = {
 344             "m/0'/": (c0, K0, cK0),
 345             }
 346         self.storage.put('master_public_keys', self.master_public_keys, True)
 347         self.create_account('1','Main account')
 348
 349
 350     def create_accounts(self):
 351         if self.seed_version == 4:
 352             self.create_old_account()
 353         else:
 354             # create default account
 355             self.create_master_keys('1')
 356             self.create_account('1','Main account')
 357
 358
 359     def create_master_keys(self, account_type):
 360         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 361         if account_type == '1':
 362             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 363             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 364             self.master_private_keys["m/0'/"] = k0
 365         elif account_type == '2of2':
 366             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 367             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 368             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 369             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 370             self.master_private_keys["m/1'/"] = k1
 371             self.master_private_keys["m/2'/"] = k2
 372         elif account_type == '2of3':
 373             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 374             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 375             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 376             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 377             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 378             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 379             self.master_private_keys["m/3'/"] = k3
 380             self.master_private_keys["m/4'/"] = k4
 381             self.master_private_keys["m/5'/"] = k5
 382
 383         self.storage.put('master_public_keys', self.master_public_keys, True)
 384         self.storage.put('master_private_keys', self.master_private_keys, True)
 385
 386     def has_master_public_keys(self, account_type):
 387         if account_type == '1':
 388             return "m/0'/" in self.master_public_keys
 389         elif account_type == '2of2':
 390             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 391         elif account_type == '2of3':
 392             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 393
 394     def find_root_by_master_key(self, c, K):
 395         for key, v in self.master_public_keys.items():
 396             if key == "m/":continue
 397             cc, KK, _ = v
 398             if (c == cc) and (K == KK):
 399                 return key
 400
 401     def deseed_root(self, seed, password):
 402         # for safety, we ask the user to enter their seed
 403         assert seed == self.get_seed(password)
 404         self.seed = ''
 405         self.storage.put('seed', '', True)
 406
 407
 408     def deseed_branch(self, k):
 409         # check that parent has no seed
 410         assert self.seed == ''
 411         self.master_private_keys.pop(k)
 412         self.storage.put('master_private_keys', self.master_private_keys, True)
 413
 414     def is_watching_only(self):
 415         return (self.seed == '') and (self.master_private_keys == {})
 416
 417
 418
 419     def account_id(self, account_type, i):
 420         if account_type == '1':
 421             return "m/0'/%d"%i
 422         elif account_type == '2of2':
 423             return "m/1'/%d & m/2'/%d"%(i,i)
 424         elif account_type == '2of3':
 425             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 426         else:
 427             raise BaseException('unknown account type')
 428
 429
 430     def num_accounts(self, account_type):
 431         keys = self.accounts.keys()
 432         i = 0
 433         while True:
 434             account_id = self.account_id(account_type, i)
 435             if account_id not in keys: break
 436             i += 1
 437         return i
 438
 439
 440     def new_account_address(self, account_type = '1'):
 441         i = self.num_accounts(account_type)
 442         k = self.account_id(account_type,i)
 443
 444         addr = self.next_addresses.get(k)
 445         if not addr:
 446             account_id, account = self.next_account(account_type)
 447             addr = account.first_address()
 448             self.next_addresses[k] = addr
 449             self.storage.put('next_addresses',self.next_addresses)
 450
 451         return k, addr
 452
 453
 454     def next_account(self, account_type = '1'):
 455
 456         i = self.num_accounts(account_type)
 457         account_id = self.account_id(account_type,i)
 458
 459         if account_type is '1':
 460             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 461             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 462             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 463
 464         elif account_type == '2of2':
 465             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 466             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 467             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 468             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 469             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 470
 471         elif account_type == '2of3':
 472             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 473             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 474             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 475             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 476             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 477             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 478             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 479
 480         return account_id, account
 481
 482
 483     def set_label(self, name, text = None):
 484         changed = False
 485         old_text = self.labels.get(name)
 486         if text:
 487             if old_text != text:
 488                 self.labels[name] = text
 489                 changed = True
 490         else:
 491             if old_text:
 492                 self.labels.pop(name)
 493                 changed = True
 494
 495         if changed:
 496             self.storage.put('labels', self.labels, True)
 497
 498         run_hook('set_label', name, text, changed)
 499         return changed
 500
 501
 502
 503     def create_account(self, account_type = '1', name = None):
 504         k, account = self.next_account(account_type)
 505         if k in self.pending_accounts:
 506             self.pending_accounts.pop(k)
 507             self.storage.put('pending_accounts', self.pending_accounts)
 508
 509         self.accounts[k] = account
 510         self.save_accounts()
 511         if name:
 512             self.set_label(k, name)
 513
 514
 515     def create_old_account(self):
 516         mpk = OldAccount.mpk_from_seed(self.seed)
 517         self.storage.put('master_public_key', mpk, True)
 518         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 519         self.save_accounts()
 520
 521
 522     def save_accounts(self):
 523         d = {}
 524         for k, v in self.accounts.items():
 525             d[k] = v.dump()
 526         self.storage.put('accounts', d, True)
 527
 528
 529
 530     def load_accounts(self):
 531         d = self.storage.get('accounts', {})
 532         self.accounts = {}
 533         for k, v in d.items():
 534             if k == 0:
 535                 v['mpk'] = self.storage.get('master_public_key')
 536                 self.accounts[k] = OldAccount(v)
 537             elif '&' in k:
 538                 self.accounts[k] = BIP32_Account_2of2(v)
 539             else:
 540                 self.accounts[k] = BIP32_Account(v)
 541
 542         self.pending_accounts = self.storage.get('pending_accounts',{})
 543
 544
 545     def delete_pending_account(self, k):
 546         self.pending_accounts.pop(k)
 547         self.storage.put('pending_accounts', self.pending_accounts)
 548
 549     def account_is_pending(self, k):
 550         return k in self.pending_accounts
 551
 552     def create_pending_account(self, acct_type, name):
 553         k, addr = self.new_account_address(acct_type)
 554         self.set_label(k, name)
 555         self.pending_accounts[k] = addr
 556         self.storage.put('pending_accounts', self.pending_accounts)
 557
 558     def get_pending_accounts(self):
 559         return self.pending_accounts.items()
 560
 561
 562     def addresses(self, include_change = True, _next=True):
 563         o = self.get_account_addresses(-1, include_change)
 564         for a in self.accounts.keys():
 565             o += self.get_account_addresses(a, include_change)
 566
 567         if _next:
 568             for addr in self.next_addresses.values():
 569                 if addr not in o:
 570                     o += [addr]
 571         return o
 572
 573
 574     def is_mine(self, address):
 575         return address in self.addresses(True)
 576
 577
 578     def is_change(self, address):
 579         if not self.is_mine(address): return False
 580         if address in self.imported_keys.keys(): return False
 581         acct, s = self.get_address_index(address)
 582         if s is None: return False
 583         return s[0] == 1
 584
 585     def get_master_public_key(self):
 586         if self.seed_version == 4:
 587             return self.storage.get("master_public_key")
 588         else:
 589             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 590             return repr((c, K))
 591
 592     def get_master_private_key(self, account, password):
 593         master_k = pw_decode( self.master_private_keys[account], password)
 594         master_c, master_K, master_Kc = self.master_public_keys[account]
 595         try:
 596             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 597             assert K.encode('hex') == master_K
 598         except:
 599             raise BaseException("Invalid password")
 600         return master_k
 601
 602
 603     def get_address_index(self, address):
 604         if address in self.imported_keys.keys():
 605             return -1, None
 606
 607         for account in self.accounts.keys():
 608             for for_change in [0,1]:
 609                 addresses = self.accounts[account].get_addresses(for_change)
 610                 for addr in addresses:
 611                     if address == addr:
 612                         return account, (for_change, addresses.index(addr))
 613
 614         for k,v in self.next_addresses.items():
 615             if v == address:
 616                 return k, (0,0)
 617
 618         raise BaseException("Address not found", address)
 619
 620
 621     def get_roots(self, account):
 622         roots = []
 623         for a in account.split('&'):
 624             s = a.strip()
 625             m = re.match("(m/\d+'/)(\d+)", s)
 626             roots.append( m.group(1) )
 627         return roots
 628
 629     def is_seeded(self, account):
 630         if type(account) is int:
 631             return self.seed is not None
 632
 633         for root in self.get_roots(account):
 634             if root not in self.master_private_keys.keys():
 635                 return False
 636         return True
 637
 638     def rebase_sequence(self, account, sequence):
 639         c, i = sequence
 640         dd = []
 641         for a in account.split('&'):
 642             s = a.strip()
 643             m = re.match("(m/\d+'/)(\d+)", s)
 644             root = m.group(1)
 645             num = int(m.group(2))
 646             dd.append( (root, [num,c,i] ) )
 647         return dd
 648
 649
 650     def get_keyID(self, account, sequence):
 651         if account == 0:
 652             return 'old'
 653
 654         rs = self.rebase_sequence(account, sequence)
 655         dd = []
 656         for root, public_sequence in rs:
 657             c, K, _ = self.master_public_keys[root]
 658             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 659             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 660         return '&'.join(dd)
 661
 662
 663
 664     def get_seed(self, password):
 665         s = pw_decode(self.seed, password)
 666         if self.seed_version == 4:
 667             seed = s
 668         else:
 669             seed = mnemonic_hash(s)
 670         #todo:  #self.sequences[0].check_seed(seed)
 671         return seed
 672
 673
 674     def get_mnemonic(self, password):
 675         import mnemonic
 676         s = pw_decode(self.seed, password)
 677         if self.seed_version == 4:
 678             return ' '.join(mnemonic.mn_encode(s))
 679         else:
 680             return s
 681
 682
 683
 684     def get_private_key(self, address, password):
 685         out = []
 686         if address in self.imported_keys.keys():
 687             out.append( pw_decode( self.imported_keys[address], password ) )
 688         else:
 689             account, sequence = self.get_address_index(address)
 690             if account == 0:
 691                 seed = self.get_seed(password)
 692                 pk = self.accounts[account].get_private_key(seed, sequence)
 693                 out.append(pk)
 694                 return out
 695
 696             # assert address == self.accounts[account].get_address(*sequence)
 697             rs = self.rebase_sequence( account, sequence)
 698             for root, public_sequence in rs:
 699
 700                 if root not in self.master_private_keys.keys(): continue
 701                 master_k = self.get_master_private_key(root, password)
 702                 master_c, _, _ = self.master_public_keys[root]
 703                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 704                 out.append(pk)
 705
 706         return out
 707
 708
 709     def add_keypairs_from_wallet(self, tx, keypairs, password):
 710         for txin in tx.inputs:
 711             address = txin['address']
 712             private_keys = self.get_private_key(address, password)
 713             for sec in private_keys:
 714                 pubkey = public_key_from_private_key(sec)
 715                 keypairs[ pubkey ] = sec
 716
 717
 718     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 719         for txin in tx.inputs:
 720             keyid = txin.get('KeyID')
 721             if keyid:
 722                 roots = []
 723                 for s in keyid.split('&'):
 724                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 725                     if not m: continue
 726                     c = m.group(1)
 727                     K = m.group(2)
 728                     sequence = m.group(3)
 729                     root = self.find_root_by_master_key(c,K)
 730                     if not root: continue
 731                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 732                     root = root + '%d'%sequence[0]
 733                     sequence = sequence[1:]
 734                     roots.append((root,sequence))
 735
 736                 account_id = " & ".join( map(lambda x:x[0], roots) )
 737                 account = self.accounts.get(account_id)
 738                 if not account: continue
 739                 addr = account.get_address(*sequence)
 740                 txin['address'] = addr # fixme: side effect
 741                 pk = self.get_private_key(addr, password)
 742                 for sec in pk:
 743                     pubkey = public_key_from_private_key(sec)
 744                     keypairs[pubkey] = sec
 745
 746
 747
 748     def signrawtransaction(self, tx, input_info, private_keys, password):
 749
 750         # check that the password is correct
 751         seed = self.get_seed(password)
 752
 753         # add input info
 754         tx.add_input_info(input_info)
 755
 756         # add redeem script for coins that are in the wallet
 757         # FIXME: add redeemPubkey too!
 758         unspent_coins = self.get_unspent_coins()
 759         for txin in tx.inputs:
 760             for item in unspent_coins:
 761                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 762                     print_error( "tx input is in unspent coins" )
 763                     txin['scriptPubKey'] = item['scriptPubKey']
 764                     account, sequence = self.get_address_index(item['address'])
 765                     if account != -1:
 766                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 767                         print_error("added redeemScript", txin['redeemScript'])
 768                     break
 769
 770
 771         # build a list of public/private keys
 772         keypairs = {}
 773
 774         # add private keys from parameter
 775         for sec in private_keys:
 776             pubkey = public_key_from_private_key(sec)
 777             keypairs[ pubkey ] = sec
 778
 779         # add private_keys from KeyID
 780         self.add_keypairs_from_KeyID(tx, keypairs, password)
 781
 782         # add private keys from wallet
 783         self.add_keypairs_from_wallet(tx, keypairs, password)
 784         self.sign_transaction(tx, keypairs)
 785
 786
 787     def sign_message(self, address, message, password):
 788         keys = self.get_private_key(address, password)
 789         assert len(keys) == 1
 790         sec = keys[0]
 791         key = regenerate_key(sec)
 792         compressed = is_compressed(sec)
 793         return key.sign_message(message, compressed, address)
 794
 795
 796     def change_gap_limit(self, value):
 797         if value >= self.gap_limit:
 798             self.gap_limit = value
 799             self.storage.put('gap_limit', self.gap_limit, True)
 800             #self.interface.poke('synchronizer')
 801             return True
 802
 803         elif value >= self.min_acceptable_gap():
 804             for key, account in self.accounts.items():
 805                 addresses = account[0]
 806                 k = self.num_unused_trailing_addresses(addresses)
 807                 n = len(addresses) - k + value
 808                 addresses = addresses[0:n]
 809                 self.accounts[key][0] = addresses
 810
 811             self.gap_limit = value
 812             self.storage.put('gap_limit', self.gap_limit, True)
 813             self.save_accounts()
 814             return True
 815         else:
 816             return False
 817
 818     def num_unused_trailing_addresses(self, addresses):
 819         k = 0
 820         for a in addresses[::-1]:
 821             if self.history.get(a):break
 822             k = k + 1
 823         return k
 824
 825     def min_acceptable_gap(self):
 826         # fixme: this assumes wallet is synchronized
 827         n = 0
 828         nmax = 0
 829
 830         for account in self.accounts.values():
 831             addresses = account.get_addresses(0)
 832             k = self.num_unused_trailing_addresses(addresses)
 833             for a in addresses[0:-k]:
 834                 if self.history.get(a):
 835                     n = 0
 836                 else:
 837                     n += 1
 838                     if n > nmax: nmax = n
 839         return nmax + 1
 840
 841
 842     def address_is_old(self, address):
 843         age = -1
 844         h = self.history.get(address, [])
 845         if h == ['*']:
 846             return True
 847         for tx_hash, tx_height in h:
 848             if tx_height == 0:
 849                 tx_age = 0
 850             else:
 851                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 852             if tx_age > age:
 853                 age = tx_age
 854         return age > 2
 855
 856
 857     def synchronize_sequence(self, account, for_change):
 858         limit = self.gap_limit_for_change if for_change else self.gap_limit
 859         new_addresses = []
 860         while True:
 861             addresses = account.get_addresses(for_change)
 862             if len(addresses) < limit:
 863                 address = account.create_new_address(for_change)
 864                 self.history[address] = []
 865                 new_addresses.append( address )
 866                 continue
 867
 868             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 869                 break
 870             else:
 871                 address = account.create_new_address(for_change)
 872                 self.history[address] = []
 873                 new_addresses.append( address )
 874
 875         return new_addresses
 876
 877
 878
 879     def create_pending_accounts(self):
 880         for account_type in ['1','2of2','2of3']:
 881             if not self.has_master_public_keys(account_type):
 882                 continue
 883             k, a = self.new_account_address(account_type)
 884             if self.address_is_old(a):
 885                 print_error( "creating account", a )
 886                 self.create_account(account_type)
 887                 self.next_addresses.pop(k)
 888
 889
 890     def synchronize_account(self, account):
 891         new = []
 892         new += self.synchronize_sequence(account, 0)
 893         new += self.synchronize_sequence(account, 1)
 894         return new
 895
 896
 897     def synchronize(self):
 898         if self.master_public_keys:
 899             self.create_pending_accounts()
 900         new = []
 901         for account in self.accounts.values():
 902             new += self.synchronize_account(account)
 903         if new:
 904             self.save_accounts()
 905             self.storage.put('addr_history', self.history, True)
 906         return new
 907
 908
 909     def is_found(self):
 910         return self.history.values() != [[]] * len(self.history)
 911
 912
 913     def add_contact(self, address, label=None):
 914         self.addressbook.append(address)
 915         self.storage.put('contacts', self.addressbook, True)
 916         if label:
 917             self.set_label(address, label)
 918
 919
 920     def delete_contact(self, addr):
 921         if addr in self.addressbook:
 922             self.addressbook.remove(addr)
 923             self.storage.put('addressbook', self.addressbook, True)
 924
 925
 926     def fill_addressbook(self):
 927         for tx_hash, tx in self.transactions.items():
 928             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 929             if is_send:
 930                 for addr, v in tx.outputs:
 931                     if not self.is_mine(addr) and addr not in self.addressbook:
 932                         self.addressbook.append(addr)
 933         # redo labels
 934         # self.update_tx_labels()
 935
 936     def get_num_tx(self, address):
 937         n = 0
 938         for tx in self.transactions.values():
 939             if address in map(lambda x:x[0], tx.outputs): n += 1
 940         return n
 941
 942
 943     def get_address_flags(self, addr):
 944         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 945         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 946         return flags
 947
 948
 949     def get_tx_value(self, tx, account=None):
 950         domain = self.get_account_addresses(account)
 951         return tx.get_value(domain, self.prevout_values)
 952
 953
 954     def update_tx_outputs(self, tx_hash):
 955         tx = self.transactions.get(tx_hash)
 956
 957         for i, (addr, value) in enumerate(tx.outputs):
 958             key = tx_hash+ ':%d'%i
 959             self.prevout_values[key] = value
 960
 961         for item in tx.inputs:
 962             if self.is_mine(item.get('address')):
 963                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 964                 self.spent_outputs.append(key)
 965
 966
 967     def get_addr_balance(self, address):
 968         assert self.is_mine(address)
 969         h = self.history.get(address,[])
 970         if h == ['*']: return 0,0
 971         c = u = 0
 972         received_coins = []   # list of coins received at address
 973
 974         for tx_hash, tx_height in h:
 975             tx = self.transactions.get(tx_hash)
 976             if not tx: continue
 977
 978             for i, (addr, value) in enumerate(tx.outputs):
 979                 if addr == address:
 980                     key = tx_hash + ':%d'%i
 981                     received_coins.append(key)
 982
 983         for tx_hash, tx_height in h:
 984             tx = self.transactions.get(tx_hash)
 985             if not tx: continue
 986             v = 0
 987
 988             for item in tx.inputs:
 989                 addr = item.get('address')
 990                 if addr == address:
 991                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 992                     value = self.prevout_values.get( key )
 993                     if key in received_coins:
 994                         v -= value
 995
 996             for i, (addr, value) in enumerate(tx.outputs):
 997                 key = tx_hash + ':%d'%i
 998                 if addr == address:
 999                     v += value
1000
1001             if tx_height:
1002                 c += v
1003             else:
1004                 u += v
1005         return c, u
1006
1007
1008     def get_account_name(self, k):
1009         if k == 0:
1010             if self.seed_version == 4:
1011                 name = 'Main account'
1012             else:
1013                 name = 'Old account'
1014         else:
1015             default = "Unnamed account"
1016             m = re.match("m/0'/(\d+)", k)
1017             if m:
1018                 num = m.group(1)
1019                 if num == '0':
1020                     default = "Main account"
1021                 else:
1022                     default = "Account %s"%num
1023
1024             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1025             if m:
1026                 num = m.group(1)
1027                 default = "2of2 account %s"%num
1028             name = self.labels.get(k, default)
1029
1030         return name
1031
1032     def get_account_names(self):
1033         accounts = {}
1034         for k, account in self.accounts.items():
1035             accounts[k] = self.get_account_name(k)
1036         if self.imported_keys:
1037             accounts[-1] = 'Imported keys'
1038         return accounts
1039
1040     def get_account_addresses(self, a, include_change=True):
1041         if a is None:
1042             o = self.addresses(True)
1043         elif a == -1:
1044             o = self.imported_keys.keys()
1045         else:
1046             ac = self.accounts[a]
1047             o = ac.get_addresses(0)
1048             if include_change: o += ac.get_addresses(1)
1049         return o
1050
1051     def get_imported_balance(self):
1052         return self.get_balance(self.imported_keys.keys())
1053
1054     def get_account_balance(self, account):
1055         return self.get_balance(self.get_account_addresses(account))
1056
1057     def get_frozen_balance(self):
1058         return self.get_balance(self.frozen_addresses)
1059
1060     def get_balance(self, domain=None):
1061         if domain is None: domain = self.addresses(True)
1062         cc = uu = 0
1063         for addr in domain:
1064             c, u = self.get_addr_balance(addr)
1065             cc += c
1066             uu += u
1067         return cc, uu
1068
1069
1070     def get_unspent_coins(self, domain=None):
1071         coins = []
1072         if domain is None: domain = self.addresses(True)
1073         for addr in domain:
1074             h = self.history.get(addr, [])
1075             if h == ['*']: continue
1076             for tx_hash, tx_height in h:
1077                 tx = self.transactions.get(tx_hash)
1078                 if tx is None: raise BaseException("Wallet not synchronized")
1079                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1080                 for output in tx.d.get('outputs'):
1081                     if output.get('address') != addr: continue
1082                     key = tx_hash + ":%d" % output.get('prevout_n')
1083                     if key in self.spent_outputs: continue
1084                     output['prevout_hash'] = tx_hash
1085                     output['height'] = tx_height
1086                     output['coinbase'] = is_coinbase
1087                     coins.append((tx_height, output))
1088
1089         # sort by age
1090         if coins:
1091             coins = sorted(coins)
1092             if coins[-1][0] != 0:
1093                 while coins[0][0] == 0:
1094                     coins = coins[1:] + [ coins[0] ]
1095         return [x[1] for x in coins]
1096
1097
1098
1099     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1100         domain = self.get_account_addresses(account) if account else None
1101         return self.choose_tx_inputs( amount, fixed_fee, domain )
1102
1103
1104     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1105         """ todo: minimize tx size """
1106         total = 0
1107         fee = self.fee if fixed_fee is None else fixed_fee
1108         if domain is None:
1109             domain = self.addresses(True)
1110
1111         for i in self.frozen_addresses:
1112             if i in domain: domain.remove(i)
1113
1114         prioritized = []
1115         for i in self.prioritized_addresses:
1116             if i in domain:
1117                 domain.remove(i)
1118                 prioritized.append(i)
1119
1120         coins = self.get_unspent_coins(domain)
1121         prioritized_coins = self.get_unspent_coins(prioritized)
1122
1123         inputs = []
1124         coins = prioritized_coins + coins
1125
1126         for item in coins:
1127             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1128                 continue
1129             addr = item.get('address')
1130             v = item.get('value')
1131             total += v
1132             inputs.append(item)
1133             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1134             if total >= amount + fee: break
1135         else:
1136             inputs = []
1137
1138         return inputs, total, fee
1139
1140
1141     def set_fee(self, fee):
1142         if self.fee != fee:
1143             self.fee = fee
1144             self.storage.put('fee_per_kb', self.fee, True)
1145
1146     def estimated_fee(self, inputs):
1147         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1148         fee = self.fee * int(round(estimated_size/1024.))
1149         if fee == 0: fee = self.fee
1150         return fee
1151
1152
1153     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1154         "add change to a transaction"
1155         change_amount = total - ( amount + fee )
1156         if change_amount != 0:
1157             if not change_addr:
1158
1159                 # send change to one of the accounts involved in the tx
1160                 address = inputs[0].get('address')
1161                 account, _ = self.get_address_index(address)
1162
1163                 if not self.use_change or account == -1:
1164                     change_addr = inputs[-1]['address']
1165                 else:
1166                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1167
1168             # Insert the change output at a random position in the outputs
1169             posn = random.randint(0, len(outputs))
1170             outputs[posn:posn] = [( change_addr,  change_amount)]
1171         return outputs
1172
1173
1174     def get_history(self, address):
1175         with self.lock:
1176             return self.history.get(address)
1177
1178
1179     def get_status(self, h):
1180         if not h: return None
1181         if h == ['*']: return '*'
1182         status = ''
1183         for tx_hash, height in h:
1184             status += tx_hash + ':%d:' % height
1185         return hashlib.sha256( status ).digest().encode('hex')
1186
1187
1188     def receive_tx_callback(self, tx_hash, tx, tx_height):
1189
1190         with self.transaction_lock:
1191             self.add_extra_addresses(tx)
1192             if not self.check_new_tx(tx_hash, tx):
1193                 # may happen due to pruning
1194                 print_error("received transaction that is no longer referenced in history", tx_hash)
1195                 return
1196             self.transactions[tx_hash] = tx
1197             self.network.interface.pending_transactions_for_notifications.append(tx)
1198             self.save_transactions()
1199             if self.verifier and tx_height>0:
1200                 self.verifier.add(tx_hash, tx_height)
1201             self.update_tx_outputs(tx_hash)
1202
1203
1204     def save_transactions(self):
1205         tx = {}
1206         for k,v in self.transactions.items():
1207             tx[k] = str(v)
1208         self.storage.put('transactions', tx, True)
1209
1210     def receive_history_callback(self, addr, hist):
1211
1212         if not self.check_new_history(addr, hist):
1213             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1214
1215         with self.lock:
1216             self.history[addr] = hist
1217             self.storage.put('addr_history', self.history, True)
1218
1219         if hist != ['*']:
1220             for tx_hash, tx_height in hist:
1221                 if tx_height>0:
1222                     # add it in case it was previously unconfirmed
1223                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1224
1225
1226     def get_tx_history(self, account=None):
1227         with self.transaction_lock:
1228             history = self.transactions.items()
1229             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1230             result = []
1231
1232             balance = 0
1233             for tx_hash, tx in history:
1234                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1235                 if v is not None: balance += v
1236
1237             c, u = self.get_account_balance(account)
1238
1239             if balance != c+u:
1240                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1241
1242             balance = c + u - balance
1243             for tx_hash, tx in history:
1244                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1245                 if not is_relevant:
1246                     continue
1247                 if value is not None:
1248                     balance += value
1249
1250                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1251                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1252
1253         return result
1254
1255
1256     def get_label(self, tx_hash):
1257         label = self.labels.get(tx_hash)
1258         is_default = (label == '') or (label is None)
1259         if is_default: label = self.get_default_label(tx_hash)
1260         return label, is_default
1261
1262
1263     def get_default_label(self, tx_hash):
1264         tx = self.transactions.get(tx_hash)
1265         default_label = ''
1266         if tx:
1267             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1268             if is_mine:
1269                 for o in tx.outputs:
1270                     o_addr, _ = o
1271                     if not self.is_mine(o_addr):
1272                         try:
1273                             default_label = self.labels[o_addr]
1274                         except KeyError:
1275                             default_label = o_addr
1276                         break
1277                 else:
1278                     default_label = '(internal)'
1279             else:
1280                 for o in tx.outputs:
1281                     o_addr, _ = o
1282                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1283                         break
1284                 else:
1285                     for o in tx.outputs:
1286                         o_addr, _ = o
1287                         if self.is_mine(o_addr):
1288                             break
1289                     else:
1290                         o_addr = None
1291
1292                 if o_addr:
1293                     dest_label = self.labels.get(o_addr)
1294                     try:
1295                         default_label = self.labels[o_addr]
1296                     except KeyError:
1297                         default_label = o_addr
1298
1299         return default_label
1300
1301
1302     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1303         for address, x in outputs:
1304             assert is_valid(address)
1305         amount = sum( map(lambda x:x[1], outputs) )
1306         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1307         if not inputs:
1308             raise ValueError("Not enough funds")
1309         self.add_input_info(inputs)
1310         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1311         return Transaction.from_io(inputs, outputs)
1312
1313
1314     def mktx_from_account(self, outputs, password, fee=None, account=None):
1315         domain = self.get_account_addresses(account) if account else None
1316         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1317
1318
1319     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1320         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1321         keypairs = {}
1322         self.add_keypairs_from_wallet(tx, keypairs, password)
1323         if keypairs:
1324             self.sign_transaction(tx, keypairs)
1325         return tx
1326
1327
1328     def add_input_info(self, inputs):
1329         for txin in inputs:
1330             address = txin['address']
1331             account, sequence = self.get_address_index(address)
1332             txin['KeyID'] = self.get_keyID(account, sequence)
1333             redeemScript = self.accounts[account].redeem_script(sequence)
1334             if redeemScript:
1335                 txin['redeemScript'] = redeemScript
1336             else:
1337                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1338
1339
1340     def sign_transaction(self, tx, keypairs):
1341         tx.sign(keypairs)
1342         run_hook('sign_transaction', tx)
1343
1344
1345     def sendtx(self, tx):
1346         # synchronous
1347         h = self.send_tx(tx)
1348         self.tx_event.wait()
1349         return self.receive_tx(h)
1350
1351     def send_tx(self, tx):
1352         # asynchronous
1353         self.tx_event.clear()
1354         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1355         return tx.hash()
1356
1357     def on_broadcast(self, i, r):
1358         self.tx_result = r.get('result')
1359         self.tx_event.set()
1360
1361     def receive_tx(self,tx_hash):
1362         out = self.tx_result
1363         if out != tx_hash:
1364             return False, "error: " + out
1365         return True, out
1366
1367
1368
1369     def update_password(self, old_password, new_password):
1370         if new_password == '': new_password = None
1371         # this will throw an exception if unicode cannot be converted
1372         decoded = pw_decode(self.seed, old_password)
1373         self.seed = pw_encode( decoded, new_password)
1374         self.storage.put('seed', self.seed, True)
1375         self.use_encryption = (new_password != None)
1376         self.storage.put('use_encryption', self.use_encryption,True)
1377         for k in self.imported_keys.keys():
1378             a = self.imported_keys[k]
1379             b = pw_decode(a, old_password)
1380             c = pw_encode(b, new_password)
1381             self.imported_keys[k] = c
1382         self.storage.put('imported_keys', self.imported_keys, True)
1383
1384         for k, v in self.master_private_keys.items():
1385             b = pw_decode(v, old_password)
1386             c = pw_encode(b, new_password)
1387             self.master_private_keys[k] = c
1388         self.storage.put('master_private_keys', self.master_private_keys, True)
1389
1390
1391     def freeze(self,addr):
1392         if self.is_mine(addr) and addr not in self.frozen_addresses:
1393             self.unprioritize(addr)
1394             self.frozen_addresses.append(addr)
1395             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1396             return True
1397         else:
1398             return False
1399
1400     def unfreeze(self,addr):
1401         if self.is_mine(addr) and addr in self.frozen_addresses:
1402             self.frozen_addresses.remove(addr)
1403             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1404             return True
1405         else:
1406             return False
1407
1408     def prioritize(self,addr):
1409         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1410             self.unfreeze(addr)
1411             self.prioritized_addresses.append(addr)
1412             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1413             return True
1414         else:
1415             return False
1416
1417     def unprioritize(self,addr):
1418         if self.is_mine(addr) and addr in self.prioritized_addresses:
1419             self.prioritized_addresses.remove(addr)
1420             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1421             return True
1422         else:
1423             return False
1424
1425
1426     def set_verifier(self, verifier):
1427         self.verifier = verifier
1428
1429         # review transactions that are in the history
1430         for addr, hist in self.history.items():
1431             if hist == ['*']: continue
1432             for tx_hash, tx_height in hist:
1433                 if tx_height>0:
1434                     # add it in case it was previously unconfirmed
1435                     self.verifier.add(tx_hash, tx_height)
1436
1437
1438         # if we are on a pruning server, remove unverified transactions
1439         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1440         for tx_hash in self.transactions.keys():
1441             if tx_hash not in vr:
1442                 self.transactions.pop(tx_hash)
1443
1444
1445
1446     def check_new_history(self, addr, hist):
1447
1448         # check that all tx in hist are relevant
1449         if hist != ['*']:
1450             for tx_hash, height in hist:
1451                 tx = self.transactions.get(tx_hash)
1452                 if not tx: continue
1453                 if not tx.has_address(addr):
1454                     return False
1455
1456         # check that we are not "orphaning" a transaction
1457         old_hist = self.history.get(addr,[])
1458         if old_hist == ['*']: return True
1459
1460         for tx_hash, height in old_hist:
1461             if tx_hash in map(lambda x:x[0], hist): continue
1462             found = False
1463             for _addr, _hist in self.history.items():
1464                 if _addr == addr: continue
1465                 if _hist == ['*']: continue
1466                 _tx_hist = map(lambda x:x[0], _hist)
1467                 if tx_hash in _tx_hist:
1468                     found = True
1469                     break
1470
1471             if not found:
1472                 tx = self.transactions.get(tx_hash)
1473                 # tx might not be there
1474                 if not tx: continue
1475
1476                 # already verified?
1477                 if self.verifier.get_height(tx_hash):
1478                     continue
1479                 # unconfirmed tx
1480                 print_error("new history is orphaning transaction:", tx_hash)
1481                 # check that all outputs are not mine, request histories
1482                 ext_requests = []
1483                 for _addr, _v in tx.outputs:
1484                     # assert not self.is_mine(_addr)
1485                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1486
1487                 ext_h = self.network.synchronous_get(ext_requests)
1488                 print_error("sync:", ext_requests, ext_h)
1489                 height = None
1490                 for h in ext_h:
1491                     if h == ['*']: continue
1492                     for item in h:
1493                         if item.get('tx_hash') == tx_hash:
1494                             height = item.get('height')
1495                 if height:
1496                     print_error("found height for", tx_hash, height)
1497                     self.verifier.add(tx_hash, height)
1498                 else:
1499                     print_error("removing orphaned tx from history", tx_hash)
1500                     self.transactions.pop(tx_hash)
1501
1502         return True
1503
1504
1505
1506     def check_new_tx(self, tx_hash, tx):
1507         # 1 check that tx is referenced in addr_history.
1508         addresses = []
1509         for addr, hist in self.history.items():
1510             if hist == ['*']:continue
1511             for txh, height in hist:
1512                 if txh == tx_hash:
1513                     addresses.append(addr)
1514
1515         if not addresses:
1516             return False
1517
1518         # 2 check that referencing addresses are in the tx
1519         for addr in addresses:
1520             if not tx.has_address(addr):
1521                 return False
1522
1523         return True
1524
1525
1526     def start_threads(self, network):
1527         from verifier import TxVerifier
1528         self.network = network
1529         self.verifier = TxVerifier(self.network, self.storage)
1530         self.verifier.start()
1531         self.set_verifier(self.verifier)
1532         self.synchronizer = WalletSynchronizer(self, network)
1533         self.synchronizer.start()
1534
1535     def stop_threads(self):
1536         self.verifier.stop()
1537         self.synchronizer.stop()
1538
1539
1540
1541     def restore(self, callback):
1542         from i18n import _
1543         def wait_for_wallet():
1544             self.set_up_to_date(False)
1545             while not self.is_up_to_date():
1546                 msg = "%s\n%s %d\n%s %.1f"%(
1547                     _("Please wait..."),
1548                     _("Addresses generated:"),
1549                     len(self.addresses(True)),_("Kilobytes received:"),
1550                     self.network.interface.bytes_received/1024.)
1551
1552                 apply(callback, (msg,))
1553                 time.sleep(0.1)
1554
1555         def wait_for_network():
1556             while not self.network.interface.is_connected:
1557                 msg = "%s \n" % (_("Connecting..."))
1558                 apply(callback, (msg,))
1559                 time.sleep(0.1)
1560
1561         # wait until we are connected, because the user might have selected another server
1562         wait_for_network()
1563
1564
1565         self.create_accounts()
1566         wait_for_wallet()
1567
1568
1569
1570
1571 class WalletSynchronizer(threading.Thread):
1572
1573
1574     def __init__(self, wallet, network):
1575         threading.Thread.__init__(self)
1576         self.daemon = True
1577         self.wallet = wallet
1578         self.network = network
1579         self.was_updated = True
1580         self.running = False
1581         self.lock = threading.Lock()
1582         self.queue = Queue.Queue()
1583
1584     def stop(self):
1585         with self.lock: self.running = False
1586
1587     def is_running(self):
1588         with self.lock: return self.running
1589
1590
1591     def subscribe_to_addresses(self, addresses):
1592         messages = []
1593         for addr in addresses:
1594             messages.append(('blockchain.address.subscribe', [addr]))
1595         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1596
1597
1598     def run(self):
1599         with self.lock:
1600             self.running = True
1601
1602         while self.is_running():
1603
1604             if not self.network.is_connected():
1605                 print_error("synchronizer: waiting for interface")
1606                 self.network.wait_until_connected()
1607
1608             self.run_interface(self.network.interface)
1609
1610
1611     def run_interface(self, interface):
1612
1613         print_error("synchronizer: connected to", interface.server)
1614
1615         requested_tx = []
1616         missing_tx = []
1617         requested_histories = {}
1618
1619         # request any missing transactions
1620         for history in self.wallet.history.values():
1621             if history == ['*']: continue
1622             for tx_hash, tx_height in history:
1623                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1624                     missing_tx.append( (tx_hash, tx_height) )
1625
1626         if missing_tx:
1627             print_error("missing tx", missing_tx)
1628
1629         # subscriptions
1630         self.subscribe_to_addresses(self.wallet.addresses(True))
1631
1632         while self.is_running():
1633             # 1. create new addresses
1634             new_addresses = self.wallet.synchronize()
1635
1636             # request missing addresses
1637             if new_addresses:
1638                 self.subscribe_to_addresses(new_addresses)
1639
1640             # request missing transactions
1641             for tx_hash, tx_height in missing_tx:
1642                 if (tx_hash, tx_height) not in requested_tx:
1643                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1644                     requested_tx.append( (tx_hash, tx_height) )
1645             missing_tx = []
1646
1647             # detect if situation has changed
1648             if interface.is_up_to_date() and self.queue.empty():
1649                 if not self.wallet.is_up_to_date():
1650                     self.wallet.set_up_to_date(True)
1651                     self.was_updated = True
1652             else:
1653                 if self.wallet.is_up_to_date():
1654                     self.wallet.set_up_to_date(False)
1655                     self.was_updated = True
1656
1657             if self.was_updated:
1658                 self.wallet.network.trigger_callback('updated')
1659                 self.was_updated = False
1660
1661             # 2. get a response
1662             try:
1663                 r = self.queue.get(block=True, timeout=1)
1664             except Queue.Empty:
1665                 continue
1666
1667             if interface != self.network.interface:
1668                 break
1669
1670             if not r:
1671                 continue
1672
1673             # 3. handle response
1674             method = r['method']
1675             params = r['params']
1676             result = r.get('result')
1677             error = r.get('error')
1678             if error:
1679                 print "error", r
1680                 continue
1681
1682             if method == 'blockchain.address.subscribe':
1683                 addr = params[0]
1684                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1685                     if requested_histories.get(addr) is None:
1686                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1687                         requested_histories[addr] = result
1688
1689             elif method == 'blockchain.address.get_history':
1690                 addr = params[0]
1691                 print_error("receiving history", addr, result)
1692                 if result == ['*']:
1693                     assert requested_histories.pop(addr) == '*'
1694                     self.wallet.receive_history_callback(addr, result)
1695                 else:
1696                     hist = []
1697                     # check that txids are unique
1698                     txids = []
1699                     for item in result:
1700                         tx_hash = item['tx_hash']
1701                         if tx_hash not in txids:
1702                             txids.append(tx_hash)
1703                             hist.append( (tx_hash, item['height']) )
1704
1705                     if len(hist) != len(result):
1706                         raise BaseException("error: server sent history with non-unique txid", result)
1707
1708                     # check that the status corresponds to what was announced
1709                     rs = requested_histories.pop(addr)
1710                     if self.wallet.get_status(hist) != rs:
1711                         raise BaseException("error: status mismatch: %s"%addr)
1712
1713                     # store received history
1714                     self.wallet.receive_history_callback(addr, hist)
1715
1716                     # request transactions that we don't have
1717                     for tx_hash, tx_height in hist:
1718                         if self.wallet.transactions.get(tx_hash) is None:
1719                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1720                                 missing_tx.append( (tx_hash, tx_height) )
1721
1722             elif method == 'blockchain.transaction.get':
1723                 tx_hash = params[0]
1724                 tx_height = params[1]
1725                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1726                 tx = Transaction(result)
1727                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1728                 self.was_updated = True
1729                 requested_tx.remove( (tx_hash, tx_height) )
1730                 print_error("received tx:", tx_hash, len(tx.raw))
1731
1732             else:
1733                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1734
1735             if self.was_updated and not requested_tx:
1736                 self.wallet.network.trigger_callback('updated')
1737                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1738
1739                 self.was_updated = False