lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         v = self.data.get(key)
 129         if v is None:
 130             v = default
 131         return v
 132
 133     def put(self, key, value, save = True):
 134
 135         with self.lock:
 136             if value is not None:
 137                 self.data[key] = value
 138             else:
 139                 self.data.pop(key)
 140             if save:
 141                 self.write()
 142
 143     def write(self):
 144         s = repr(self.data)
 145         f = open(self.path,"w")
 146         f.write( s )
 147         f.close()
 148         if 'ANDROID_DATA' not in os.environ:
 149             import stat
 150             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 151
 152
 153 class Wallet:
 154
 155     def __init__(self, storage):
 156
 157         self.storage = storage
 158         self.electrum_version = ELECTRUM_VERSION
 159         self.gap_limit_for_change = 3 # constant
 160
 161         # saved fields
 162         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 163
 164         self.gap_limit             = storage.get('gap_limit', 5)
 165         self.use_change            = storage.get('use_change',True)
 166         self.use_encryption        = storage.get('use_encryption', False)
 167         self.seed                  = storage.get('seed', '')               # encrypted
 168         self.labels                = storage.get('labels', {})
 169         self.frozen_addresses      = storage.get('frozen_addresses',[])
 170         self.addressbook           = storage.get('contacts', [])
 171
 172         self.imported_keys         = storage.get('imported_keys',{})
 173         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 174
 175         self.fee                   = int(storage.get('fee_per_kb',20000))
 176
 177         self.master_public_keys = storage.get('master_public_keys',{})
 178         self.master_private_keys = storage.get('master_private_keys', {})
 179
 180         self.next_addresses = storage.get('next_addresses',{})
 181
 182         if self.seed_version not in [4, 6]:
 183             msg = "This wallet seed is not supported."
 184             if self.seed_version in [5]:
 185                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 186                 print msg
 187                 sys.exit(1)
 188
 189         # This attribute is set when wallet.start_threads is called.
 190         self.synchronizer = None
 191
 192         self.load_accounts()
 193
 194         self.transactions = {}
 195         tx_list = self.storage.get('transactions',{})
 196         for k,v in tx_list.items():
 197             try:
 198                 tx = Transaction(v)
 199             except Exception:
 200                 print_msg("Warning: Cannot deserialize transactions. skipping")
 201                 continue
 202
 203             self.add_extra_addresses(tx)
 204             self.transactions[k] = tx
 205
 206         for h,tx in self.transactions.items():
 207             if not self.check_new_tx(h, tx):
 208                 print_error("removing unreferenced tx", h)
 209                 self.transactions.pop(h)
 210
 211
 212         # not saved
 213         self.prevout_values = {}     # my own transaction outputs
 214         self.spent_outputs = []
 215
 216         # spv
 217         self.verifier = None
 218
 219         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 220         # interface.is_up_to_date() returns true when all requests have been answered and processed
 221         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 222
 223         self.up_to_date = False
 224         self.lock = threading.Lock()
 225         self.transaction_lock = threading.Lock()
 226         self.tx_event = threading.Event()
 227
 228         for tx_hash, tx in self.transactions.items():
 229             self.update_tx_outputs(tx_hash)
 230
 231
 232     def add_extra_addresses(self, tx):
 233         h = tx.hash()
 234         # find the address corresponding to pay-to-pubkey inputs
 235         tx.add_extra_addresses(self.transactions)
 236         for o in tx.d.get('outputs'):
 237             if o.get('is_pubkey'):
 238                 for tx2 in self.transactions.values():
 239                     tx2.add_extra_addresses({h:tx})
 240
 241
 242
 243
 244     def set_up_to_date(self,b):
 245         with self.lock: self.up_to_date = b
 246
 247
 248     def is_up_to_date(self):
 249         with self.lock: return self.up_to_date
 250
 251
 252     def update(self):
 253         self.up_to_date = False
 254         while not self.is_up_to_date():
 255             time.sleep(0.1)
 256
 257
 258     def import_key(self, sec, password):
 259         # check password
 260         seed = self.get_seed(password)
 261         try:
 262             address = address_from_private_key(sec)
 263         except Exception:
 264             raise Exception('Invalid private key')
 265
 266         if self.is_mine(address):
 267             raise Exception('Address already in wallet')
 268
 269         # store the originally requested keypair into the imported keys table
 270         self.imported_keys[address] = pw_encode(sec, password )
 271         self.storage.put('imported_keys', self.imported_keys, True)
 272         if self.synchronizer:
 273             self.synchronizer.subscribe_to_addresses([address])
 274         return address
 275
 276     def delete_imported_key(self, addr):
 277         if addr in self.imported_keys:
 278             self.imported_keys.pop(addr)
 279             self.storage.put('imported_keys', self.imported_keys, True)
 280
 281
 282     def make_seed(self):
 283         import mnemonic, ecdsa
 284         entropy = ecdsa.util.randrange( pow(2,160) )
 285         nonce = 0
 286         while True:
 287             ss = "%040x"%(entropy+nonce)
 288             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 289             # we keep only 13 words, that's approximately 139 bits of entropy
 290             words = mnemonic.mn_encode(s)[0:13]
 291             seed = ' '.join(words)
 292             if is_seed(seed):
 293                 break  # this will remove 8 bits of entropy
 294             nonce += 1
 295
 296         return seed
 297
 298
 299     def init_seed(self, seed):
 300         import mnemonic
 301
 302         if self.seed:
 303             raise Exception("a seed exists")
 304
 305         if not seed:
 306             self.seed = self.make_seed()
 307             self.seed_version = SEED_VERSION
 308             return
 309
 310         # find out what kind of wallet we are
 311         try:
 312             seed.strip().decode('hex')
 313             self.seed_version = 4
 314             self.seed = str(seed)
 315             return
 316         except Exception:
 317             pass
 318
 319         words = seed.split()
 320         try:
 321             mnemonic.mn_decode(words)
 322             uses_electrum_words = True
 323         except Exception:
 324             uses_electrum_words = False
 325
 326         if uses_electrum_words and len(words) != 13:
 327             self.seed_version = 4
 328             self.seed = mnemonic.mn_decode(words)
 329         else:
 330             #assert is_seed(seed)
 331             self.seed_version = SEED_VERSION
 332             self.seed = seed
 333
 334
 335     def save_seed(self, password):
 336         if password:
 337             self.seed = pw_encode( self.seed, password)
 338             self.use_encryption = True
 339         self.storage.put('seed', self.seed, True)
 340         self.storage.put('seed_version', self.seed_version, True)
 341         self.storage.put('use_encryption', self.use_encryption,True)
 342         self.create_accounts(password)
 343
 344
 345     def create_watching_only_wallet(self, params):
 346         K0, c0 = params
 347         if not K0:
 348             return
 349
 350         if not c0:
 351             self.seed_version = 4
 352             self.storage.put('seed_version', self.seed_version, True)
 353             self.create_old_account(K0)
 354             return
 355
 356         cK0 = ""
 357         self.master_public_keys = {
 358             "m/0'/": (c0, K0, cK0),
 359             }
 360         self.storage.put('master_public_keys', self.master_public_keys, True)
 361         self.storage.put('seed_version', self.seed_version, True)
 362         self.create_account('1','Main account')
 363
 364
 365     def create_accounts(self, password):
 366         seed = pw_decode(self.seed, password)
 367
 368         if self.seed_version == 4:
 369             mpk = OldAccount.mpk_from_seed(seed)
 370             self.create_old_account(mpk)
 371         else:
 372             # create default account
 373             self.create_master_keys('1', password)
 374             self.create_account('1','Main account')
 375
 376
 377     def create_master_keys(self, account_type, password):
 378         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 379         if account_type == '1':
 380             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 381             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 382             self.master_private_keys["m/0'/"] = pw_encode(k0, password)
 383         elif account_type == '2of2':
 384             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 385             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 386             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 387             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 388             self.master_private_keys["m/1'/"] = pw_encode(k1, password)
 389             self.master_private_keys["m/2'/"] = pw_encode(k2, password)
 390         elif account_type == '2of3':
 391             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 392             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 393             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 394             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 395             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 396             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 397             self.master_private_keys["m/3'/"] = pw_encode(k3, password)
 398             self.master_private_keys["m/4'/"] = pw_encode(k4, password)
 399             self.master_private_keys["m/5'/"] = pw_encode(k5, password)
 400
 401         self.storage.put('master_public_keys', self.master_public_keys, True)
 402         self.storage.put('master_private_keys', self.master_private_keys, True)
 403
 404     def has_master_public_keys(self, account_type):
 405         if account_type == '1':
 406             return "m/0'/" in self.master_public_keys
 407         elif account_type == '2of2':
 408             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 409         elif account_type == '2of3':
 410             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 411
 412     def find_root_by_master_key(self, c, K):
 413         for key, v in self.master_public_keys.items():
 414             if key == "m/":continue
 415             cc, KK, _ = v
 416             if (c == cc) and (K == KK):
 417                 return key
 418
 419     def deseed_root(self, seed, password):
 420         # for safety, we ask the user to enter their seed
 421         assert seed == self.get_seed(password)
 422         self.seed = ''
 423         self.storage.put('seed', '', True)
 424
 425
 426     def deseed_branch(self, k):
 427         # check that parent has no seed
 428         # assert self.seed == ''
 429         k = self.master_private_keys.pop(k)
 430         self.storage.put('master_private_keys', self.master_private_keys, True)
 431         return k
 432
 433     def is_watching_only(self):
 434         return (self.seed == '') and (self.master_private_keys == {})
 435
 436
 437
 438     def account_id(self, account_type, i):
 439         if account_type == '1':
 440             return "m/0'/%d"%i
 441         elif account_type == '2of2':
 442             return "m/1'/%d & m/2'/%d"%(i,i)
 443         elif account_type == '2of3':
 444             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 445         else:
 446             raise Exception('unknown account type')
 447
 448
 449     def num_accounts(self, account_type):
 450         keys = self.accounts.keys()
 451         i = 0
 452         while True:
 453             account_id = self.account_id(account_type, i)
 454             if account_id not in keys: break
 455             i += 1
 456         return i
 457
 458
 459     def new_account_address(self, account_type = '1'):
 460         i = self.num_accounts(account_type)
 461         k = self.account_id(account_type,i)
 462
 463         addr = self.next_addresses.get(k)
 464         if not addr:
 465             account_id, account = self.next_account(account_type)
 466             addr = account.first_address()
 467             self.next_addresses[k] = addr
 468             self.storage.put('next_addresses',self.next_addresses)
 469
 470         return k, addr
 471
 472
 473     def next_account(self, account_type = '1'):
 474
 475         i = self.num_accounts(account_type)
 476         account_id = self.account_id(account_type,i)
 477
 478         if account_type is '1':
 479             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 480             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 481             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 482
 483         elif account_type == '2of2':
 484             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 485             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 486             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 487             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 488             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 489
 490         elif account_type == '2of3':
 491             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 492             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 493             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 494             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 495             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 496             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 497             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 498
 499         return account_id, account
 500
 501
 502     def set_label(self, name, text = None):
 503         changed = False
 504         old_text = self.labels.get(name)
 505         if text:
 506             if old_text != text:
 507                 self.labels[name] = text
 508                 changed = True
 509         else:
 510             if old_text:
 511                 self.labels.pop(name)
 512                 changed = True
 513
 514         if changed:
 515             self.storage.put('labels', self.labels, True)
 516
 517         run_hook('set_label', name, text, changed)
 518         return changed
 519
 520
 521
 522     def create_account(self, account_type = '1', name = None):
 523         k, account = self.next_account(account_type)
 524         if k in self.pending_accounts:
 525             self.pending_accounts.pop(k)
 526             self.storage.put('pending_accounts', self.pending_accounts)
 527
 528         self.accounts[k] = account
 529         self.save_accounts()
 530         if name:
 531             self.set_label(k, name)
 532
 533
 534     def create_old_account(self, mpk):
 535         self.storage.put('master_public_key', mpk, True)
 536         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 537         self.save_accounts()
 538
 539
 540     def save_accounts(self):
 541         d = {}
 542         for k, v in self.accounts.items():
 543             d[k] = v.dump()
 544         self.storage.put('accounts', d, True)
 545
 546
 547
 548     def load_accounts(self):
 549         d = self.storage.get('accounts', {})
 550         self.accounts = {}
 551         for k, v in d.items():
 552             if k == 0:
 553                 v['mpk'] = self.storage.get('master_public_key')
 554                 self.accounts[k] = OldAccount(v)
 555             elif '&' in k:
 556                 self.accounts[k] = BIP32_Account_2of2(v)
 557             else:
 558                 self.accounts[k] = BIP32_Account(v)
 559
 560         self.pending_accounts = self.storage.get('pending_accounts',{})
 561
 562
 563     def delete_pending_account(self, k):
 564         self.pending_accounts.pop(k)
 565         self.storage.put('pending_accounts', self.pending_accounts)
 566
 567     def account_is_pending(self, k):
 568         return k in self.pending_accounts
 569
 570     def create_pending_account(self, acct_type, name):
 571         k, addr = self.new_account_address(acct_type)
 572         self.set_label(k, name)
 573         self.pending_accounts[k] = addr
 574         self.storage.put('pending_accounts', self.pending_accounts)
 575
 576     def get_pending_accounts(self):
 577         return self.pending_accounts.items()
 578
 579
 580     def addresses(self, include_change = True, _next=True):
 581         o = self.get_account_addresses(-1, include_change)
 582         for a in self.accounts.keys():
 583             o += self.get_account_addresses(a, include_change)
 584
 585         if _next:
 586             for addr in self.next_addresses.values():
 587                 if addr not in o:
 588                     o += [addr]
 589         return o
 590
 591
 592     def is_mine(self, address):
 593         return address in self.addresses(True)
 594
 595
 596     def is_change(self, address):
 597         if not self.is_mine(address): return False
 598         if address in self.imported_keys.keys(): return False
 599         acct, s = self.get_address_index(address)
 600         if s is None: return False
 601         return s[0] == 1
 602
 603     def get_master_public_key(self):
 604         if self.seed_version == 4:
 605             return self.storage.get("master_public_key")
 606         else:
 607             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 608             return repr((c, K))
 609
 610     def get_master_private_key(self, account, password):
 611         master_k = pw_decode( self.master_private_keys[account], password)
 612         master_c, master_K, master_Kc = self.master_public_keys[account]
 613         try:
 614             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 615             assert K.encode('hex') == master_K
 616         except Exception:
 617             raise Exception("Invalid password")
 618         return master_k
 619
 620
 621     def get_address_index(self, address):
 622         if address in self.imported_keys.keys():
 623             return -1, None
 624
 625         for account in self.accounts.keys():
 626             for for_change in [0,1]:
 627                 addresses = self.accounts[account].get_addresses(for_change)
 628                 for addr in addresses:
 629                     if address == addr:
 630                         return account, (for_change, addresses.index(addr))
 631
 632         for k,v in self.next_addresses.items():
 633             if v == address:
 634                 return k, (0,0)
 635
 636         raise Exception("Address not found", address)
 637
 638
 639     def get_roots(self, account):
 640         roots = []
 641         for a in account.split('&'):
 642             s = a.strip()
 643             m = re.match("(m/\d+'/)(\d+)", s)
 644             roots.append( m.group(1) )
 645         return roots
 646
 647     def is_seeded(self, account):
 648         if type(account) is int:
 649             return self.seed is not None
 650
 651         for root in self.get_roots(account):
 652             if root not in self.master_private_keys.keys():
 653                 return False
 654         return True
 655
 656     def rebase_sequence(self, account, sequence):
 657         c, i = sequence
 658         dd = []
 659         for a in account.split('&'):
 660             s = a.strip()
 661             m = re.match("(m/\d+'/)(\d+)", s)
 662             root = m.group(1)
 663             num = int(m.group(2))
 664             dd.append( (root, [num,c,i] ) )
 665         return dd
 666
 667
 668     def get_keyID(self, account, sequence):
 669         if account == 0:
 670             a, b = sequence
 671             mpk = self.storage.get('master_public_key')
 672             return 'old(%s,%d,%d)'%(mpk,a,b)
 673
 674         rs = self.rebase_sequence(account, sequence)
 675         dd = []
 676         for root, public_sequence in rs:
 677             c, K, _ = self.master_public_keys[root]
 678             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 679             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 680         return '&'.join(dd)
 681
 682
 683
 684     def get_seed(self, password):
 685         s = pw_decode(self.seed, password)
 686         if self.seed_version == 4:
 687             seed = s
 688             self.accounts[0].check_seed(seed)
 689         else:
 690             seed = mnemonic_hash(s)
 691         return seed
 692
 693
 694     def get_mnemonic(self, password):
 695         import mnemonic
 696         s = pw_decode(self.seed, password)
 697         if self.seed_version == 4:
 698             return ' '.join(mnemonic.mn_encode(s))
 699         else:
 700             return s
 701
 702
 703
 704     def get_private_key(self, address, password):
 705         if self.is_watching_only():
 706             return []
 707
 708         # first check the provided password
 709         seed = self.get_seed(password)
 710
 711         out = []
 712         if address in self.imported_keys.keys():
 713             out.append( pw_decode( self.imported_keys[address], password ) )
 714         else:
 715             account, sequence = self.get_address_index(address)
 716             if account == 0:
 717                 pk = self.accounts[account].get_private_key(seed, sequence)
 718                 out.append(pk)
 719                 return out
 720
 721             # assert address == self.accounts[account].get_address(*sequence)
 722             rs = self.rebase_sequence( account, sequence)
 723             for root, public_sequence in rs:
 724
 725                 if root not in self.master_private_keys.keys(): continue
 726                 master_k = self.get_master_private_key(root, password)
 727                 master_c, _, _ = self.master_public_keys[root]
 728                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 729                 out.append(pk)
 730
 731         return out
 732
 733
 734     def add_keypairs_from_wallet(self, tx, keypairs, password):
 735         for txin in tx.inputs:
 736             address = txin['address']
 737             if not self.is_mine(address):
 738                 continue
 739             private_keys = self.get_private_key(address, password)
 740             for sec in private_keys:
 741                 pubkey = public_key_from_private_key(sec)
 742                 keypairs[ pubkey ] = sec
 743                 if address in self.imported_keys.keys():
 744                     txin['redeemPubkey'] = pubkey
 745
 746
 747     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 748         # first check the provided password
 749         seed = self.get_seed(password)
 750
 751         for txin in tx.inputs:
 752             keyid = txin.get('KeyID')
 753             if keyid:
 754
 755                 if self.seed_version == 4:
 756                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 757                     if not m: continue
 758                     mpk = m.group(1)
 759                     if mpk != self.storage.get('master_public_key'): continue
 760                     for_change = int(m.group(2))
 761                     num = int(m.group(3))
 762                     account = self.accounts[0]
 763                     addr = account.get_address(for_change, num)
 764                     txin['address'] = addr # fixme: side effect
 765                     pk = account.get_private_key(seed, (for_change, num))
 766                     pubkey = public_key_from_private_key(pk)
 767                     keypairs[pubkey] = pk
 768                     continue
 769
 770
 771                 roots = []
 772                 for s in keyid.split('&'):
 773                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 774                     if not m: continue
 775                     c = m.group(1)
 776                     K = m.group(2)
 777                     sequence = m.group(3)
 778                     root = self.find_root_by_master_key(c,K)
 779                     if not root: continue
 780                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 781                     root = root + '%d'%sequence[0]
 782                     sequence = sequence[1:]
 783                     roots.append((root,sequence))
 784
 785                 account_id = " & ".join( map(lambda x:x[0], roots) )
 786                 account = self.accounts.get(account_id)
 787                 if not account: continue
 788                 addr = account.get_address(*sequence)
 789                 txin['address'] = addr # fixme: side effect
 790                 pk = self.get_private_key(addr, password)
 791                 for sec in pk:
 792                     pubkey = public_key_from_private_key(sec)
 793                     keypairs[pubkey] = sec
 794
 795
 796
 797     def signrawtransaction(self, tx, input_info, private_keys, password):
 798
 799         # check that the password is correct
 800         seed = self.get_seed(password)
 801
 802         # add input info
 803         tx.add_input_info(input_info)
 804
 805         # add redeem script for coins that are in the wallet
 806         # FIXME: add redeemPubkey too!
 807         unspent_coins = self.get_unspent_coins()
 808         for txin in tx.inputs:
 809             for item in unspent_coins:
 810                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 811                     print_error( "tx input is in unspent coins" )
 812                     txin['scriptPubKey'] = item['scriptPubKey']
 813                     account, sequence = self.get_address_index(item['address'])
 814                     if account != -1:
 815                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 816                         print_error("added redeemScript", txin['redeemScript'])
 817                     break
 818
 819
 820         # build a list of public/private keys
 821         keypairs = {}
 822
 823         # add private keys from parameter
 824         for sec in private_keys:
 825             pubkey = public_key_from_private_key(sec)
 826             keypairs[ pubkey ] = sec
 827
 828         # add private_keys from KeyID
 829         self.add_keypairs_from_KeyID(tx, keypairs, password)
 830
 831         # add private keys from wallet
 832         self.add_keypairs_from_wallet(tx, keypairs, password)
 833         self.sign_transaction(tx, keypairs)
 834
 835
 836     def sign_message(self, address, message, password):
 837         keys = self.get_private_key(address, password)
 838         assert len(keys) == 1
 839         sec = keys[0]
 840         key = regenerate_key(sec)
 841         compressed = is_compressed(sec)
 842         return key.sign_message(message, compressed, address)
 843
 844
 845     def change_gap_limit(self, value):
 846         if value >= self.gap_limit:
 847             self.gap_limit = value
 848             self.storage.put('gap_limit', self.gap_limit, True)
 849             #self.interface.poke('synchronizer')
 850             return True
 851
 852         elif value >= self.min_acceptable_gap():
 853             for key, account in self.accounts.items():
 854                 addresses = account[0]
 855                 k = self.num_unused_trailing_addresses(addresses)
 856                 n = len(addresses) - k + value
 857                 addresses = addresses[0:n]
 858                 self.accounts[key][0] = addresses
 859
 860             self.gap_limit = value
 861             self.storage.put('gap_limit', self.gap_limit, True)
 862             self.save_accounts()
 863             return True
 864         else:
 865             return False
 866
 867     def num_unused_trailing_addresses(self, addresses):
 868         k = 0
 869         for a in addresses[::-1]:
 870             if self.history.get(a):break
 871             k = k + 1
 872         return k
 873
 874     def min_acceptable_gap(self):
 875         # fixme: this assumes wallet is synchronized
 876         n = 0
 877         nmax = 0
 878
 879         for account in self.accounts.values():
 880             addresses = account.get_addresses(0)
 881             k = self.num_unused_trailing_addresses(addresses)
 882             for a in addresses[0:-k]:
 883                 if self.history.get(a):
 884                     n = 0
 885                 else:
 886                     n += 1
 887                     if n > nmax: nmax = n
 888         return nmax + 1
 889
 890
 891     def address_is_old(self, address):
 892         age = -1
 893         h = self.history.get(address, [])
 894         if h == ['*']:
 895             return True
 896         for tx_hash, tx_height in h:
 897             if tx_height == 0:
 898                 tx_age = 0
 899             else:
 900                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 901             if tx_age > age:
 902                 age = tx_age
 903         return age > 2
 904
 905
 906     def synchronize_sequence(self, account, for_change):
 907         limit = self.gap_limit_for_change if for_change else self.gap_limit
 908         new_addresses = []
 909         while True:
 910             addresses = account.get_addresses(for_change)
 911             if len(addresses) < limit:
 912                 address = account.create_new_address(for_change)
 913                 self.history[address] = []
 914                 new_addresses.append( address )
 915                 continue
 916
 917             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 918                 break
 919             else:
 920                 address = account.create_new_address(for_change)
 921                 self.history[address] = []
 922                 new_addresses.append( address )
 923
 924         return new_addresses
 925
 926
 927
 928     def create_pending_accounts(self):
 929         for account_type in ['1','2of2','2of3']:
 930             if not self.has_master_public_keys(account_type):
 931                 continue
 932             k, a = self.new_account_address(account_type)
 933             if self.address_is_old(a):
 934                 print_error( "creating account", a )
 935                 self.create_account(account_type)
 936                 self.next_addresses.pop(k)
 937
 938
 939     def synchronize_account(self, account):
 940         new = []
 941         new += self.synchronize_sequence(account, 0)
 942         new += self.synchronize_sequence(account, 1)
 943         return new
 944
 945
 946     def synchronize(self):
 947         if self.master_public_keys:
 948             self.create_pending_accounts()
 949         new = []
 950         for account in self.accounts.values():
 951             new += self.synchronize_account(account)
 952         if new:
 953             self.save_accounts()
 954             self.storage.put('addr_history', self.history, True)
 955         return new
 956
 957
 958     def is_found(self):
 959         return self.history.values() != [[]] * len(self.history)
 960
 961
 962     def add_contact(self, address, label=None):
 963         self.addressbook.append(address)
 964         self.storage.put('contacts', self.addressbook, True)
 965         if label:
 966             self.set_label(address, label)
 967
 968
 969     def delete_contact(self, addr):
 970         if addr in self.addressbook:
 971             self.addressbook.remove(addr)
 972             self.storage.put('addressbook', self.addressbook, True)
 973
 974
 975     def fill_addressbook(self):
 976         for tx_hash, tx in self.transactions.items():
 977             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 978             if is_send:
 979                 for addr, v in tx.outputs:
 980                     if not self.is_mine(addr) and addr not in self.addressbook:
 981                         self.addressbook.append(addr)
 982         # redo labels
 983         # self.update_tx_labels()
 984
 985     def get_num_tx(self, address):
 986         n = 0
 987         for tx in self.transactions.values():
 988             if address in map(lambda x:x[0], tx.outputs): n += 1
 989         return n
 990
 991
 992     def get_address_flags(self, addr):
 993         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 994         flags += "F" if addr in self.frozen_addresses else "-"
 995         return flags
 996
 997
 998     def get_tx_value(self, tx, account=None):
 999         domain = self.get_account_addresses(account)
1000         return tx.get_value(domain, self.prevout_values)
1001
1002
1003     def update_tx_outputs(self, tx_hash):
1004         tx = self.transactions.get(tx_hash)
1005
1006         for i, (addr, value) in enumerate(tx.outputs):
1007             key = tx_hash+ ':%d'%i
1008             self.prevout_values[key] = value
1009
1010         for item in tx.inputs:
1011             if self.is_mine(item.get('address')):
1012                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1013                 self.spent_outputs.append(key)
1014
1015
1016     def get_addr_balance(self, address):
1017         assert self.is_mine(address)
1018         h = self.history.get(address,[])
1019         if h == ['*']: return 0,0
1020         c = u = 0
1021         received_coins = []   # list of coins received at address
1022
1023         for tx_hash, tx_height in h:
1024             tx = self.transactions.get(tx_hash)
1025             if not tx: continue
1026
1027             for i, (addr, value) in enumerate(tx.outputs):
1028                 if addr == address:
1029                     key = tx_hash + ':%d'%i
1030                     received_coins.append(key)
1031
1032         for tx_hash, tx_height in h:
1033             tx = self.transactions.get(tx_hash)
1034             if not tx: continue
1035             v = 0
1036
1037             for item in tx.inputs:
1038                 addr = item.get('address')
1039                 if addr == address:
1040                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1041                     value = self.prevout_values.get( key )
1042                     if key in received_coins:
1043                         v -= value
1044
1045             for i, (addr, value) in enumerate(tx.outputs):
1046                 key = tx_hash + ':%d'%i
1047                 if addr == address:
1048                     v += value
1049
1050             if tx_height:
1051                 c += v
1052             else:
1053                 u += v
1054         return c, u
1055
1056
1057     def get_account_name(self, k):
1058         if k == 0:
1059             if self.seed_version == 4:
1060                 name = 'Main account'
1061             else:
1062                 name = 'Old account'
1063         else:
1064             default = "Unnamed account"
1065             m = re.match("m/0'/(\d+)", k)
1066             if m:
1067                 num = m.group(1)
1068                 if num == '0':
1069                     default = "Main account"
1070                 else:
1071                     default = "Account %s"%num
1072
1073             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1074             if m:
1075                 num = m.group(1)
1076                 default = "2of2 account %s"%num
1077             name = self.labels.get(k, default)
1078
1079         return name
1080
1081     def get_account_names(self):
1082         accounts = {}
1083         for k, account in self.accounts.items():
1084             accounts[k] = self.get_account_name(k)
1085         if self.imported_keys:
1086             accounts[-1] = 'Imported keys'
1087         return accounts
1088
1089     def get_account_addresses(self, a, include_change=True):
1090         if a is None:
1091             o = self.addresses(True)
1092         elif a == -1:
1093             o = self.imported_keys.keys()
1094         else:
1095             ac = self.accounts[a]
1096             o = ac.get_addresses(0)
1097             if include_change: o += ac.get_addresses(1)
1098         return o
1099
1100     def get_imported_balance(self):
1101         return self.get_balance(self.imported_keys.keys())
1102
1103     def get_account_balance(self, account):
1104         return self.get_balance(self.get_account_addresses(account))
1105
1106     def get_frozen_balance(self):
1107         return self.get_balance(self.frozen_addresses)
1108
1109     def get_balance(self, domain=None):
1110         if domain is None: domain = self.addresses(True)
1111         cc = uu = 0
1112         for addr in domain:
1113             c, u = self.get_addr_balance(addr)
1114             cc += c
1115             uu += u
1116         return cc, uu
1117
1118
1119     def get_unspent_coins(self, domain=None):
1120         coins = []
1121         if domain is None: domain = self.addresses(True)
1122         for addr in domain:
1123             h = self.history.get(addr, [])
1124             if h == ['*']: continue
1125             for tx_hash, tx_height in h:
1126                 tx = self.transactions.get(tx_hash)
1127                 if tx is None: raise Exception("Wallet not synchronized")
1128                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1129                 for o in tx.d.get('outputs'):
1130                     output = o.copy()
1131                     if output.get('address') != addr: continue
1132                     key = tx_hash + ":%d" % output.get('prevout_n')
1133                     if key in self.spent_outputs: continue
1134                     output['prevout_hash'] = tx_hash
1135                     output['height'] = tx_height
1136                     output['coinbase'] = is_coinbase
1137                     coins.append((tx_height, output))
1138
1139         # sort by age
1140         if coins:
1141             coins = sorted(coins)
1142             if coins[-1][0] != 0:
1143                 while coins[0][0] == 0:
1144                     coins = coins[1:] + [ coins[0] ]
1145         return [x[1] for x in coins]
1146
1147
1148     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1149         """ todo: minimize tx size """
1150         total = 0
1151         fee = self.fee if fixed_fee is None else fixed_fee
1152         if domain is None:
1153             domain = self.addresses(True)
1154
1155         for i in self.frozen_addresses:
1156             if i in domain: domain.remove(i)
1157
1158         coins = self.get_unspent_coins(domain)
1159         inputs = []
1160
1161         for item in coins:
1162             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1163                 continue
1164             addr = item.get('address')
1165             v = item.get('value')
1166             total += v
1167             inputs.append(item)
1168             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1169             if total >= amount + fee: break
1170         else:
1171             inputs = []
1172
1173         return inputs, total, fee
1174
1175
1176     def set_fee(self, fee):
1177         if self.fee != fee:
1178             self.fee = fee
1179             self.storage.put('fee_per_kb', self.fee, True)
1180
1181     def estimated_fee(self, inputs):
1182         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1183         fee = self.fee * int(round(estimated_size/1024.))
1184         if fee == 0: fee = self.fee
1185         return fee
1186
1187
1188     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1189         "add change to a transaction"
1190         change_amount = total - ( amount + fee )
1191         if change_amount > DUST_THRESHOLD:
1192             if not change_addr:
1193
1194                 # send change to one of the accounts involved in the tx
1195                 address = inputs[0].get('address')
1196                 account, _ = self.get_address_index(address)
1197
1198                 if not self.use_change or account == -1:
1199                     change_addr = inputs[-1]['address']
1200                 else:
1201                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1202
1203             # Insert the change output at a random position in the outputs
1204             posn = random.randint(0, len(outputs))
1205             outputs[posn:posn] = [( change_addr,  change_amount)]
1206         return outputs
1207
1208
1209     def get_history(self, address):
1210         with self.lock:
1211             return self.history.get(address)
1212
1213
1214     def get_status(self, h):
1215         if not h: return None
1216         if h == ['*']: return '*'
1217         status = ''
1218         for tx_hash, height in h:
1219             status += tx_hash + ':%d:' % height
1220         return hashlib.sha256( status ).digest().encode('hex')
1221
1222
1223     def receive_tx_callback(self, tx_hash, tx, tx_height):
1224
1225         with self.transaction_lock:
1226             self.add_extra_addresses(tx)
1227             if not self.check_new_tx(tx_hash, tx):
1228                 # may happen due to pruning
1229                 print_error("received transaction that is no longer referenced in history", tx_hash)
1230                 return
1231             self.transactions[tx_hash] = tx
1232             self.network.interface.pending_transactions_for_notifications.append(tx)
1233             self.save_transactions()
1234             if self.verifier and tx_height>0:
1235                 self.verifier.add(tx_hash, tx_height)
1236             self.update_tx_outputs(tx_hash)
1237
1238
1239     def save_transactions(self):
1240         tx = {}
1241         for k,v in self.transactions.items():
1242             tx[k] = str(v)
1243         self.storage.put('transactions', tx, True)
1244
1245     def receive_history_callback(self, addr, hist):
1246
1247         if not self.check_new_history(addr, hist):
1248             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1249
1250         with self.lock:
1251             self.history[addr] = hist
1252             self.storage.put('addr_history', self.history, True)
1253
1254         if hist != ['*']:
1255             for tx_hash, tx_height in hist:
1256                 if tx_height>0:
1257                     # add it in case it was previously unconfirmed
1258                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1259
1260
1261     def get_tx_history(self, account=None):
1262         if not self.verifier:
1263             return []
1264
1265         with self.transaction_lock:
1266             history = self.transactions.items()
1267             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1268             result = []
1269
1270             balance = 0
1271             for tx_hash, tx in history:
1272                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1273                 if v is not None: balance += v
1274
1275             c, u = self.get_account_balance(account)
1276
1277             if balance != c+u:
1278                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1279
1280             balance = c + u - balance
1281             for tx_hash, tx in history:
1282                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1283                 if not is_relevant:
1284                     continue
1285                 if value is not None:
1286                     balance += value
1287
1288                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1289                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1290
1291         return result
1292
1293
1294     def get_label(self, tx_hash):
1295         label = self.labels.get(tx_hash)
1296         is_default = (label == '') or (label is None)
1297         if is_default: label = self.get_default_label(tx_hash)
1298         return label, is_default
1299
1300
1301     def get_default_label(self, tx_hash):
1302         tx = self.transactions.get(tx_hash)
1303         default_label = ''
1304         if tx:
1305             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1306             if is_mine:
1307                 for o in tx.outputs:
1308                     o_addr, _ = o
1309                     if not self.is_mine(o_addr):
1310                         try:
1311                             default_label = self.labels[o_addr]
1312                         except KeyError:
1313                             default_label = o_addr
1314                         break
1315                 else:
1316                     default_label = '(internal)'
1317             else:
1318                 for o in tx.outputs:
1319                     o_addr, _ = o
1320                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1321                         break
1322                 else:
1323                     for o in tx.outputs:
1324                         o_addr, _ = o
1325                         if self.is_mine(o_addr):
1326                             break
1327                     else:
1328                         o_addr = None
1329
1330                 if o_addr:
1331                     dest_label = self.labels.get(o_addr)
1332                     try:
1333                         default_label = self.labels[o_addr]
1334                     except KeyError:
1335                         default_label = o_addr
1336
1337         return default_label
1338
1339
1340     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1341         for address, x in outputs:
1342             assert is_valid(address)
1343         amount = sum( map(lambda x:x[1], outputs) )
1344         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1345         if not inputs:
1346             raise ValueError("Not enough funds")
1347         self.add_input_info(inputs)
1348         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1349         return Transaction.from_io(inputs, outputs)
1350
1351
1352     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1353         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1354         keypairs = {}
1355         self.add_keypairs_from_wallet(tx, keypairs, password)
1356         if keypairs:
1357             self.sign_transaction(tx, keypairs)
1358         return tx
1359
1360
1361     def add_input_info(self, inputs):
1362         for txin in inputs:
1363             address = txin['address']
1364             if address in self.imported_keys.keys():
1365                 continue
1366             account, sequence = self.get_address_index(address)
1367             txin['KeyID'] = self.get_keyID(account, sequence)
1368             redeemScript = self.accounts[account].redeem_script(sequence)
1369             if redeemScript:
1370                 txin['redeemScript'] = redeemScript
1371             else:
1372                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1373
1374
1375     def sign_transaction(self, tx, keypairs):
1376         tx.sign(keypairs)
1377         run_hook('sign_transaction', tx)
1378
1379
1380     def sendtx(self, tx):
1381         # synchronous
1382         h = self.send_tx(tx)
1383         self.tx_event.wait()
1384         return self.receive_tx(h, tx)
1385
1386     def send_tx(self, tx):
1387         # asynchronous
1388         self.tx_event.clear()
1389         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1390         return tx.hash()
1391
1392     def on_broadcast(self, i, r):
1393         self.tx_result = r.get('result')
1394         self.tx_event.set()
1395
1396     def receive_tx(self, tx_hash, tx):
1397         out = self.tx_result
1398         if out != tx_hash:
1399             return False, "error: " + out
1400         run_hook('receive_tx', tx, self)
1401         return True, out
1402
1403
1404
1405     def update_password(self, old_password, new_password):
1406         if new_password == '': new_password = None
1407         decoded = self.get_seed(old_password)
1408         self.seed = pw_encode( decoded, new_password)
1409         self.storage.put('seed', self.seed, True)
1410         self.use_encryption = (new_password != None)
1411         self.storage.put('use_encryption', self.use_encryption,True)
1412         for k in self.imported_keys.keys():
1413             a = self.imported_keys[k]
1414             b = pw_decode(a, old_password)
1415             c = pw_encode(b, new_password)
1416             self.imported_keys[k] = c
1417         self.storage.put('imported_keys', self.imported_keys, True)
1418
1419         for k, v in self.master_private_keys.items():
1420             b = pw_decode(v, old_password)
1421             c = pw_encode(b, new_password)
1422             self.master_private_keys[k] = c
1423         self.storage.put('master_private_keys', self.master_private_keys, True)
1424
1425
1426     def freeze(self,addr):
1427         if self.is_mine(addr) and addr not in self.frozen_addresses:
1428             self.frozen_addresses.append(addr)
1429             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1430             return True
1431         else:
1432             return False
1433
1434     def unfreeze(self,addr):
1435         if self.is_mine(addr) and addr in self.frozen_addresses:
1436             self.frozen_addresses.remove(addr)
1437             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1438             return True
1439         else:
1440             return False
1441
1442
1443     def set_verifier(self, verifier):
1444         self.verifier = verifier
1445
1446         # review transactions that are in the history
1447         for addr, hist in self.history.items():
1448             if hist == ['*']: continue
1449             for tx_hash, tx_height in hist:
1450                 if tx_height>0:
1451                     # add it in case it was previously unconfirmed
1452                     self.verifier.add(tx_hash, tx_height)
1453
1454
1455         # if we are on a pruning server, remove unverified transactions
1456         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1457         for tx_hash in self.transactions.keys():
1458             if tx_hash not in vr:
1459                 self.transactions.pop(tx_hash)
1460
1461
1462
1463     def check_new_history(self, addr, hist):
1464
1465         # check that all tx in hist are relevant
1466         if hist != ['*']:
1467             for tx_hash, height in hist:
1468                 tx = self.transactions.get(tx_hash)
1469                 if not tx: continue
1470                 if not tx.has_address(addr):
1471                     return False
1472
1473         # check that we are not "orphaning" a transaction
1474         old_hist = self.history.get(addr,[])
1475         if old_hist == ['*']: return True
1476
1477         for tx_hash, height in old_hist:
1478             if tx_hash in map(lambda x:x[0], hist): continue
1479             found = False
1480             for _addr, _hist in self.history.items():
1481                 if _addr == addr: continue
1482                 if _hist == ['*']: continue
1483                 _tx_hist = map(lambda x:x[0], _hist)
1484                 if tx_hash in _tx_hist:
1485                     found = True
1486                     break
1487
1488             if not found:
1489                 tx = self.transactions.get(tx_hash)
1490                 # tx might not be there
1491                 if not tx: continue
1492
1493                 # already verified?
1494                 if self.verifier.get_height(tx_hash):
1495                     continue
1496                 # unconfirmed tx
1497                 print_error("new history is orphaning transaction:", tx_hash)
1498                 # check that all outputs are not mine, request histories
1499                 ext_requests = []
1500                 for _addr, _v in tx.outputs:
1501                     # assert not self.is_mine(_addr)
1502                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1503
1504                 ext_h = self.network.synchronous_get(ext_requests)
1505                 print_error("sync:", ext_requests, ext_h)
1506                 height = None
1507                 for h in ext_h:
1508                     if h == ['*']: continue
1509                     for item in h:
1510                         if item.get('tx_hash') == tx_hash:
1511                             height = item.get('height')
1512                 if height:
1513                     print_error("found height for", tx_hash, height)
1514                     self.verifier.add(tx_hash, height)
1515                 else:
1516                     print_error("removing orphaned tx from history", tx_hash)
1517                     self.transactions.pop(tx_hash)
1518
1519         return True
1520
1521
1522
1523     def check_new_tx(self, tx_hash, tx):
1524         # 1 check that tx is referenced in addr_history.
1525         addresses = []
1526         for addr, hist in self.history.items():
1527             if hist == ['*']:continue
1528             for txh, height in hist:
1529                 if txh == tx_hash:
1530                     addresses.append(addr)
1531
1532         if not addresses:
1533             return False
1534
1535         # 2 check that referencing addresses are in the tx
1536         for addr in addresses:
1537             if not tx.has_address(addr):
1538                 return False
1539
1540         return True
1541
1542
1543     def start_threads(self, network):
1544         from verifier import TxVerifier
1545         self.network = network
1546         if self.network is not None:
1547             self.verifier = TxVerifier(self.network, self.storage)
1548             self.verifier.start()
1549             self.set_verifier(self.verifier)
1550             self.synchronizer = WalletSynchronizer(self, network)
1551             self.synchronizer.start()
1552         else:
1553             self.verifier = None
1554             self.synchronizer =None
1555
1556     def stop_threads(self):
1557         if self.network:
1558             self.verifier.stop()
1559             self.synchronizer.stop()
1560
1561
1562     def restore(self, callback):
1563         from i18n import _
1564         def wait_for_wallet():
1565             self.set_up_to_date(False)
1566             while not self.is_up_to_date():
1567                 msg = "%s\n%s %d\n%s %.1f"%(
1568                     _("Please wait..."),
1569                     _("Addresses generated:"),
1570                     len(self.addresses(True)),
1571                     _("Kilobytes received:"),
1572                     self.network.interface.bytes_received/1024.)
1573
1574                 apply(callback, (msg,))
1575                 time.sleep(0.1)
1576
1577         def wait_for_network():
1578             while not self.network.is_connected():
1579                 msg = "%s \n" % (_("Connecting..."))
1580                 apply(callback, (msg,))
1581                 time.sleep(0.1)
1582
1583         # wait until we are connected, because the user might have selected another server
1584         if self.network:
1585             wait_for_network()
1586             wait_for_wallet()
1587         else:
1588             self.synchronize()
1589
1590         self.fill_addressbook()
1591
1592
1593
1594
1595 class WalletSynchronizer(threading.Thread):
1596
1597
1598     def __init__(self, wallet, network):
1599         threading.Thread.__init__(self)
1600         self.daemon = True
1601         self.wallet = wallet
1602         self.network = network
1603         self.was_updated = True
1604         self.running = False
1605         self.lock = threading.Lock()
1606         self.queue = Queue.Queue()
1607
1608     def stop(self):
1609         with self.lock: self.running = False
1610
1611     def is_running(self):
1612         with self.lock: return self.running
1613
1614
1615     def subscribe_to_addresses(self, addresses):
1616         messages = []
1617         for addr in addresses:
1618             messages.append(('blockchain.address.subscribe', [addr]))
1619         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1620
1621
1622     def run(self):
1623         with self.lock:
1624             self.running = True
1625
1626         while self.is_running():
1627
1628             if not self.network.is_connected():
1629                 self.network.wait_until_connected()
1630
1631             self.run_interface()
1632
1633
1634     def run_interface(self):
1635
1636         print_error("synchronizer: connected to", self.network.main_server())
1637
1638         requested_tx = []
1639         missing_tx = []
1640         requested_histories = {}
1641
1642         # request any missing transactions
1643         for history in self.wallet.history.values():
1644             if history == ['*']: continue
1645             for tx_hash, tx_height in history:
1646                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1647                     missing_tx.append( (tx_hash, tx_height) )
1648
1649         if missing_tx:
1650             print_error("missing tx", missing_tx)
1651
1652         # subscriptions
1653         self.subscribe_to_addresses(self.wallet.addresses(True))
1654
1655         while self.is_running():
1656             # 1. create new addresses
1657             new_addresses = self.wallet.synchronize()
1658
1659             # request missing addresses
1660             if new_addresses:
1661                 self.subscribe_to_addresses(new_addresses)
1662
1663             # request missing transactions
1664             for tx_hash, tx_height in missing_tx:
1665                 if (tx_hash, tx_height) not in requested_tx:
1666                     self.network.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1667                     requested_tx.append( (tx_hash, tx_height) )
1668             missing_tx = []
1669
1670             # detect if situation has changed
1671             if self.network.is_up_to_date() and self.queue.empty():
1672                 if not self.wallet.is_up_to_date():
1673                     self.wallet.set_up_to_date(True)
1674                     self.was_updated = True
1675             else:
1676                 if self.wallet.is_up_to_date():
1677                     self.wallet.set_up_to_date(False)
1678                     self.was_updated = True
1679
1680             if self.was_updated:
1681                 self.network.trigger_callback('updated')
1682                 self.was_updated = False
1683
1684             # 2. get a response
1685             try:
1686                 r = self.queue.get(block=True, timeout=1)
1687             except Queue.Empty:
1688                 continue
1689
1690             # see if it changed
1691             #if interface != self.network.interface:
1692             #    break
1693
1694             if not r:
1695                 continue
1696
1697             # 3. handle response
1698             method = r['method']
1699             params = r['params']
1700             result = r.get('result')
1701             error = r.get('error')
1702             if error:
1703                 print "error", r
1704                 continue
1705
1706             if method == 'blockchain.address.subscribe':
1707                 addr = params[0]
1708                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1709                     if requested_histories.get(addr) is None:
1710                         self.network.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1711                         requested_histories[addr] = result
1712
1713             elif method == 'blockchain.address.get_history':
1714                 addr = params[0]
1715                 print_error("receiving history", addr, result)
1716                 if result == ['*']:
1717                     assert requested_histories.pop(addr) == '*'
1718                     self.wallet.receive_history_callback(addr, result)
1719                 else:
1720                     hist = []
1721                     # check that txids are unique
1722                     txids = []
1723                     for item in result:
1724                         tx_hash = item['tx_hash']
1725                         if tx_hash not in txids:
1726                             txids.append(tx_hash)
1727                             hist.append( (tx_hash, item['height']) )
1728
1729                     if len(hist) != len(result):
1730                         raise Exception("error: server sent history with non-unique txid", result)
1731
1732                     # check that the status corresponds to what was announced
1733                     rs = requested_histories.pop(addr)
1734                     if self.wallet.get_status(hist) != rs:
1735                         raise Exception("error: status mismatch: %s"%addr)
1736
1737                     # store received history
1738                     self.wallet.receive_history_callback(addr, hist)
1739
1740                     # request transactions that we don't have
1741                     for tx_hash, tx_height in hist:
1742                         if self.wallet.transactions.get(tx_hash) is None:
1743                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1744                                 missing_tx.append( (tx_hash, tx_height) )
1745
1746             elif method == 'blockchain.transaction.get':
1747                 tx_hash = params[0]
1748                 tx_height = params[1]
1749                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1750                 tx = Transaction(result)
1751                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1752                 self.was_updated = True
1753                 requested_tx.remove( (tx_hash, tx_height) )
1754                 print_error("received tx:", tx_hash, len(tx.raw))
1755
1756             else:
1757                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1758
1759             if self.was_updated and not requested_tx:
1760                 self.network.trigger_callback('updated')
1761                 # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1762                 self.network.trigger_callback("new_transaction")
1763                 self.was_updated = False