lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         v = self.data.get(key)
 129         if v is None:
 130             v = default
 131         return v
 132
 133     def put(self, key, value, save = True):
 134
 135         with self.lock:
 136             if value is not None:
 137                 self.data[key] = value
 138             else:
 139                 self.data.pop(key)
 140             if save:
 141                 self.write()
 142
 143     def write(self):
 144         s = repr(self.data)
 145         f = open(self.path,"w")
 146         f.write( s )
 147         f.close()
 148         if 'ANDROID_DATA' not in os.environ:
 149             import stat
 150             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 151
 152
 153 class Wallet:
 154
 155     def __init__(self, storage):
 156
 157         self.storage = storage
 158         self.electrum_version = ELECTRUM_VERSION
 159         self.gap_limit_for_change = 3 # constant
 160
 161         # saved fields
 162         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 163
 164         self.gap_limit             = storage.get('gap_limit', 5)
 165         self.use_change            = storage.get('use_change',True)
 166         self.use_encryption        = storage.get('use_encryption', False)
 167         self.seed                  = storage.get('seed', '')               # encrypted
 168         self.labels                = storage.get('labels', {})
 169         self.frozen_addresses      = storage.get('frozen_addresses',[])
 170         self.addressbook           = storage.get('contacts', [])
 171
 172         self.imported_keys         = storage.get('imported_keys',{})
 173         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 174
 175         self.fee                   = int(storage.get('fee_per_kb',20000))
 176
 177         self.master_public_keys = storage.get('master_public_keys',{})
 178         self.master_private_keys = storage.get('master_private_keys', {})
 179
 180         self.next_addresses = storage.get('next_addresses',{})
 181
 182         if self.seed_version not in [4, 6]:
 183             msg = "This wallet seed is not supported."
 184             if self.seed_version in [5]:
 185                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 186                 print msg
 187                 sys.exit(1)
 188
 189         # This attribute is set when wallet.start_threads is called.
 190         self.synchronizer = None
 191
 192         self.load_accounts()
 193
 194         self.transactions = {}
 195         tx_list = self.storage.get('transactions',{})
 196         for k,v in tx_list.items():
 197             try:
 198                 tx = Transaction(v)
 199             except Exception:
 200                 print_msg("Warning: Cannot deserialize transactions. skipping")
 201                 continue
 202
 203             self.add_extra_addresses(tx)
 204             self.transactions[k] = tx
 205
 206         for h,tx in self.transactions.items():
 207             if not self.check_new_tx(h, tx):
 208                 print_error("removing unreferenced tx", h)
 209                 self.transactions.pop(h)
 210
 211
 212         # not saved
 213         self.prevout_values = {}     # my own transaction outputs
 214         self.spent_outputs = []
 215
 216         # spv
 217         self.verifier = None
 218
 219         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 220         # interface.is_up_to_date() returns true when all requests have been answered and processed
 221         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 222
 223         self.up_to_date = False
 224         self.lock = threading.Lock()
 225         self.transaction_lock = threading.Lock()
 226         self.tx_event = threading.Event()
 227
 228         for tx_hash, tx in self.transactions.items():
 229             self.update_tx_outputs(tx_hash)
 230
 231
 232     def add_extra_addresses(self, tx):
 233         h = tx.hash()
 234         # find the address corresponding to pay-to-pubkey inputs
 235         tx.add_extra_addresses(self.transactions)
 236         for o in tx.d.get('outputs'):
 237             if o.get('is_pubkey'):
 238                 for tx2 in self.transactions.values():
 239                     tx2.add_extra_addresses({h:tx})
 240
 241
 242
 243
 244     def set_up_to_date(self,b):
 245         with self.lock: self.up_to_date = b
 246
 247
 248     def is_up_to_date(self):
 249         with self.lock: return self.up_to_date
 250
 251
 252     def update(self):
 253         self.up_to_date = False
 254         while not self.is_up_to_date():
 255             time.sleep(0.1)
 256
 257
 258     def import_key(self, sec, password):
 259         # check password
 260         seed = self.get_seed(password)
 261         try:
 262             address = address_from_private_key(sec)
 263         except Exception:
 264             raise Exception('Invalid private key')
 265
 266         if self.is_mine(address):
 267             raise Exception('Address already in wallet')
 268
 269         # store the originally requested keypair into the imported keys table
 270         self.imported_keys[address] = pw_encode(sec, password )
 271         self.storage.put('imported_keys', self.imported_keys, True)
 272         if self.synchronizer:
 273             self.synchronizer.subscribe_to_addresses([address])
 274         return address
 275
 276     def delete_imported_key(self, addr):
 277         if addr in self.imported_keys:
 278             self.imported_keys.pop(addr)
 279             self.storage.put('imported_keys', self.imported_keys, True)
 280
 281
 282     def make_seed(self):
 283         import mnemonic, ecdsa
 284         entropy = ecdsa.util.randrange( pow(2,160) )
 285         nonce = 0
 286         while True:
 287             ss = "%040x"%(entropy+nonce)
 288             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 289             # we keep only 13 words, that's approximately 139 bits of entropy
 290             words = mnemonic.mn_encode(s)[0:13]
 291             seed = ' '.join(words)
 292             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 293                 break  # this removes 12 bits of entropy
 294             nonce += 1
 295
 296         return seed
 297
 298
 299     def init_seed(self, seed):
 300         import mnemonic
 301
 302         if self.seed:
 303             raise Exception("a seed exists")
 304
 305         if not seed:
 306             self.seed = random_seed(128)
 307             self.seed_version = 4
 308             return
 309
 310         #if not seed:
 311         #    self.seed = self.make_seed()
 312         #    self.seed_version = SEED_VERSION
 313         #    return
 314
 315         # find out what kind of wallet we are
 316         try:
 317             seed.strip().decode('hex')
 318             self.seed_version = 4
 319             self.seed = str(seed)
 320             return
 321         except Exception:
 322             pass
 323
 324         words = seed.split()
 325         self.seed_version = 4
 326         self.seed = mnemonic.mn_decode(words)
 327
 328         #try:
 329         #    mnemonic.mn_decode(words)
 330         #    uses_electrum_words = True
 331         #except Exception:
 332         #    uses_electrum_words = False
 333         #
 334         #if uses_electrum_words and len(words) != 13:
 335         #    self.seed_version = 4
 336         #    self.seed = mnemonic.mn_decode(words)
 337         #else:
 338         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 339         #    self.seed_version = SEED_VERSION
 340         #    self.seed = seed
 341
 342
 343     def save_seed(self):
 344         self.storage.put('seed', self.seed, True)
 345         self.storage.put('seed_version', self.seed_version, True)
 346         self.create_accounts()
 347
 348
 349     def create_watching_only_wallet(self, params):
 350         K0, c0 = params
 351         if not K0:
 352             return
 353
 354         if not c0:
 355             self.seed_version = 4
 356             self.storage.put('seed_version', self.seed_version, True)
 357             self.create_old_account(K0)
 358             return
 359
 360         cK0 = ""
 361         self.master_public_keys = {
 362             "m/0'/": (c0, K0, cK0),
 363             }
 364         self.storage.put('master_public_keys', self.master_public_keys, True)
 365         self.storage.put('seed_version', self.seed_version, True)
 366         self.create_account('1','Main account')
 367
 368
 369     def create_accounts(self):
 370         if self.seed_version == 4:
 371             mpk = OldAccount.mpk_from_seed(self.seed)
 372             self.create_old_account(mpk)
 373         else:
 374             # create default account
 375             self.create_master_keys('1')
 376             self.create_account('1','Main account')
 377
 378
 379     def create_master_keys(self, account_type):
 380         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 381         if account_type == '1':
 382             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 383             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 384             self.master_private_keys["m/0'/"] = k0
 385         elif account_type == '2of2':
 386             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 387             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 388             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 389             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 390             self.master_private_keys["m/1'/"] = k1
 391             self.master_private_keys["m/2'/"] = k2
 392         elif account_type == '2of3':
 393             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 394             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 395             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 396             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 397             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 398             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 399             self.master_private_keys["m/3'/"] = k3
 400             self.master_private_keys["m/4'/"] = k4
 401             self.master_private_keys["m/5'/"] = k5
 402
 403         self.storage.put('master_public_keys', self.master_public_keys, True)
 404         self.storage.put('master_private_keys', self.master_private_keys, True)
 405
 406     def has_master_public_keys(self, account_type):
 407         if account_type == '1':
 408             return "m/0'/" in self.master_public_keys
 409         elif account_type == '2of2':
 410             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 411         elif account_type == '2of3':
 412             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 413
 414     def find_root_by_master_key(self, c, K):
 415         for key, v in self.master_public_keys.items():
 416             if key == "m/":continue
 417             cc, KK, _ = v
 418             if (c == cc) and (K == KK):
 419                 return key
 420
 421     def deseed_root(self, seed, password):
 422         # for safety, we ask the user to enter their seed
 423         assert seed == self.get_seed(password)
 424         self.seed = ''
 425         self.storage.put('seed', '', True)
 426
 427
 428     def deseed_branch(self, k):
 429         # check that parent has no seed
 430         assert self.seed == ''
 431         self.master_private_keys.pop(k)
 432         self.storage.put('master_private_keys', self.master_private_keys, True)
 433
 434     def is_watching_only(self):
 435         return (self.seed == '') and (self.master_private_keys == {})
 436
 437
 438
 439     def account_id(self, account_type, i):
 440         if account_type == '1':
 441             return "m/0'/%d"%i
 442         elif account_type == '2of2':
 443             return "m/1'/%d & m/2'/%d"%(i,i)
 444         elif account_type == '2of3':
 445             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 446         else:
 447             raise Exception('unknown account type')
 448
 449
 450     def num_accounts(self, account_type):
 451         keys = self.accounts.keys()
 452         i = 0
 453         while True:
 454             account_id = self.account_id(account_type, i)
 455             if account_id not in keys: break
 456             i += 1
 457         return i
 458
 459
 460     def new_account_address(self, account_type = '1'):
 461         i = self.num_accounts(account_type)
 462         k = self.account_id(account_type,i)
 463
 464         addr = self.next_addresses.get(k)
 465         if not addr:
 466             account_id, account = self.next_account(account_type)
 467             addr = account.first_address()
 468             self.next_addresses[k] = addr
 469             self.storage.put('next_addresses',self.next_addresses)
 470
 471         return k, addr
 472
 473
 474     def next_account(self, account_type = '1'):
 475
 476         i = self.num_accounts(account_type)
 477         account_id = self.account_id(account_type,i)
 478
 479         if account_type is '1':
 480             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 481             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 482             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 483
 484         elif account_type == '2of2':
 485             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 486             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 487             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 488             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 489             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 490
 491         elif account_type == '2of3':
 492             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 493             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 494             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 495             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 496             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 497             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 498             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 499
 500         return account_id, account
 501
 502
 503     def set_label(self, name, text = None):
 504         changed = False
 505         old_text = self.labels.get(name)
 506         if text:
 507             if old_text != text:
 508                 self.labels[name] = text
 509                 changed = True
 510         else:
 511             if old_text:
 512                 self.labels.pop(name)
 513                 changed = True
 514
 515         if changed:
 516             self.storage.put('labels', self.labels, True)
 517
 518         run_hook('set_label', name, text, changed)
 519         return changed
 520
 521
 522
 523     def create_account(self, account_type = '1', name = None):
 524         k, account = self.next_account(account_type)
 525         if k in self.pending_accounts:
 526             self.pending_accounts.pop(k)
 527             self.storage.put('pending_accounts', self.pending_accounts)
 528
 529         self.accounts[k] = account
 530         self.save_accounts()
 531         if name:
 532             self.set_label(k, name)
 533
 534
 535     def create_old_account(self, mpk):
 536         self.storage.put('master_public_key', mpk, True)
 537         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 538         self.save_accounts()
 539
 540
 541     def save_accounts(self):
 542         d = {}
 543         for k, v in self.accounts.items():
 544             d[k] = v.dump()
 545         self.storage.put('accounts', d, True)
 546
 547
 548
 549     def load_accounts(self):
 550         d = self.storage.get('accounts', {})
 551         self.accounts = {}
 552         for k, v in d.items():
 553             if k == 0:
 554                 v['mpk'] = self.storage.get('master_public_key')
 555                 self.accounts[k] = OldAccount(v)
 556             elif '&' in k:
 557                 self.accounts[k] = BIP32_Account_2of2(v)
 558             else:
 559                 self.accounts[k] = BIP32_Account(v)
 560
 561         self.pending_accounts = self.storage.get('pending_accounts',{})
 562
 563
 564     def delete_pending_account(self, k):
 565         self.pending_accounts.pop(k)
 566         self.storage.put('pending_accounts', self.pending_accounts)
 567
 568     def account_is_pending(self, k):
 569         return k in self.pending_accounts
 570
 571     def create_pending_account(self, acct_type, name):
 572         k, addr = self.new_account_address(acct_type)
 573         self.set_label(k, name)
 574         self.pending_accounts[k] = addr
 575         self.storage.put('pending_accounts', self.pending_accounts)
 576
 577     def get_pending_accounts(self):
 578         return self.pending_accounts.items()
 579
 580
 581     def addresses(self, include_change = True, _next=True):
 582         o = self.get_account_addresses(-1, include_change)
 583         for a in self.accounts.keys():
 584             o += self.get_account_addresses(a, include_change)
 585
 586         if _next:
 587             for addr in self.next_addresses.values():
 588                 if addr not in o:
 589                     o += [addr]
 590         return o
 591
 592
 593     def is_mine(self, address):
 594         return address in self.addresses(True)
 595
 596
 597     def is_change(self, address):
 598         if not self.is_mine(address): return False
 599         if address in self.imported_keys.keys(): return False
 600         acct, s = self.get_address_index(address)
 601         if s is None: return False
 602         return s[0] == 1
 603
 604     def get_master_public_key(self):
 605         if self.seed_version == 4:
 606             return self.storage.get("master_public_key")
 607         else:
 608             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 609             return repr((c, K))
 610
 611     def get_master_private_key(self, account, password):
 612         master_k = pw_decode( self.master_private_keys[account], password)
 613         master_c, master_K, master_Kc = self.master_public_keys[account]
 614         try:
 615             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 616             assert K.encode('hex') == master_K
 617         except Exception:
 618             raise Exception("Invalid password")
 619         return master_k
 620
 621
 622     def get_address_index(self, address):
 623         if address in self.imported_keys.keys():
 624             return -1, None
 625
 626         for account in self.accounts.keys():
 627             for for_change in [0,1]:
 628                 addresses = self.accounts[account].get_addresses(for_change)
 629                 for addr in addresses:
 630                     if address == addr:
 631                         return account, (for_change, addresses.index(addr))
 632
 633         for k,v in self.next_addresses.items():
 634             if v == address:
 635                 return k, (0,0)
 636
 637         raise Exception("Address not found", address)
 638
 639
 640     def get_roots(self, account):
 641         roots = []
 642         for a in account.split('&'):
 643             s = a.strip()
 644             m = re.match("(m/\d+'/)(\d+)", s)
 645             roots.append( m.group(1) )
 646         return roots
 647
 648     def is_seeded(self, account):
 649         if type(account) is int:
 650             return self.seed is not None
 651
 652         for root in self.get_roots(account):
 653             if root not in self.master_private_keys.keys():
 654                 return False
 655         return True
 656
 657     def rebase_sequence(self, account, sequence):
 658         c, i = sequence
 659         dd = []
 660         for a in account.split('&'):
 661             s = a.strip()
 662             m = re.match("(m/\d+'/)(\d+)", s)
 663             root = m.group(1)
 664             num = int(m.group(2))
 665             dd.append( (root, [num,c,i] ) )
 666         return dd
 667
 668
 669     def get_keyID(self, account, sequence):
 670         if account == 0:
 671             a, b = sequence
 672             mpk = self.storage.get('master_public_key')
 673             return 'old(%s,%d,%d)'%(mpk,a,b)
 674
 675         rs = self.rebase_sequence(account, sequence)
 676         dd = []
 677         for root, public_sequence in rs:
 678             c, K, _ = self.master_public_keys[root]
 679             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 680             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 681         return '&'.join(dd)
 682
 683
 684
 685     def get_seed(self, password):
 686         s = pw_decode(self.seed, password)
 687         if self.seed_version == 4:
 688             seed = s
 689             self.accounts[0].check_seed(seed)
 690         else:
 691             seed = mnemonic_hash(s)
 692         return seed
 693
 694
 695     def get_mnemonic(self, password):
 696         import mnemonic
 697         s = pw_decode(self.seed, password)
 698         if self.seed_version == 4:
 699             return ' '.join(mnemonic.mn_encode(s))
 700         else:
 701             return s
 702
 703
 704
 705     def get_private_key(self, address, password):
 706         if self.is_watching_only():
 707             return []
 708
 709         # first check the provided password
 710         seed = self.get_seed(password)
 711
 712         out = []
 713         if address in self.imported_keys.keys():
 714             out.append( pw_decode( self.imported_keys[address], password ) )
 715         else:
 716             account, sequence = self.get_address_index(address)
 717             if account == 0:
 718                 pk = self.accounts[account].get_private_key(seed, sequence)
 719                 out.append(pk)
 720                 return out
 721
 722             # assert address == self.accounts[account].get_address(*sequence)
 723             rs = self.rebase_sequence( account, sequence)
 724             for root, public_sequence in rs:
 725
 726                 if root not in self.master_private_keys.keys(): continue
 727                 master_k = self.get_master_private_key(root, password)
 728                 master_c, _, _ = self.master_public_keys[root]
 729                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 730                 out.append(pk)
 731
 732         return out
 733
 734
 735     def add_keypairs_from_wallet(self, tx, keypairs, password):
 736         for txin in tx.inputs:
 737             address = txin['address']
 738             private_keys = self.get_private_key(address, password)
 739             for sec in private_keys:
 740                 pubkey = public_key_from_private_key(sec)
 741                 keypairs[ pubkey ] = sec
 742                 if address in self.imported_keys.keys():
 743                     txin['redeemPubkey'] = pubkey
 744
 745
 746     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 747         for txin in tx.inputs:
 748             keyid = txin.get('KeyID')
 749             if keyid:
 750
 751                 if self.seed_version==4:
 752                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 753                     if not m: continue
 754                     mpk = m.group(1)
 755                     if mpk != self.storage.get('master_public_key'): continue
 756                     index = int(m.group(2))
 757                     num = int(m.group(3))
 758                     account = self.accounts[0]
 759                     addr = account.get_address(index, num)
 760                     txin['address'] = addr # fixme: side effect
 761                     pk = self.get_private_key(addr, password)
 762                     for sec in pk:
 763                         pubkey = public_key_from_private_key(sec)
 764                         keypairs[pubkey] = sec
 765                     continue
 766
 767
 768                 roots = []
 769                 for s in keyid.split('&'):
 770                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 771                     if not m: continue
 772                     c = m.group(1)
 773                     K = m.group(2)
 774                     sequence = m.group(3)
 775                     root = self.find_root_by_master_key(c,K)
 776                     if not root: continue
 777                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 778                     root = root + '%d'%sequence[0]
 779                     sequence = sequence[1:]
 780                     roots.append((root,sequence))
 781
 782                 account_id = " & ".join( map(lambda x:x[0], roots) )
 783                 account = self.accounts.get(account_id)
 784                 if not account: continue
 785                 addr = account.get_address(*sequence)
 786                 txin['address'] = addr # fixme: side effect
 787                 pk = self.get_private_key(addr, password)
 788                 for sec in pk:
 789                     pubkey = public_key_from_private_key(sec)
 790                     keypairs[pubkey] = sec
 791
 792
 793
 794     def signrawtransaction(self, tx, input_info, private_keys, password):
 795
 796         # check that the password is correct
 797         seed = self.get_seed(password)
 798
 799         # add input info
 800         tx.add_input_info(input_info)
 801
 802         # add redeem script for coins that are in the wallet
 803         # FIXME: add redeemPubkey too!
 804         unspent_coins = self.get_unspent_coins()
 805         for txin in tx.inputs:
 806             for item in unspent_coins:
 807                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 808                     print_error( "tx input is in unspent coins" )
 809                     txin['scriptPubKey'] = item['scriptPubKey']
 810                     account, sequence = self.get_address_index(item['address'])
 811                     if account != -1:
 812                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 813                         print_error("added redeemScript", txin['redeemScript'])
 814                     break
 815
 816
 817         # build a list of public/private keys
 818         keypairs = {}
 819
 820         # add private keys from parameter
 821         for sec in private_keys:
 822             pubkey = public_key_from_private_key(sec)
 823             keypairs[ pubkey ] = sec
 824
 825         # add private_keys from KeyID
 826         self.add_keypairs_from_KeyID(tx, keypairs, password)
 827
 828         # add private keys from wallet
 829         self.add_keypairs_from_wallet(tx, keypairs, password)
 830         self.sign_transaction(tx, keypairs)
 831
 832
 833     def sign_message(self, address, message, password):
 834         keys = self.get_private_key(address, password)
 835         assert len(keys) == 1
 836         sec = keys[0]
 837         key = regenerate_key(sec)
 838         compressed = is_compressed(sec)
 839         return key.sign_message(message, compressed, address)
 840
 841
 842     def change_gap_limit(self, value):
 843         if value >= self.gap_limit:
 844             self.gap_limit = value
 845             self.storage.put('gap_limit', self.gap_limit, True)
 846             #self.interface.poke('synchronizer')
 847             return True
 848
 849         elif value >= self.min_acceptable_gap():
 850             for key, account in self.accounts.items():
 851                 addresses = account[0]
 852                 k = self.num_unused_trailing_addresses(addresses)
 853                 n = len(addresses) - k + value
 854                 addresses = addresses[0:n]
 855                 self.accounts[key][0] = addresses
 856
 857             self.gap_limit = value
 858             self.storage.put('gap_limit', self.gap_limit, True)
 859             self.save_accounts()
 860             return True
 861         else:
 862             return False
 863
 864     def num_unused_trailing_addresses(self, addresses):
 865         k = 0
 866         for a in addresses[::-1]:
 867             if self.history.get(a):break
 868             k = k + 1
 869         return k
 870
 871     def min_acceptable_gap(self):
 872         # fixme: this assumes wallet is synchronized
 873         n = 0
 874         nmax = 0
 875
 876         for account in self.accounts.values():
 877             addresses = account.get_addresses(0)
 878             k = self.num_unused_trailing_addresses(addresses)
 879             for a in addresses[0:-k]:
 880                 if self.history.get(a):
 881                     n = 0
 882                 else:
 883                     n += 1
 884                     if n > nmax: nmax = n
 885         return nmax + 1
 886
 887
 888     def address_is_old(self, address):
 889         age = -1
 890         h = self.history.get(address, [])
 891         if h == ['*']:
 892             return True
 893         for tx_hash, tx_height in h:
 894             if tx_height == 0:
 895                 tx_age = 0
 896             else:
 897                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 898             if tx_age > age:
 899                 age = tx_age
 900         return age > 2
 901
 902
 903     def synchronize_sequence(self, account, for_change):
 904         limit = self.gap_limit_for_change if for_change else self.gap_limit
 905         new_addresses = []
 906         while True:
 907             addresses = account.get_addresses(for_change)
 908             if len(addresses) < limit:
 909                 address = account.create_new_address(for_change)
 910                 self.history[address] = []
 911                 new_addresses.append( address )
 912                 continue
 913
 914             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 915                 break
 916             else:
 917                 address = account.create_new_address(for_change)
 918                 self.history[address] = []
 919                 new_addresses.append( address )
 920
 921         return new_addresses
 922
 923
 924
 925     def create_pending_accounts(self):
 926         for account_type in ['1','2of2','2of3']:
 927             if not self.has_master_public_keys(account_type):
 928                 continue
 929             k, a = self.new_account_address(account_type)
 930             if self.address_is_old(a):
 931                 print_error( "creating account", a )
 932                 self.create_account(account_type)
 933                 self.next_addresses.pop(k)
 934
 935
 936     def synchronize_account(self, account):
 937         new = []
 938         new += self.synchronize_sequence(account, 0)
 939         new += self.synchronize_sequence(account, 1)
 940         return new
 941
 942
 943     def synchronize(self):
 944         if self.master_public_keys:
 945             self.create_pending_accounts()
 946         new = []
 947         for account in self.accounts.values():
 948             new += self.synchronize_account(account)
 949         if new:
 950             self.save_accounts()
 951             self.storage.put('addr_history', self.history, True)
 952         return new
 953
 954
 955     def is_found(self):
 956         return self.history.values() != [[]] * len(self.history)
 957
 958
 959     def add_contact(self, address, label=None):
 960         self.addressbook.append(address)
 961         self.storage.put('contacts', self.addressbook, True)
 962         if label:
 963             self.set_label(address, label)
 964
 965
 966     def delete_contact(self, addr):
 967         if addr in self.addressbook:
 968             self.addressbook.remove(addr)
 969             self.storage.put('addressbook', self.addressbook, True)
 970
 971
 972     def fill_addressbook(self):
 973         for tx_hash, tx in self.transactions.items():
 974             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 975             if is_send:
 976                 for addr, v in tx.outputs:
 977                     if not self.is_mine(addr) and addr not in self.addressbook:
 978                         self.addressbook.append(addr)
 979         # redo labels
 980         # self.update_tx_labels()
 981
 982     def get_num_tx(self, address):
 983         n = 0
 984         for tx in self.transactions.values():
 985             if address in map(lambda x:x[0], tx.outputs): n += 1
 986         return n
 987
 988
 989     def get_address_flags(self, addr):
 990         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 991         flags += "F" if addr in self.frozen_addresses else "-"
 992         return flags
 993
 994
 995     def get_tx_value(self, tx, account=None):
 996         domain = self.get_account_addresses(account)
 997         return tx.get_value(domain, self.prevout_values)
 998
 999
1000     def update_tx_outputs(self, tx_hash):
1001         tx = self.transactions.get(tx_hash)
1002
1003         for i, (addr, value) in enumerate(tx.outputs):
1004             key = tx_hash+ ':%d'%i
1005             self.prevout_values[key] = value
1006
1007         for item in tx.inputs:
1008             if self.is_mine(item.get('address')):
1009                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1010                 self.spent_outputs.append(key)
1011
1012
1013     def get_addr_balance(self, address):
1014         assert self.is_mine(address)
1015         h = self.history.get(address,[])
1016         if h == ['*']: return 0,0
1017         c = u = 0
1018         received_coins = []   # list of coins received at address
1019
1020         for tx_hash, tx_height in h:
1021             tx = self.transactions.get(tx_hash)
1022             if not tx: continue
1023
1024             for i, (addr, value) in enumerate(tx.outputs):
1025                 if addr == address:
1026                     key = tx_hash + ':%d'%i
1027                     received_coins.append(key)
1028
1029         for tx_hash, tx_height in h:
1030             tx = self.transactions.get(tx_hash)
1031             if not tx: continue
1032             v = 0
1033
1034             for item in tx.inputs:
1035                 addr = item.get('address')
1036                 if addr == address:
1037                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1038                     value = self.prevout_values.get( key )
1039                     if key in received_coins:
1040                         v -= value
1041
1042             for i, (addr, value) in enumerate(tx.outputs):
1043                 key = tx_hash + ':%d'%i
1044                 if addr == address:
1045                     v += value
1046
1047             if tx_height:
1048                 c += v
1049             else:
1050                 u += v
1051         return c, u
1052
1053
1054     def get_account_name(self, k):
1055         if k == 0:
1056             if self.seed_version == 4:
1057                 name = 'Main account'
1058             else:
1059                 name = 'Old account'
1060         else:
1061             default = "Unnamed account"
1062             m = re.match("m/0'/(\d+)", k)
1063             if m:
1064                 num = m.group(1)
1065                 if num == '0':
1066                     default = "Main account"
1067                 else:
1068                     default = "Account %s"%num
1069
1070             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1071             if m:
1072                 num = m.group(1)
1073                 default = "2of2 account %s"%num
1074             name = self.labels.get(k, default)
1075
1076         return name
1077
1078     def get_account_names(self):
1079         accounts = {}
1080         for k, account in self.accounts.items():
1081             accounts[k] = self.get_account_name(k)
1082         if self.imported_keys:
1083             accounts[-1] = 'Imported keys'
1084         return accounts
1085
1086     def get_account_addresses(self, a, include_change=True):
1087         if a is None:
1088             o = self.addresses(True)
1089         elif a == -1:
1090             o = self.imported_keys.keys()
1091         else:
1092             ac = self.accounts[a]
1093             o = ac.get_addresses(0)
1094             if include_change: o += ac.get_addresses(1)
1095         return o
1096
1097     def get_imported_balance(self):
1098         return self.get_balance(self.imported_keys.keys())
1099
1100     def get_account_balance(self, account):
1101         return self.get_balance(self.get_account_addresses(account))
1102
1103     def get_frozen_balance(self):
1104         return self.get_balance(self.frozen_addresses)
1105
1106     def get_balance(self, domain=None):
1107         if domain is None: domain = self.addresses(True)
1108         cc = uu = 0
1109         for addr in domain:
1110             c, u = self.get_addr_balance(addr)
1111             cc += c
1112             uu += u
1113         return cc, uu
1114
1115
1116     def get_unspent_coins(self, domain=None):
1117         coins = []
1118         if domain is None: domain = self.addresses(True)
1119         for addr in domain:
1120             h = self.history.get(addr, [])
1121             if h == ['*']: continue
1122             for tx_hash, tx_height in h:
1123                 tx = self.transactions.get(tx_hash)
1124                 if tx is None: raise Exception("Wallet not synchronized")
1125                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1126                 for o in tx.d.get('outputs'):
1127                     output = o.copy()
1128                     if output.get('address') != addr: continue
1129                     key = tx_hash + ":%d" % output.get('prevout_n')
1130                     if key in self.spent_outputs: continue
1131                     output['prevout_hash'] = tx_hash
1132                     output['height'] = tx_height
1133                     output['coinbase'] = is_coinbase
1134                     coins.append((tx_height, output))
1135
1136         # sort by age
1137         if coins:
1138             coins = sorted(coins)
1139             if coins[-1][0] != 0:
1140                 while coins[0][0] == 0:
1141                     coins = coins[1:] + [ coins[0] ]
1142         return [x[1] for x in coins]
1143
1144
1145     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1146         """ todo: minimize tx size """
1147         total = 0
1148         fee = self.fee if fixed_fee is None else fixed_fee
1149         if domain is None:
1150             domain = self.addresses(True)
1151
1152         for i in self.frozen_addresses:
1153             if i in domain: domain.remove(i)
1154
1155         coins = self.get_unspent_coins(domain)
1156         inputs = []
1157
1158         for item in coins:
1159             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1160                 continue
1161             addr = item.get('address')
1162             v = item.get('value')
1163             total += v
1164             inputs.append(item)
1165             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1166             if total >= amount + fee: break
1167         else:
1168             inputs = []
1169
1170         return inputs, total, fee
1171
1172
1173     def set_fee(self, fee):
1174         if self.fee != fee:
1175             self.fee = fee
1176             self.storage.put('fee_per_kb', self.fee, True)
1177
1178     def estimated_fee(self, inputs):
1179         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1180         fee = self.fee * int(round(estimated_size/1024.))
1181         if fee == 0: fee = self.fee
1182         return fee
1183
1184
1185     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1186         "add change to a transaction"
1187         change_amount = total - ( amount + fee )
1188         if change_amount > DUST_THRESHOLD:
1189             if not change_addr:
1190
1191                 # send change to one of the accounts involved in the tx
1192                 address = inputs[0].get('address')
1193                 account, _ = self.get_address_index(address)
1194
1195                 if not self.use_change or account == -1:
1196                     change_addr = inputs[-1]['address']
1197                 else:
1198                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1199
1200             # Insert the change output at a random position in the outputs
1201             posn = random.randint(0, len(outputs))
1202             outputs[posn:posn] = [( change_addr,  change_amount)]
1203         return outputs
1204
1205
1206     def get_history(self, address):
1207         with self.lock:
1208             return self.history.get(address)
1209
1210
1211     def get_status(self, h):
1212         if not h: return None
1213         if h == ['*']: return '*'
1214         status = ''
1215         for tx_hash, height in h:
1216             status += tx_hash + ':%d:' % height
1217         return hashlib.sha256( status ).digest().encode('hex')
1218
1219
1220     def receive_tx_callback(self, tx_hash, tx, tx_height):
1221
1222         with self.transaction_lock:
1223             self.add_extra_addresses(tx)
1224             if not self.check_new_tx(tx_hash, tx):
1225                 # may happen due to pruning
1226                 print_error("received transaction that is no longer referenced in history", tx_hash)
1227                 return
1228             self.transactions[tx_hash] = tx
1229             self.network.interface.pending_transactions_for_notifications.append(tx)
1230             self.save_transactions()
1231             if self.verifier and tx_height>0:
1232                 self.verifier.add(tx_hash, tx_height)
1233             self.update_tx_outputs(tx_hash)
1234
1235
1236     def save_transactions(self):
1237         tx = {}
1238         for k,v in self.transactions.items():
1239             tx[k] = str(v)
1240         self.storage.put('transactions', tx, True)
1241
1242     def receive_history_callback(self, addr, hist):
1243
1244         if not self.check_new_history(addr, hist):
1245             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1246
1247         with self.lock:
1248             self.history[addr] = hist
1249             self.storage.put('addr_history', self.history, True)
1250
1251         if hist != ['*']:
1252             for tx_hash, tx_height in hist:
1253                 if tx_height>0:
1254                     # add it in case it was previously unconfirmed
1255                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1256
1257
1258     def get_tx_history(self, account=None):
1259         if not self.verifier:
1260             return []
1261
1262         with self.transaction_lock:
1263             history = self.transactions.items()
1264             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1265             result = []
1266
1267             balance = 0
1268             for tx_hash, tx in history:
1269                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1270                 if v is not None: balance += v
1271
1272             c, u = self.get_account_balance(account)
1273
1274             if balance != c+u:
1275                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1276
1277             balance = c + u - balance
1278             for tx_hash, tx in history:
1279                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1280                 if not is_relevant:
1281                     continue
1282                 if value is not None:
1283                     balance += value
1284
1285                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1286                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1287
1288         return result
1289
1290
1291     def get_label(self, tx_hash):
1292         label = self.labels.get(tx_hash)
1293         is_default = (label == '') or (label is None)
1294         if is_default: label = self.get_default_label(tx_hash)
1295         return label, is_default
1296
1297
1298     def get_default_label(self, tx_hash):
1299         tx = self.transactions.get(tx_hash)
1300         default_label = ''
1301         if tx:
1302             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1303             if is_mine:
1304                 for o in tx.outputs:
1305                     o_addr, _ = o
1306                     if not self.is_mine(o_addr):
1307                         try:
1308                             default_label = self.labels[o_addr]
1309                         except KeyError:
1310                             default_label = o_addr
1311                         break
1312                 else:
1313                     default_label = '(internal)'
1314             else:
1315                 for o in tx.outputs:
1316                     o_addr, _ = o
1317                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1318                         break
1319                 else:
1320                     for o in tx.outputs:
1321                         o_addr, _ = o
1322                         if self.is_mine(o_addr):
1323                             break
1324                     else:
1325                         o_addr = None
1326
1327                 if o_addr:
1328                     dest_label = self.labels.get(o_addr)
1329                     try:
1330                         default_label = self.labels[o_addr]
1331                     except KeyError:
1332                         default_label = o_addr
1333
1334         return default_label
1335
1336
1337     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1338         for address, x in outputs:
1339             assert is_valid(address)
1340         amount = sum( map(lambda x:x[1], outputs) )
1341         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1342         if not inputs:
1343             raise ValueError("Not enough funds")
1344         self.add_input_info(inputs)
1345         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1346         return Transaction.from_io(inputs, outputs)
1347
1348
1349     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1350         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1351         keypairs = {}
1352         self.add_keypairs_from_wallet(tx, keypairs, password)
1353         if keypairs:
1354             self.sign_transaction(tx, keypairs)
1355         return tx
1356
1357
1358     def add_input_info(self, inputs):
1359         for txin in inputs:
1360             address = txin['address']
1361             if address in self.imported_keys.keys():
1362                 continue
1363             account, sequence = self.get_address_index(address)
1364             txin['KeyID'] = self.get_keyID(account, sequence)
1365             redeemScript = self.accounts[account].redeem_script(sequence)
1366             if redeemScript:
1367                 txin['redeemScript'] = redeemScript
1368             else:
1369                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1370
1371
1372     def sign_transaction(self, tx, keypairs):
1373         tx.sign(keypairs)
1374         run_hook('sign_transaction', tx)
1375
1376
1377     def sendtx(self, tx):
1378         # synchronous
1379         h = self.send_tx(tx)
1380         self.tx_event.wait()
1381         return self.receive_tx(h)
1382
1383     def send_tx(self, tx):
1384         # asynchronous
1385         self.tx_event.clear()
1386         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1387         return tx.hash()
1388
1389     def on_broadcast(self, i, r):
1390         self.tx_result = r.get('result')
1391         self.tx_event.set()
1392
1393     def receive_tx(self,tx_hash):
1394         out = self.tx_result
1395         if out != tx_hash:
1396             return False, "error: " + out
1397         return True, out
1398
1399
1400
1401     def update_password(self, old_password, new_password):
1402         if new_password == '': new_password = None
1403         decoded = self.get_seed(old_password)
1404         self.seed = pw_encode( decoded, new_password)
1405         self.storage.put('seed', self.seed, True)
1406         self.use_encryption = (new_password != None)
1407         self.storage.put('use_encryption', self.use_encryption,True)
1408         for k in self.imported_keys.keys():
1409             a = self.imported_keys[k]
1410             b = pw_decode(a, old_password)
1411             c = pw_encode(b, new_password)
1412             self.imported_keys[k] = c
1413         self.storage.put('imported_keys', self.imported_keys, True)
1414
1415         for k, v in self.master_private_keys.items():
1416             b = pw_decode(v, old_password)
1417             c = pw_encode(b, new_password)
1418             self.master_private_keys[k] = c
1419         self.storage.put('master_private_keys', self.master_private_keys, True)
1420
1421
1422     def freeze(self,addr):
1423         if self.is_mine(addr) and addr not in self.frozen_addresses:
1424             self.frozen_addresses.append(addr)
1425             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1426             return True
1427         else:
1428             return False
1429
1430     def unfreeze(self,addr):
1431         if self.is_mine(addr) and addr in self.frozen_addresses:
1432             self.frozen_addresses.remove(addr)
1433             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1434             return True
1435         else:
1436             return False
1437
1438
1439     def set_verifier(self, verifier):
1440         self.verifier = verifier
1441
1442         # review transactions that are in the history
1443         for addr, hist in self.history.items():
1444             if hist == ['*']: continue
1445             for tx_hash, tx_height in hist:
1446                 if tx_height>0:
1447                     # add it in case it was previously unconfirmed
1448                     self.verifier.add(tx_hash, tx_height)
1449
1450
1451         # if we are on a pruning server, remove unverified transactions
1452         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1453         for tx_hash in self.transactions.keys():
1454             if tx_hash not in vr:
1455                 self.transactions.pop(tx_hash)
1456
1457
1458
1459     def check_new_history(self, addr, hist):
1460
1461         # check that all tx in hist are relevant
1462         if hist != ['*']:
1463             for tx_hash, height in hist:
1464                 tx = self.transactions.get(tx_hash)
1465                 if not tx: continue
1466                 if not tx.has_address(addr):
1467                     return False
1468
1469         # check that we are not "orphaning" a transaction
1470         old_hist = self.history.get(addr,[])
1471         if old_hist == ['*']: return True
1472
1473         for tx_hash, height in old_hist:
1474             if tx_hash in map(lambda x:x[0], hist): continue
1475             found = False
1476             for _addr, _hist in self.history.items():
1477                 if _addr == addr: continue
1478                 if _hist == ['*']: continue
1479                 _tx_hist = map(lambda x:x[0], _hist)
1480                 if tx_hash in _tx_hist:
1481                     found = True
1482                     break
1483
1484             if not found:
1485                 tx = self.transactions.get(tx_hash)
1486                 # tx might not be there
1487                 if not tx: continue
1488
1489                 # already verified?
1490                 if self.verifier.get_height(tx_hash):
1491                     continue
1492                 # unconfirmed tx
1493                 print_error("new history is orphaning transaction:", tx_hash)
1494                 # check that all outputs are not mine, request histories
1495                 ext_requests = []
1496                 for _addr, _v in tx.outputs:
1497                     # assert not self.is_mine(_addr)
1498                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1499
1500                 ext_h = self.network.synchronous_get(ext_requests)
1501                 print_error("sync:", ext_requests, ext_h)
1502                 height = None
1503                 for h in ext_h:
1504                     if h == ['*']: continue
1505                     for item in h:
1506                         if item.get('tx_hash') == tx_hash:
1507                             height = item.get('height')
1508                 if height:
1509                     print_error("found height for", tx_hash, height)
1510                     self.verifier.add(tx_hash, height)
1511                 else:
1512                     print_error("removing orphaned tx from history", tx_hash)
1513                     self.transactions.pop(tx_hash)
1514
1515         return True
1516
1517
1518
1519     def check_new_tx(self, tx_hash, tx):
1520         # 1 check that tx is referenced in addr_history.
1521         addresses = []
1522         for addr, hist in self.history.items():
1523             if hist == ['*']:continue
1524             for txh, height in hist:
1525                 if txh == tx_hash:
1526                     addresses.append(addr)
1527
1528         if not addresses:
1529             return False
1530
1531         # 2 check that referencing addresses are in the tx
1532         for addr in addresses:
1533             if not tx.has_address(addr):
1534                 return False
1535
1536         return True
1537
1538
1539     def start_threads(self, network):
1540         from verifier import TxVerifier
1541         self.network = network
1542         if self.network:
1543             self.verifier = TxVerifier(self.network, self.storage)
1544             self.verifier.start()
1545             self.set_verifier(self.verifier)
1546             self.synchronizer = WalletSynchronizer(self, network)
1547             self.synchronizer.start()
1548         else:
1549             self.verifier = None
1550             self.synchronizer =None
1551
1552     def stop_threads(self):
1553         if self.network:
1554             self.verifier.stop()
1555             self.synchronizer.stop()
1556
1557
1558     def restore(self, callback):
1559         from i18n import _
1560         def wait_for_wallet():
1561             self.set_up_to_date(False)
1562             while not self.is_up_to_date():
1563                 msg = "%s\n%s %d\n%s %.1f"%(
1564                     _("Please wait..."),
1565                     _("Addresses generated:"),
1566                     len(self.addresses(True)),
1567                     _("Kilobytes received:"),
1568                     self.network.interface.bytes_received/1024.)
1569
1570                 apply(callback, (msg,))
1571                 time.sleep(0.1)
1572
1573         def wait_for_network():
1574             while not self.network.is_connected():
1575                 msg = "%s \n" % (_("Connecting..."))
1576                 apply(callback, (msg,))
1577                 time.sleep(0.1)
1578
1579         # wait until we are connected, because the user might have selected another server
1580         if self.network:
1581             wait_for_network()
1582             wait_for_wallet()
1583         else:
1584             self.synchronize()
1585
1586         self.fill_addressbook()
1587
1588
1589
1590
1591 class WalletSynchronizer(threading.Thread):
1592
1593
1594     def __init__(self, wallet, network):
1595         threading.Thread.__init__(self)
1596         self.daemon = True
1597         self.wallet = wallet
1598         self.network = network
1599         self.was_updated = True
1600         self.running = False
1601         self.lock = threading.Lock()
1602         self.queue = Queue.Queue()
1603
1604     def stop(self):
1605         with self.lock: self.running = False
1606
1607     def is_running(self):
1608         with self.lock: return self.running
1609
1610
1611     def subscribe_to_addresses(self, addresses):
1612         messages = []
1613         for addr in addresses:
1614             messages.append(('blockchain.address.subscribe', [addr]))
1615         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1616
1617
1618     def run(self):
1619         with self.lock:
1620             self.running = True
1621
1622         while self.is_running():
1623
1624             if not self.network.is_connected():
1625                 print_error("synchronizer: waiting for interface")
1626                 self.network.wait_until_connected()
1627
1628             self.run_interface(self.network.interface)
1629
1630
1631     def run_interface(self, interface):
1632
1633         print_error("synchronizer: connected to", interface.server)
1634
1635         requested_tx = []
1636         missing_tx = []
1637         requested_histories = {}
1638
1639         # request any missing transactions
1640         for history in self.wallet.history.values():
1641             if history == ['*']: continue
1642             for tx_hash, tx_height in history:
1643                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1644                     missing_tx.append( (tx_hash, tx_height) )
1645
1646         if missing_tx:
1647             print_error("missing tx", missing_tx)
1648
1649         # subscriptions
1650         self.subscribe_to_addresses(self.wallet.addresses(True))
1651
1652         while self.is_running():
1653             # 1. create new addresses
1654             new_addresses = self.wallet.synchronize()
1655
1656             # request missing addresses
1657             if new_addresses:
1658                 self.subscribe_to_addresses(new_addresses)
1659
1660             # request missing transactions
1661             for tx_hash, tx_height in missing_tx:
1662                 if (tx_hash, tx_height) not in requested_tx:
1663                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1664                     requested_tx.append( (tx_hash, tx_height) )
1665             missing_tx = []
1666
1667             # detect if situation has changed
1668             if interface.is_up_to_date() and self.queue.empty():
1669                 if not self.wallet.is_up_to_date():
1670                     self.wallet.set_up_to_date(True)
1671                     self.was_updated = True
1672             else:
1673                 if self.wallet.is_up_to_date():
1674                     self.wallet.set_up_to_date(False)
1675                     self.was_updated = True
1676
1677             if self.was_updated:
1678                 self.wallet.network.trigger_callback('updated')
1679                 self.was_updated = False
1680
1681             # 2. get a response
1682             try:
1683                 r = self.queue.get(block=True, timeout=1)
1684             except Queue.Empty:
1685                 continue
1686
1687             if interface != self.network.interface:
1688                 break
1689
1690             if not r:
1691                 continue
1692
1693             # 3. handle response
1694             method = r['method']
1695             params = r['params']
1696             result = r.get('result')
1697             error = r.get('error')
1698             if error:
1699                 print "error", r
1700                 continue
1701
1702             if method == 'blockchain.address.subscribe':
1703                 addr = params[0]
1704                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1705                     if requested_histories.get(addr) is None:
1706                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1707                         requested_histories[addr] = result
1708
1709             elif method == 'blockchain.address.get_history':
1710                 addr = params[0]
1711                 print_error("receiving history", addr, result)
1712                 if result == ['*']:
1713                     assert requested_histories.pop(addr) == '*'
1714                     self.wallet.receive_history_callback(addr, result)
1715                 else:
1716                     hist = []
1717                     # check that txids are unique
1718                     txids = []
1719                     for item in result:
1720                         tx_hash = item['tx_hash']
1721                         if tx_hash not in txids:
1722                             txids.append(tx_hash)
1723                             hist.append( (tx_hash, item['height']) )
1724
1725                     if len(hist) != len(result):
1726                         raise Exception("error: server sent history with non-unique txid", result)
1727
1728                     # check that the status corresponds to what was announced
1729                     rs = requested_histories.pop(addr)
1730                     if self.wallet.get_status(hist) != rs:
1731                         raise Exception("error: status mismatch: %s"%addr)
1732
1733                     # store received history
1734                     self.wallet.receive_history_callback(addr, hist)
1735
1736                     # request transactions that we don't have
1737                     for tx_hash, tx_height in hist:
1738                         if self.wallet.transactions.get(tx_hash) is None:
1739                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1740                                 missing_tx.append( (tx_hash, tx_height) )
1741
1742             elif method == 'blockchain.transaction.get':
1743                 tx_hash = params[0]
1744                 tx_height = params[1]
1745                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1746                 tx = Transaction(result)
1747                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1748                 self.was_updated = True
1749                 requested_tx.remove( (tx_hash, tx_height) )
1750                 print_error("received tx:", tx_hash, len(tx.raw))
1751
1752             else:
1753                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1754
1755             if self.was_updated and not requested_tx:
1756                 self.wallet.network.trigger_callback('updated')
1757                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1758
1759                 self.was_updated = False