lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         v = self.data.get(key)
 129         if v is None:
 130             v = default
 131         return v
 132
 133     def put(self, key, value, save = True):
 134
 135         with self.lock:
 136             if value is not None:
 137                 self.data[key] = value
 138             else:
 139                 self.data.pop(key)
 140             if save:
 141                 self.write()
 142
 143     def write(self):
 144         s = repr(self.data)
 145         f = open(self.path,"w")
 146         f.write( s )
 147         f.close()
 148         if 'ANDROID_DATA' not in os.environ:
 149             import stat
 150             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 151
 152
 153 class Wallet:
 154
 155     def __init__(self, storage):
 156
 157         self.storage = storage
 158         self.electrum_version = ELECTRUM_VERSION
 159         self.gap_limit_for_change = 3 # constant
 160
 161         # saved fields
 162         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 163
 164         self.gap_limit             = storage.get('gap_limit', 5)
 165         self.use_change            = storage.get('use_change',True)
 166         self.use_encryption        = storage.get('use_encryption', False)
 167         self.seed                  = storage.get('seed', '')               # encrypted
 168         self.labels                = storage.get('labels', {})
 169         self.frozen_addresses      = storage.get('frozen_addresses',[])
 170         self.addressbook           = storage.get('contacts', [])
 171
 172         self.imported_keys         = storage.get('imported_keys',{})
 173         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 174
 175         self.fee                   = int(storage.get('fee_per_kb',20000))
 176
 177         self.master_public_keys = storage.get('master_public_keys',{})
 178         self.master_private_keys = storage.get('master_private_keys', {})
 179
 180         self.next_addresses = storage.get('next_addresses',{})
 181
 182         if self.seed_version not in [4, 6]:
 183             msg = "This wallet seed is not supported."
 184             if self.seed_version in [5]:
 185                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 186                 print msg
 187                 sys.exit(1)
 188
 189         # This attribute is set when wallet.start_threads is called.
 190         self.synchronizer = None
 191
 192         self.load_accounts()
 193
 194         self.transactions = {}
 195         tx_list = self.storage.get('transactions',{})
 196         for k,v in tx_list.items():
 197             try:
 198                 tx = Transaction(v)
 199             except Exception:
 200                 print_msg("Warning: Cannot deserialize transactions. skipping")
 201                 continue
 202
 203             self.add_extra_addresses(tx)
 204             self.transactions[k] = tx
 205
 206         for h,tx in self.transactions.items():
 207             if not self.check_new_tx(h, tx):
 208                 print_error("removing unreferenced tx", h)
 209                 self.transactions.pop(h)
 210
 211
 212         # not saved
 213         self.prevout_values = {}     # my own transaction outputs
 214         self.spent_outputs = []
 215
 216         # spv
 217         self.verifier = None
 218
 219         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 220         # interface.is_up_to_date() returns true when all requests have been answered and processed
 221         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 222
 223         self.up_to_date = False
 224         self.lock = threading.Lock()
 225         self.transaction_lock = threading.Lock()
 226         self.tx_event = threading.Event()
 227
 228         for tx_hash, tx in self.transactions.items():
 229             self.update_tx_outputs(tx_hash)
 230
 231
 232     def add_extra_addresses(self, tx):
 233         h = tx.hash()
 234         # find the address corresponding to pay-to-pubkey inputs
 235         tx.add_extra_addresses(self.transactions)
 236         for o in tx.d.get('outputs'):
 237             if o.get('is_pubkey'):
 238                 for tx2 in self.transactions.values():
 239                     tx2.add_extra_addresses({h:tx})
 240
 241
 242
 243
 244     def set_up_to_date(self,b):
 245         with self.lock: self.up_to_date = b
 246
 247
 248     def is_up_to_date(self):
 249         with self.lock: return self.up_to_date
 250
 251
 252     def update(self):
 253         self.up_to_date = False
 254         while not self.is_up_to_date():
 255             time.sleep(0.1)
 256
 257
 258     def import_key(self, sec, password):
 259         # check password
 260         seed = self.get_seed(password)
 261         try:
 262             address = address_from_private_key(sec)
 263         except Exception:
 264             raise Exception('Invalid private key')
 265
 266         if self.is_mine(address):
 267             raise Exception('Address already in wallet')
 268
 269         # store the originally requested keypair into the imported keys table
 270         self.imported_keys[address] = pw_encode(sec, password )
 271         self.storage.put('imported_keys', self.imported_keys, True)
 272         if self.synchronizer:
 273             self.synchronizer.subscribe_to_addresses([address])
 274         return address
 275
 276     def delete_imported_key(self, addr):
 277         if addr in self.imported_keys:
 278             self.imported_keys.pop(addr)
 279             self.storage.put('imported_keys', self.imported_keys, True)
 280
 281
 282     def make_seed(self):
 283         import mnemonic, ecdsa
 284         entropy = ecdsa.util.randrange( pow(2,160) )
 285         nonce = 0
 286         while True:
 287             ss = "%040x"%(entropy+nonce)
 288             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 289             # we keep only 13 words, that's approximately 139 bits of entropy
 290             words = mnemonic.mn_encode(s)[0:13]
 291             seed = ' '.join(words)
 292             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 293                 break  # this removes 12 bits of entropy
 294             nonce += 1
 295
 296         return seed
 297
 298
 299     def init_seed(self, seed):
 300         import mnemonic
 301
 302         if self.seed:
 303             raise Exception("a seed exists")
 304
 305         if not seed:
 306             self.seed = random_seed(128)
 307             self.seed_version = 4
 308             return
 309
 310         #if not seed:
 311         #    self.seed = self.make_seed()
 312         #    self.seed_version = SEED_VERSION
 313         #    return
 314
 315         # find out what kind of wallet we are
 316         try:
 317             seed.strip().decode('hex')
 318             self.seed_version = 4
 319             self.seed = str(seed)
 320             return
 321         except Exception:
 322             pass
 323
 324         words = seed.split()
 325         self.seed_version = 4
 326         self.seed = mnemonic.mn_decode(words)
 327
 328         #try:
 329         #    mnemonic.mn_decode(words)
 330         #    uses_electrum_words = True
 331         #except Exception:
 332         #    uses_electrum_words = False
 333         #
 334         #if uses_electrum_words and len(words) != 13:
 335         #    self.seed_version = 4
 336         #    self.seed = mnemonic.mn_decode(words)
 337         #else:
 338         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 339         #    self.seed_version = SEED_VERSION
 340         #    self.seed = seed
 341
 342
 343     def save_seed(self):
 344         self.storage.put('seed', self.seed, True)
 345         self.storage.put('seed_version', self.seed_version, True)
 346         self.create_accounts()
 347
 348
 349     def create_watching_only_wallet(self, params):
 350         K0, c0 = params
 351         if not K0:
 352             return
 353
 354         if not c0:
 355             self.seed_version = 4
 356             self.storage.put('seed_version', self.seed_version, True)
 357             self.create_old_account(K0)
 358             return
 359
 360         cK0 = ""
 361         self.master_public_keys = {
 362             "m/0'/": (c0, K0, cK0),
 363             }
 364         self.storage.put('master_public_keys', self.master_public_keys, True)
 365         self.storage.put('seed_version', self.seed_version, True)
 366         self.create_account('1','Main account')
 367
 368
 369     def create_accounts(self):
 370         if self.seed_version == 4:
 371             mpk = OldAccount.mpk_from_seed(self.seed)
 372             self.create_old_account(mpk)
 373         else:
 374             # create default account
 375             self.create_master_keys('1')
 376             self.create_account('1','Main account')
 377
 378
 379     def create_master_keys(self, account_type):
 380         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 381         if account_type == '1':
 382             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 383             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 384             self.master_private_keys["m/0'/"] = k0
 385         elif account_type == '2of2':
 386             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 387             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 388             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 389             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 390             self.master_private_keys["m/1'/"] = k1
 391             self.master_private_keys["m/2'/"] = k2
 392         elif account_type == '2of3':
 393             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 394             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 395             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 396             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 397             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 398             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 399             self.master_private_keys["m/3'/"] = k3
 400             self.master_private_keys["m/4'/"] = k4
 401             self.master_private_keys["m/5'/"] = k5
 402
 403         self.storage.put('master_public_keys', self.master_public_keys, True)
 404         self.storage.put('master_private_keys', self.master_private_keys, True)
 405
 406     def has_master_public_keys(self, account_type):
 407         if account_type == '1':
 408             return "m/0'/" in self.master_public_keys
 409         elif account_type == '2of2':
 410             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 411         elif account_type == '2of3':
 412             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 413
 414     def find_root_by_master_key(self, c, K):
 415         for key, v in self.master_public_keys.items():
 416             if key == "m/":continue
 417             cc, KK, _ = v
 418             if (c == cc) and (K == KK):
 419                 return key
 420
 421     def deseed_root(self, seed, password):
 422         # for safety, we ask the user to enter their seed
 423         assert seed == self.get_seed(password)
 424         self.seed = ''
 425         self.storage.put('seed', '', True)
 426
 427
 428     def deseed_branch(self, k):
 429         # check that parent has no seed
 430         assert self.seed == ''
 431         self.master_private_keys.pop(k)
 432         self.storage.put('master_private_keys', self.master_private_keys, True)
 433
 434     def is_watching_only(self):
 435         return (self.seed == '') and (self.master_private_keys == {})
 436
 437
 438
 439     def account_id(self, account_type, i):
 440         if account_type == '1':
 441             return "m/0'/%d"%i
 442         elif account_type == '2of2':
 443             return "m/1'/%d & m/2'/%d"%(i,i)
 444         elif account_type == '2of3':
 445             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 446         else:
 447             raise Exception('unknown account type')
 448
 449
 450     def num_accounts(self, account_type):
 451         keys = self.accounts.keys()
 452         i = 0
 453         while True:
 454             account_id = self.account_id(account_type, i)
 455             if account_id not in keys: break
 456             i += 1
 457         return i
 458
 459
 460     def new_account_address(self, account_type = '1'):
 461         i = self.num_accounts(account_type)
 462         k = self.account_id(account_type,i)
 463
 464         addr = self.next_addresses.get(k)
 465         if not addr:
 466             account_id, account = self.next_account(account_type)
 467             addr = account.first_address()
 468             self.next_addresses[k] = addr
 469             self.storage.put('next_addresses',self.next_addresses)
 470
 471         return k, addr
 472
 473
 474     def next_account(self, account_type = '1'):
 475
 476         i = self.num_accounts(account_type)
 477         account_id = self.account_id(account_type,i)
 478
 479         if account_type is '1':
 480             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 481             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 482             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 483
 484         elif account_type == '2of2':
 485             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 486             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 487             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 488             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 489             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 490
 491         elif account_type == '2of3':
 492             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 493             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 494             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 495             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 496             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 497             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 498             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 499
 500         return account_id, account
 501
 502
 503     def set_label(self, name, text = None):
 504         changed = False
 505         old_text = self.labels.get(name)
 506         if text:
 507             if old_text != text:
 508                 self.labels[name] = text
 509                 changed = True
 510         else:
 511             if old_text:
 512                 self.labels.pop(name)
 513                 changed = True
 514
 515         if changed:
 516             self.storage.put('labels', self.labels, True)
 517
 518         run_hook('set_label', name, text, changed)
 519         return changed
 520
 521
 522
 523     def create_account(self, account_type = '1', name = None):
 524         k, account = self.next_account(account_type)
 525         if k in self.pending_accounts:
 526             self.pending_accounts.pop(k)
 527             self.storage.put('pending_accounts', self.pending_accounts)
 528
 529         self.accounts[k] = account
 530         self.save_accounts()
 531         if name:
 532             self.set_label(k, name)
 533
 534
 535     def create_old_account(self, mpk):
 536         self.storage.put('master_public_key', mpk, True)
 537         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 538         self.save_accounts()
 539
 540
 541     def save_accounts(self):
 542         d = {}
 543         for k, v in self.accounts.items():
 544             d[k] = v.dump()
 545         self.storage.put('accounts', d, True)
 546
 547
 548
 549     def load_accounts(self):
 550         d = self.storage.get('accounts', {})
 551         self.accounts = {}
 552         for k, v in d.items():
 553             if k == 0:
 554                 v['mpk'] = self.storage.get('master_public_key')
 555                 self.accounts[k] = OldAccount(v)
 556             elif '&' in k:
 557                 self.accounts[k] = BIP32_Account_2of2(v)
 558             else:
 559                 self.accounts[k] = BIP32_Account(v)
 560
 561         self.pending_accounts = self.storage.get('pending_accounts',{})
 562
 563
 564     def delete_pending_account(self, k):
 565         self.pending_accounts.pop(k)
 566         self.storage.put('pending_accounts', self.pending_accounts)
 567
 568     def account_is_pending(self, k):
 569         return k in self.pending_accounts
 570
 571     def create_pending_account(self, acct_type, name):
 572         k, addr = self.new_account_address(acct_type)
 573         self.set_label(k, name)
 574         self.pending_accounts[k] = addr
 575         self.storage.put('pending_accounts', self.pending_accounts)
 576
 577     def get_pending_accounts(self):
 578         return self.pending_accounts.items()
 579
 580
 581     def addresses(self, include_change = True, _next=True):
 582         o = self.get_account_addresses(-1, include_change)
 583         for a in self.accounts.keys():
 584             o += self.get_account_addresses(a, include_change)
 585
 586         if _next:
 587             for addr in self.next_addresses.values():
 588                 if addr not in o:
 589                     o += [addr]
 590         return o
 591
 592
 593     def is_mine(self, address):
 594         return address in self.addresses(True)
 595
 596
 597     def is_change(self, address):
 598         if not self.is_mine(address): return False
 599         if address in self.imported_keys.keys(): return False
 600         acct, s = self.get_address_index(address)
 601         if s is None: return False
 602         return s[0] == 1
 603
 604     def get_master_public_key(self):
 605         if self.seed_version == 4:
 606             return self.storage.get("master_public_key")
 607         else:
 608             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 609             return repr((c, K))
 610
 611     def get_master_private_key(self, account, password):
 612         master_k = pw_decode( self.master_private_keys[account], password)
 613         master_c, master_K, master_Kc = self.master_public_keys[account]
 614         try:
 615             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 616             assert K.encode('hex') == master_K
 617         except Exception:
 618             raise Exception("Invalid password")
 619         return master_k
 620
 621
 622     def get_address_index(self, address):
 623         if address in self.imported_keys.keys():
 624             return -1, None
 625
 626         for account in self.accounts.keys():
 627             for for_change in [0,1]:
 628                 addresses = self.accounts[account].get_addresses(for_change)
 629                 for addr in addresses:
 630                     if address == addr:
 631                         return account, (for_change, addresses.index(addr))
 632
 633         for k,v in self.next_addresses.items():
 634             if v == address:
 635                 return k, (0,0)
 636
 637         raise Exception("Address not found", address)
 638
 639
 640     def get_roots(self, account):
 641         roots = []
 642         for a in account.split('&'):
 643             s = a.strip()
 644             m = re.match("(m/\d+'/)(\d+)", s)
 645             roots.append( m.group(1) )
 646         return roots
 647
 648     def is_seeded(self, account):
 649         if type(account) is int:
 650             return self.seed is not None
 651
 652         for root in self.get_roots(account):
 653             if root not in self.master_private_keys.keys():
 654                 return False
 655         return True
 656
 657     def rebase_sequence(self, account, sequence):
 658         c, i = sequence
 659         dd = []
 660         for a in account.split('&'):
 661             s = a.strip()
 662             m = re.match("(m/\d+'/)(\d+)", s)
 663             root = m.group(1)
 664             num = int(m.group(2))
 665             dd.append( (root, [num,c,i] ) )
 666         return dd
 667
 668
 669     def get_keyID(self, account, sequence):
 670         if account == 0:
 671             a, b = sequence
 672             mpk = self.storage.get('master_public_key')
 673             return 'old(%s,%d,%d)'%(mpk,a,b)
 674
 675         rs = self.rebase_sequence(account, sequence)
 676         dd = []
 677         for root, public_sequence in rs:
 678             c, K, _ = self.master_public_keys[root]
 679             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 680             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 681         return '&'.join(dd)
 682
 683
 684
 685     def get_seed(self, password):
 686         s = pw_decode(self.seed, password)
 687         if self.seed_version == 4:
 688             seed = s
 689             self.accounts[0].check_seed(seed)
 690         else:
 691             seed = mnemonic_hash(s)
 692         return seed
 693
 694
 695     def get_mnemonic(self, password):
 696         import mnemonic
 697         s = pw_decode(self.seed, password)
 698         if self.seed_version == 4:
 699             return ' '.join(mnemonic.mn_encode(s))
 700         else:
 701             return s
 702
 703
 704
 705     def get_private_key(self, address, password):
 706         if self.is_watching_only():
 707             return []
 708
 709         # first check the provided password
 710         seed = self.get_seed(password)
 711
 712         out = []
 713         if address in self.imported_keys.keys():
 714             out.append( pw_decode( self.imported_keys[address], password ) )
 715         else:
 716             account, sequence = self.get_address_index(address)
 717             if account == 0:
 718                 pk = self.accounts[account].get_private_key(seed, sequence)
 719                 out.append(pk)
 720                 return out
 721
 722             # assert address == self.accounts[account].get_address(*sequence)
 723             rs = self.rebase_sequence( account, sequence)
 724             for root, public_sequence in rs:
 725
 726                 if root not in self.master_private_keys.keys(): continue
 727                 master_k = self.get_master_private_key(root, password)
 728                 master_c, _, _ = self.master_public_keys[root]
 729                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 730                 out.append(pk)
 731
 732         return out
 733
 734
 735     def add_keypairs_from_wallet(self, tx, keypairs, password):
 736         for txin in tx.inputs:
 737             address = txin['address']
 738             private_keys = self.get_private_key(address, password)
 739             for sec in private_keys:
 740                 pubkey = public_key_from_private_key(sec)
 741                 keypairs[ pubkey ] = sec
 742                 if address in self.imported_keys.keys():
 743                     txin['redeemPubkey'] = pubkey
 744
 745
 746     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 747         # first check the provided password
 748         seed = self.get_seed(password)
 749
 750         for txin in tx.inputs:
 751             keyid = txin.get('KeyID')
 752             if keyid:
 753
 754                 if self.seed_version == 4:
 755                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 756                     if not m: continue
 757                     mpk = m.group(1)
 758                     if mpk != self.storage.get('master_public_key'): continue
 759                     for_change = int(m.group(2))
 760                     num = int(m.group(3))
 761                     account = self.accounts[0]
 762                     addr = account.get_address(for_change, num)
 763                     txin['address'] = addr # fixme: side effect
 764                     pk = account.get_private_key(seed, (for_change, num))
 765                     pubkey = public_key_from_private_key(pk)
 766                     keypairs[pubkey] = pk
 767                     continue
 768
 769
 770                 roots = []
 771                 for s in keyid.split('&'):
 772                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 773                     if not m: continue
 774                     c = m.group(1)
 775                     K = m.group(2)
 776                     sequence = m.group(3)
 777                     root = self.find_root_by_master_key(c,K)
 778                     if not root: continue
 779                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 780                     root = root + '%d'%sequence[0]
 781                     sequence = sequence[1:]
 782                     roots.append((root,sequence))
 783
 784                 account_id = " & ".join( map(lambda x:x[0], roots) )
 785                 account = self.accounts.get(account_id)
 786                 if not account: continue
 787                 addr = account.get_address(*sequence)
 788                 txin['address'] = addr # fixme: side effect
 789                 pk = self.get_private_key(addr, password)
 790                 for sec in pk:
 791                     pubkey = public_key_from_private_key(sec)
 792                     keypairs[pubkey] = sec
 793
 794
 795
 796     def signrawtransaction(self, tx, input_info, private_keys, password):
 797
 798         # check that the password is correct
 799         seed = self.get_seed(password)
 800
 801         # add input info
 802         tx.add_input_info(input_info)
 803
 804         # add redeem script for coins that are in the wallet
 805         # FIXME: add redeemPubkey too!
 806         unspent_coins = self.get_unspent_coins()
 807         for txin in tx.inputs:
 808             for item in unspent_coins:
 809                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 810                     print_error( "tx input is in unspent coins" )
 811                     txin['scriptPubKey'] = item['scriptPubKey']
 812                     account, sequence = self.get_address_index(item['address'])
 813                     if account != -1:
 814                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 815                         print_error("added redeemScript", txin['redeemScript'])
 816                     break
 817
 818
 819         # build a list of public/private keys
 820         keypairs = {}
 821
 822         # add private keys from parameter
 823         for sec in private_keys:
 824             pubkey = public_key_from_private_key(sec)
 825             keypairs[ pubkey ] = sec
 826
 827         # add private_keys from KeyID
 828         self.add_keypairs_from_KeyID(tx, keypairs, password)
 829
 830         # add private keys from wallet
 831         self.add_keypairs_from_wallet(tx, keypairs, password)
 832         self.sign_transaction(tx, keypairs)
 833
 834
 835     def sign_message(self, address, message, password):
 836         keys = self.get_private_key(address, password)
 837         assert len(keys) == 1
 838         sec = keys[0]
 839         key = regenerate_key(sec)
 840         compressed = is_compressed(sec)
 841         return key.sign_message(message, compressed, address)
 842
 843
 844     def change_gap_limit(self, value):
 845         if value >= self.gap_limit:
 846             self.gap_limit = value
 847             self.storage.put('gap_limit', self.gap_limit, True)
 848             #self.interface.poke('synchronizer')
 849             return True
 850
 851         elif value >= self.min_acceptable_gap():
 852             for key, account in self.accounts.items():
 853                 addresses = account[0]
 854                 k = self.num_unused_trailing_addresses(addresses)
 855                 n = len(addresses) - k + value
 856                 addresses = addresses[0:n]
 857                 self.accounts[key][0] = addresses
 858
 859             self.gap_limit = value
 860             self.storage.put('gap_limit', self.gap_limit, True)
 861             self.save_accounts()
 862             return True
 863         else:
 864             return False
 865
 866     def num_unused_trailing_addresses(self, addresses):
 867         k = 0
 868         for a in addresses[::-1]:
 869             if self.history.get(a):break
 870             k = k + 1
 871         return k
 872
 873     def min_acceptable_gap(self):
 874         # fixme: this assumes wallet is synchronized
 875         n = 0
 876         nmax = 0
 877
 878         for account in self.accounts.values():
 879             addresses = account.get_addresses(0)
 880             k = self.num_unused_trailing_addresses(addresses)
 881             for a in addresses[0:-k]:
 882                 if self.history.get(a):
 883                     n = 0
 884                 else:
 885                     n += 1
 886                     if n > nmax: nmax = n
 887         return nmax + 1
 888
 889
 890     def address_is_old(self, address):
 891         age = -1
 892         h = self.history.get(address, [])
 893         if h == ['*']:
 894             return True
 895         for tx_hash, tx_height in h:
 896             if tx_height == 0:
 897                 tx_age = 0
 898             else:
 899                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 900             if tx_age > age:
 901                 age = tx_age
 902         return age > 2
 903
 904
 905     def synchronize_sequence(self, account, for_change):
 906         limit = self.gap_limit_for_change if for_change else self.gap_limit
 907         new_addresses = []
 908         while True:
 909             addresses = account.get_addresses(for_change)
 910             if len(addresses) < limit:
 911                 address = account.create_new_address(for_change)
 912                 self.history[address] = []
 913                 new_addresses.append( address )
 914                 continue
 915
 916             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 917                 break
 918             else:
 919                 address = account.create_new_address(for_change)
 920                 self.history[address] = []
 921                 new_addresses.append( address )
 922
 923         return new_addresses
 924
 925
 926
 927     def create_pending_accounts(self):
 928         for account_type in ['1','2of2','2of3']:
 929             if not self.has_master_public_keys(account_type):
 930                 continue
 931             k, a = self.new_account_address(account_type)
 932             if self.address_is_old(a):
 933                 print_error( "creating account", a )
 934                 self.create_account(account_type)
 935                 self.next_addresses.pop(k)
 936
 937
 938     def synchronize_account(self, account):
 939         new = []
 940         new += self.synchronize_sequence(account, 0)
 941         new += self.synchronize_sequence(account, 1)
 942         return new
 943
 944
 945     def synchronize(self):
 946         if self.master_public_keys:
 947             self.create_pending_accounts()
 948         new = []
 949         for account in self.accounts.values():
 950             new += self.synchronize_account(account)
 951         if new:
 952             self.save_accounts()
 953             self.storage.put('addr_history', self.history, True)
 954         return new
 955
 956
 957     def is_found(self):
 958         return self.history.values() != [[]] * len(self.history)
 959
 960
 961     def add_contact(self, address, label=None):
 962         self.addressbook.append(address)
 963         self.storage.put('contacts', self.addressbook, True)
 964         if label:
 965             self.set_label(address, label)
 966
 967
 968     def delete_contact(self, addr):
 969         if addr in self.addressbook:
 970             self.addressbook.remove(addr)
 971             self.storage.put('addressbook', self.addressbook, True)
 972
 973
 974     def fill_addressbook(self):
 975         for tx_hash, tx in self.transactions.items():
 976             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 977             if is_send:
 978                 for addr, v in tx.outputs:
 979                     if not self.is_mine(addr) and addr not in self.addressbook:
 980                         self.addressbook.append(addr)
 981         # redo labels
 982         # self.update_tx_labels()
 983
 984     def get_num_tx(self, address):
 985         n = 0
 986         for tx in self.transactions.values():
 987             if address in map(lambda x:x[0], tx.outputs): n += 1
 988         return n
 989
 990
 991     def get_address_flags(self, addr):
 992         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 993         flags += "F" if addr in self.frozen_addresses else "-"
 994         return flags
 995
 996
 997     def get_tx_value(self, tx, account=None):
 998         domain = self.get_account_addresses(account)
 999         return tx.get_value(domain, self.prevout_values)
1000
1001
1002     def update_tx_outputs(self, tx_hash):
1003         tx = self.transactions.get(tx_hash)
1004
1005         for i, (addr, value) in enumerate(tx.outputs):
1006             key = tx_hash+ ':%d'%i
1007             self.prevout_values[key] = value
1008
1009         for item in tx.inputs:
1010             if self.is_mine(item.get('address')):
1011                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1012                 self.spent_outputs.append(key)
1013
1014
1015     def get_addr_balance(self, address):
1016         assert self.is_mine(address)
1017         h = self.history.get(address,[])
1018         if h == ['*']: return 0,0
1019         c = u = 0
1020         received_coins = []   # list of coins received at address
1021
1022         for tx_hash, tx_height in h:
1023             tx = self.transactions.get(tx_hash)
1024             if not tx: continue
1025
1026             for i, (addr, value) in enumerate(tx.outputs):
1027                 if addr == address:
1028                     key = tx_hash + ':%d'%i
1029                     received_coins.append(key)
1030
1031         for tx_hash, tx_height in h:
1032             tx = self.transactions.get(tx_hash)
1033             if not tx: continue
1034             v = 0
1035
1036             for item in tx.inputs:
1037                 addr = item.get('address')
1038                 if addr == address:
1039                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1040                     value = self.prevout_values.get( key )
1041                     if key in received_coins:
1042                         v -= value
1043
1044             for i, (addr, value) in enumerate(tx.outputs):
1045                 key = tx_hash + ':%d'%i
1046                 if addr == address:
1047                     v += value
1048
1049             if tx_height:
1050                 c += v
1051             else:
1052                 u += v
1053         return c, u
1054
1055
1056     def get_account_name(self, k):
1057         if k == 0:
1058             if self.seed_version == 4:
1059                 name = 'Main account'
1060             else:
1061                 name = 'Old account'
1062         else:
1063             default = "Unnamed account"
1064             m = re.match("m/0'/(\d+)", k)
1065             if m:
1066                 num = m.group(1)
1067                 if num == '0':
1068                     default = "Main account"
1069                 else:
1070                     default = "Account %s"%num
1071
1072             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1073             if m:
1074                 num = m.group(1)
1075                 default = "2of2 account %s"%num
1076             name = self.labels.get(k, default)
1077
1078         return name
1079
1080     def get_account_names(self):
1081         accounts = {}
1082         for k, account in self.accounts.items():
1083             accounts[k] = self.get_account_name(k)
1084         if self.imported_keys:
1085             accounts[-1] = 'Imported keys'
1086         return accounts
1087
1088     def get_account_addresses(self, a, include_change=True):
1089         if a is None:
1090             o = self.addresses(True)
1091         elif a == -1:
1092             o = self.imported_keys.keys()
1093         else:
1094             ac = self.accounts[a]
1095             o = ac.get_addresses(0)
1096             if include_change: o += ac.get_addresses(1)
1097         return o
1098
1099     def get_imported_balance(self):
1100         return self.get_balance(self.imported_keys.keys())
1101
1102     def get_account_balance(self, account):
1103         return self.get_balance(self.get_account_addresses(account))
1104
1105     def get_frozen_balance(self):
1106         return self.get_balance(self.frozen_addresses)
1107
1108     def get_balance(self, domain=None):
1109         if domain is None: domain = self.addresses(True)
1110         cc = uu = 0
1111         for addr in domain:
1112             c, u = self.get_addr_balance(addr)
1113             cc += c
1114             uu += u
1115         return cc, uu
1116
1117
1118     def get_unspent_coins(self, domain=None):
1119         coins = []
1120         if domain is None: domain = self.addresses(True)
1121         for addr in domain:
1122             h = self.history.get(addr, [])
1123             if h == ['*']: continue
1124             for tx_hash, tx_height in h:
1125                 tx = self.transactions.get(tx_hash)
1126                 if tx is None: raise Exception("Wallet not synchronized")
1127                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1128                 for o in tx.d.get('outputs'):
1129                     output = o.copy()
1130                     if output.get('address') != addr: continue
1131                     key = tx_hash + ":%d" % output.get('prevout_n')
1132                     if key in self.spent_outputs: continue
1133                     output['prevout_hash'] = tx_hash
1134                     output['height'] = tx_height
1135                     output['coinbase'] = is_coinbase
1136                     coins.append((tx_height, output))
1137
1138         # sort by age
1139         if coins:
1140             coins = sorted(coins)
1141             if coins[-1][0] != 0:
1142                 while coins[0][0] == 0:
1143                     coins = coins[1:] + [ coins[0] ]
1144         return [x[1] for x in coins]
1145
1146
1147     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1148         """ todo: minimize tx size """
1149         total = 0
1150         fee = self.fee if fixed_fee is None else fixed_fee
1151         if domain is None:
1152             domain = self.addresses(True)
1153
1154         for i in self.frozen_addresses:
1155             if i in domain: domain.remove(i)
1156
1157         coins = self.get_unspent_coins(domain)
1158         inputs = []
1159
1160         for item in coins:
1161             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1162                 continue
1163             addr = item.get('address')
1164             v = item.get('value')
1165             total += v
1166             inputs.append(item)
1167             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1168             if total >= amount + fee: break
1169         else:
1170             inputs = []
1171
1172         return inputs, total, fee
1173
1174
1175     def set_fee(self, fee):
1176         if self.fee != fee:
1177             self.fee = fee
1178             self.storage.put('fee_per_kb', self.fee, True)
1179
1180     def estimated_fee(self, inputs):
1181         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1182         fee = self.fee * int(round(estimated_size/1024.))
1183         if fee == 0: fee = self.fee
1184         return fee
1185
1186
1187     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1188         "add change to a transaction"
1189         change_amount = total - ( amount + fee )
1190         if change_amount > DUST_THRESHOLD:
1191             if not change_addr:
1192
1193                 # send change to one of the accounts involved in the tx
1194                 address = inputs[0].get('address')
1195                 account, _ = self.get_address_index(address)
1196
1197                 if not self.use_change or account == -1:
1198                     change_addr = inputs[-1]['address']
1199                 else:
1200                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1201
1202             # Insert the change output at a random position in the outputs
1203             posn = random.randint(0, len(outputs))
1204             outputs[posn:posn] = [( change_addr,  change_amount)]
1205         return outputs
1206
1207
1208     def get_history(self, address):
1209         with self.lock:
1210             return self.history.get(address)
1211
1212
1213     def get_status(self, h):
1214         if not h: return None
1215         if h == ['*']: return '*'
1216         status = ''
1217         for tx_hash, height in h:
1218             status += tx_hash + ':%d:' % height
1219         return hashlib.sha256( status ).digest().encode('hex')
1220
1221
1222     def receive_tx_callback(self, tx_hash, tx, tx_height):
1223
1224         with self.transaction_lock:
1225             self.add_extra_addresses(tx)
1226             if not self.check_new_tx(tx_hash, tx):
1227                 # may happen due to pruning
1228                 print_error("received transaction that is no longer referenced in history", tx_hash)
1229                 return
1230             self.transactions[tx_hash] = tx
1231             self.network.interface.pending_transactions_for_notifications.append(tx)
1232             self.save_transactions()
1233             if self.verifier and tx_height>0:
1234                 self.verifier.add(tx_hash, tx_height)
1235             self.update_tx_outputs(tx_hash)
1236
1237
1238     def save_transactions(self):
1239         tx = {}
1240         for k,v in self.transactions.items():
1241             tx[k] = str(v)
1242         self.storage.put('transactions', tx, True)
1243
1244     def receive_history_callback(self, addr, hist):
1245
1246         if not self.check_new_history(addr, hist):
1247             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1248
1249         with self.lock:
1250             self.history[addr] = hist
1251             self.storage.put('addr_history', self.history, True)
1252
1253         if hist != ['*']:
1254             for tx_hash, tx_height in hist:
1255                 if tx_height>0:
1256                     # add it in case it was previously unconfirmed
1257                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1258
1259
1260     def get_tx_history(self, account=None):
1261         if not self.verifier:
1262             return []
1263
1264         with self.transaction_lock:
1265             history = self.transactions.items()
1266             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1267             result = []
1268
1269             balance = 0
1270             for tx_hash, tx in history:
1271                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1272                 if v is not None: balance += v
1273
1274             c, u = self.get_account_balance(account)
1275
1276             if balance != c+u:
1277                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1278
1279             balance = c + u - balance
1280             for tx_hash, tx in history:
1281                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1282                 if not is_relevant:
1283                     continue
1284                 if value is not None:
1285                     balance += value
1286
1287                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1288                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1289
1290         return result
1291
1292
1293     def get_label(self, tx_hash):
1294         label = self.labels.get(tx_hash)
1295         is_default = (label == '') or (label is None)
1296         if is_default: label = self.get_default_label(tx_hash)
1297         return label, is_default
1298
1299
1300     def get_default_label(self, tx_hash):
1301         tx = self.transactions.get(tx_hash)
1302         default_label = ''
1303         if tx:
1304             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1305             if is_mine:
1306                 for o in tx.outputs:
1307                     o_addr, _ = o
1308                     if not self.is_mine(o_addr):
1309                         try:
1310                             default_label = self.labels[o_addr]
1311                         except KeyError:
1312                             default_label = o_addr
1313                         break
1314                 else:
1315                     default_label = '(internal)'
1316             else:
1317                 for o in tx.outputs:
1318                     o_addr, _ = o
1319                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1320                         break
1321                 else:
1322                     for o in tx.outputs:
1323                         o_addr, _ = o
1324                         if self.is_mine(o_addr):
1325                             break
1326                     else:
1327                         o_addr = None
1328
1329                 if o_addr:
1330                     dest_label = self.labels.get(o_addr)
1331                     try:
1332                         default_label = self.labels[o_addr]
1333                     except KeyError:
1334                         default_label = o_addr
1335
1336         return default_label
1337
1338
1339     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1340         for address, x in outputs:
1341             assert is_valid(address)
1342         amount = sum( map(lambda x:x[1], outputs) )
1343         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1344         if not inputs:
1345             raise ValueError("Not enough funds")
1346         self.add_input_info(inputs)
1347         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1348         return Transaction.from_io(inputs, outputs)
1349
1350
1351     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1352         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1353         keypairs = {}
1354         self.add_keypairs_from_wallet(tx, keypairs, password)
1355         if keypairs:
1356             self.sign_transaction(tx, keypairs)
1357         return tx
1358
1359
1360     def add_input_info(self, inputs):
1361         for txin in inputs:
1362             address = txin['address']
1363             if address in self.imported_keys.keys():
1364                 continue
1365             account, sequence = self.get_address_index(address)
1366             txin['KeyID'] = self.get_keyID(account, sequence)
1367             redeemScript = self.accounts[account].redeem_script(sequence)
1368             if redeemScript:
1369                 txin['redeemScript'] = redeemScript
1370             else:
1371                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1372
1373
1374     def sign_transaction(self, tx, keypairs):
1375         tx.sign(keypairs)
1376         run_hook('sign_transaction', tx)
1377
1378
1379     def sendtx(self, tx):
1380         # synchronous
1381         h = self.send_tx(tx)
1382         self.tx_event.wait()
1383         return self.receive_tx(h)
1384
1385     def send_tx(self, tx):
1386         # asynchronous
1387         self.tx_event.clear()
1388         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1389         return tx.hash()
1390
1391     def on_broadcast(self, i, r):
1392         self.tx_result = r.get('result')
1393         self.tx_event.set()
1394
1395     def receive_tx(self,tx_hash):
1396         out = self.tx_result
1397         if out != tx_hash:
1398             return False, "error: " + out
1399         return True, out
1400
1401
1402
1403     def update_password(self, old_password, new_password):
1404         if new_password == '': new_password = None
1405         decoded = self.get_seed(old_password)
1406         self.seed = pw_encode( decoded, new_password)
1407         self.storage.put('seed', self.seed, True)
1408         self.use_encryption = (new_password != None)
1409         self.storage.put('use_encryption', self.use_encryption,True)
1410         for k in self.imported_keys.keys():
1411             a = self.imported_keys[k]
1412             b = pw_decode(a, old_password)
1413             c = pw_encode(b, new_password)
1414             self.imported_keys[k] = c
1415         self.storage.put('imported_keys', self.imported_keys, True)
1416
1417         for k, v in self.master_private_keys.items():
1418             b = pw_decode(v, old_password)
1419             c = pw_encode(b, new_password)
1420             self.master_private_keys[k] = c
1421         self.storage.put('master_private_keys', self.master_private_keys, True)
1422
1423
1424     def freeze(self,addr):
1425         if self.is_mine(addr) and addr not in self.frozen_addresses:
1426             self.frozen_addresses.append(addr)
1427             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1428             return True
1429         else:
1430             return False
1431
1432     def unfreeze(self,addr):
1433         if self.is_mine(addr) and addr in self.frozen_addresses:
1434             self.frozen_addresses.remove(addr)
1435             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1436             return True
1437         else:
1438             return False
1439
1440
1441     def set_verifier(self, verifier):
1442         self.verifier = verifier
1443
1444         # review transactions that are in the history
1445         for addr, hist in self.history.items():
1446             if hist == ['*']: continue
1447             for tx_hash, tx_height in hist:
1448                 if tx_height>0:
1449                     # add it in case it was previously unconfirmed
1450                     self.verifier.add(tx_hash, tx_height)
1451
1452
1453         # if we are on a pruning server, remove unverified transactions
1454         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1455         for tx_hash in self.transactions.keys():
1456             if tx_hash not in vr:
1457                 self.transactions.pop(tx_hash)
1458
1459
1460
1461     def check_new_history(self, addr, hist):
1462
1463         # check that all tx in hist are relevant
1464         if hist != ['*']:
1465             for tx_hash, height in hist:
1466                 tx = self.transactions.get(tx_hash)
1467                 if not tx: continue
1468                 if not tx.has_address(addr):
1469                     return False
1470
1471         # check that we are not "orphaning" a transaction
1472         old_hist = self.history.get(addr,[])
1473         if old_hist == ['*']: return True
1474
1475         for tx_hash, height in old_hist:
1476             if tx_hash in map(lambda x:x[0], hist): continue
1477             found = False
1478             for _addr, _hist in self.history.items():
1479                 if _addr == addr: continue
1480                 if _hist == ['*']: continue
1481                 _tx_hist = map(lambda x:x[0], _hist)
1482                 if tx_hash in _tx_hist:
1483                     found = True
1484                     break
1485
1486             if not found:
1487                 tx = self.transactions.get(tx_hash)
1488                 # tx might not be there
1489                 if not tx: continue
1490
1491                 # already verified?
1492                 if self.verifier.get_height(tx_hash):
1493                     continue
1494                 # unconfirmed tx
1495                 print_error("new history is orphaning transaction:", tx_hash)
1496                 # check that all outputs are not mine, request histories
1497                 ext_requests = []
1498                 for _addr, _v in tx.outputs:
1499                     # assert not self.is_mine(_addr)
1500                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1501
1502                 ext_h = self.network.synchronous_get(ext_requests)
1503                 print_error("sync:", ext_requests, ext_h)
1504                 height = None
1505                 for h in ext_h:
1506                     if h == ['*']: continue
1507                     for item in h:
1508                         if item.get('tx_hash') == tx_hash:
1509                             height = item.get('height')
1510                 if height:
1511                     print_error("found height for", tx_hash, height)
1512                     self.verifier.add(tx_hash, height)
1513                 else:
1514                     print_error("removing orphaned tx from history", tx_hash)
1515                     self.transactions.pop(tx_hash)
1516
1517         return True
1518
1519
1520
1521     def check_new_tx(self, tx_hash, tx):
1522         # 1 check that tx is referenced in addr_history.
1523         addresses = []
1524         for addr, hist in self.history.items():
1525             if hist == ['*']:continue
1526             for txh, height in hist:
1527                 if txh == tx_hash:
1528                     addresses.append(addr)
1529
1530         if not addresses:
1531             return False
1532
1533         # 2 check that referencing addresses are in the tx
1534         for addr in addresses:
1535             if not tx.has_address(addr):
1536                 return False
1537
1538         return True
1539
1540
1541     def start_threads(self, network):
1542         from verifier import TxVerifier
1543         self.network = network
1544         if self.network:
1545             self.verifier = TxVerifier(self.network, self.storage)
1546             self.verifier.start()
1547             self.set_verifier(self.verifier)
1548             self.synchronizer = WalletSynchronizer(self, network)
1549             self.synchronizer.start()
1550         else:
1551             self.verifier = None
1552             self.synchronizer =None
1553
1554     def stop_threads(self):
1555         if self.network:
1556             self.verifier.stop()
1557             self.synchronizer.stop()
1558
1559
1560     def restore(self, callback):
1561         from i18n import _
1562         def wait_for_wallet():
1563             self.set_up_to_date(False)
1564             while not self.is_up_to_date():
1565                 msg = "%s\n%s %d\n%s %.1f"%(
1566                     _("Please wait..."),
1567                     _("Addresses generated:"),
1568                     len(self.addresses(True)),
1569                     _("Kilobytes received:"),
1570                     self.network.interface.bytes_received/1024.)
1571
1572                 apply(callback, (msg,))
1573                 time.sleep(0.1)
1574
1575         def wait_for_network():
1576             while not self.network.is_connected():
1577                 msg = "%s \n" % (_("Connecting..."))
1578                 apply(callback, (msg,))
1579                 time.sleep(0.1)
1580
1581         # wait until we are connected, because the user might have selected another server
1582         if self.network:
1583             wait_for_network()
1584             wait_for_wallet()
1585         else:
1586             self.synchronize()
1587
1588         self.fill_addressbook()
1589
1590
1591
1592
1593 class WalletSynchronizer(threading.Thread):
1594
1595
1596     def __init__(self, wallet, network):
1597         threading.Thread.__init__(self)
1598         self.daemon = True
1599         self.wallet = wallet
1600         self.network = network
1601         self.was_updated = True
1602         self.running = False
1603         self.lock = threading.Lock()
1604         self.queue = Queue.Queue()
1605
1606     def stop(self):
1607         with self.lock: self.running = False
1608
1609     def is_running(self):
1610         with self.lock: return self.running
1611
1612
1613     def subscribe_to_addresses(self, addresses):
1614         messages = []
1615         for addr in addresses:
1616             messages.append(('blockchain.address.subscribe', [addr]))
1617         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1618
1619
1620     def run(self):
1621         with self.lock:
1622             self.running = True
1623
1624         while self.is_running():
1625
1626             if not self.network.is_connected():
1627                 print_error("synchronizer: waiting for interface")
1628                 self.network.wait_until_connected()
1629
1630             self.run_interface(self.network.interface)
1631
1632
1633     def run_interface(self, interface):
1634
1635         print_error("synchronizer: connected to", interface.server)
1636
1637         requested_tx = []
1638         missing_tx = []
1639         requested_histories = {}
1640
1641         # request any missing transactions
1642         for history in self.wallet.history.values():
1643             if history == ['*']: continue
1644             for tx_hash, tx_height in history:
1645                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1646                     missing_tx.append( (tx_hash, tx_height) )
1647
1648         if missing_tx:
1649             print_error("missing tx", missing_tx)
1650
1651         # subscriptions
1652         self.subscribe_to_addresses(self.wallet.addresses(True))
1653
1654         while self.is_running():
1655             # 1. create new addresses
1656             new_addresses = self.wallet.synchronize()
1657
1658             # request missing addresses
1659             if new_addresses:
1660                 self.subscribe_to_addresses(new_addresses)
1661
1662             # request missing transactions
1663             for tx_hash, tx_height in missing_tx:
1664                 if (tx_hash, tx_height) not in requested_tx:
1665                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1666                     requested_tx.append( (tx_hash, tx_height) )
1667             missing_tx = []
1668
1669             # detect if situation has changed
1670             if interface.is_up_to_date() and self.queue.empty():
1671                 if not self.wallet.is_up_to_date():
1672                     self.wallet.set_up_to_date(True)
1673                     self.was_updated = True
1674             else:
1675                 if self.wallet.is_up_to_date():
1676                     self.wallet.set_up_to_date(False)
1677                     self.was_updated = True
1678
1679             if self.was_updated:
1680                 self.wallet.network.trigger_callback('updated')
1681                 self.was_updated = False
1682
1683             # 2. get a response
1684             try:
1685                 r = self.queue.get(block=True, timeout=1)
1686             except Queue.Empty:
1687                 continue
1688
1689             if interface != self.network.interface:
1690                 break
1691
1692             if not r:
1693                 continue
1694
1695             # 3. handle response
1696             method = r['method']
1697             params = r['params']
1698             result = r.get('result')
1699             error = r.get('error')
1700             if error:
1701                 print "error", r
1702                 continue
1703
1704             if method == 'blockchain.address.subscribe':
1705                 addr = params[0]
1706                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1707                     if requested_histories.get(addr) is None:
1708                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1709                         requested_histories[addr] = result
1710
1711             elif method == 'blockchain.address.get_history':
1712                 addr = params[0]
1713                 print_error("receiving history", addr, result)
1714                 if result == ['*']:
1715                     assert requested_histories.pop(addr) == '*'
1716                     self.wallet.receive_history_callback(addr, result)
1717                 else:
1718                     hist = []
1719                     # check that txids are unique
1720                     txids = []
1721                     for item in result:
1722                         tx_hash = item['tx_hash']
1723                         if tx_hash not in txids:
1724                             txids.append(tx_hash)
1725                             hist.append( (tx_hash, item['height']) )
1726
1727                     if len(hist) != len(result):
1728                         raise Exception("error: server sent history with non-unique txid", result)
1729
1730                     # check that the status corresponds to what was announced
1731                     rs = requested_histories.pop(addr)
1732                     if self.wallet.get_status(hist) != rs:
1733                         raise Exception("error: status mismatch: %s"%addr)
1734
1735                     # store received history
1736                     self.wallet.receive_history_callback(addr, hist)
1737
1738                     # request transactions that we don't have
1739                     for tx_hash, tx_height in hist:
1740                         if self.wallet.transactions.get(tx_hash) is None:
1741                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1742                                 missing_tx.append( (tx_hash, tx_height) )
1743
1744             elif method == 'blockchain.transaction.get':
1745                 tx_hash = params[0]
1746                 tx_height = params[1]
1747                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1748                 tx = Transaction(result)
1749                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1750                 self.was_updated = True
1751                 requested_tx.remove( (tx_hash, tx_height) )
1752                 print_error("received tx:", tx_hash, len(tx.raw))
1753
1754             else:
1755                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1756
1757             if self.was_updated and not requested_tx:
1758                 self.wallet.network.trigger_callback('updated')
1759                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1760
1761                 self.was_updated = False