lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         v = self.data.get(key)
 129         if v is None:
 130             v = default
 131         return v
 132
 133     def put(self, key, value, save = True):
 134
 135         with self.lock:
 136             if value is not None:
 137                 self.data[key] = value
 138             else:
 139                 self.data.pop(key)
 140             if save:
 141                 self.write()
 142
 143     def write(self):
 144         s = repr(self.data)
 145         f = open(self.path,"w")
 146         f.write( s )
 147         f.close()
 148         if 'ANDROID_DATA' not in os.environ:
 149             import stat
 150             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 151
 152
 153
 154
 155
 156 class Wallet:
 157
 158     def __init__(self, storage):
 159
 160         self.storage = storage
 161         self.electrum_version = ELECTRUM_VERSION
 162         self.gap_limit_for_change = 3 # constant
 163
 164         # saved fields
 165         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 166
 167         self.gap_limit             = storage.get('gap_limit', 5)
 168         self.use_change            = storage.get('use_change',True)
 169         self.use_encryption        = storage.get('use_encryption', False)
 170         self.seed                  = storage.get('seed', '')               # encrypted
 171         self.labels                = storage.get('labels', {})
 172         self.frozen_addresses      = storage.get('frozen_addresses',[])
 173         self.addressbook           = storage.get('contacts', [])
 174
 175         self.imported_keys         = storage.get('imported_keys',{})
 176         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 177
 178         self.fee                   = int(storage.get('fee_per_kb',20000))
 179
 180         self.master_public_keys = storage.get('master_public_keys',{})
 181         self.master_private_keys = storage.get('master_private_keys', {})
 182
 183         self.next_addresses = storage.get('next_addresses',{})
 184
 185
 186         # This attribute is set when wallet.start_threads is called.
 187         self.synchronizer = None
 188
 189         self.load_accounts()
 190
 191         self.transactions = {}
 192         tx_list = self.storage.get('transactions',{})
 193         for k,v in tx_list.items():
 194             try:
 195                 tx = Transaction(v)
 196             except Exception:
 197                 print_msg("Warning: Cannot deserialize transactions. skipping")
 198                 continue
 199
 200             self.add_extra_addresses(tx)
 201             self.transactions[k] = tx
 202
 203         for h,tx in self.transactions.items():
 204             if not self.check_new_tx(h, tx):
 205                 print_error("removing unreferenced tx", h)
 206                 self.transactions.pop(h)
 207
 208
 209         # not saved
 210         self.prevout_values = {}     # my own transaction outputs
 211         self.spent_outputs = []
 212
 213         # spv
 214         self.verifier = None
 215
 216         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 217         # interface.is_up_to_date() returns true when all requests have been answered and processed
 218         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 219
 220         self.up_to_date = False
 221         self.lock = threading.Lock()
 222         self.transaction_lock = threading.Lock()
 223         self.tx_event = threading.Event()
 224
 225         for tx_hash, tx in self.transactions.items():
 226             self.update_tx_outputs(tx_hash)
 227
 228
 229     def add_extra_addresses(self, tx):
 230         h = tx.hash()
 231         # find the address corresponding to pay-to-pubkey inputs
 232         tx.add_extra_addresses(self.transactions)
 233         for o in tx.d.get('outputs'):
 234             if o.get('is_pubkey'):
 235                 for tx2 in self.transactions.values():
 236                     tx2.add_extra_addresses({h:tx})
 237
 238
 239
 240
 241     def set_up_to_date(self,b):
 242         with self.lock: self.up_to_date = b
 243
 244
 245     def is_up_to_date(self):
 246         with self.lock: return self.up_to_date
 247
 248
 249     def update(self):
 250         self.up_to_date = False
 251         while not self.is_up_to_date():
 252             time.sleep(0.1)
 253
 254
 255     def import_key(self, sec, password):
 256         # check password
 257         seed = self.get_seed(password)
 258         try:
 259             address = address_from_private_key(sec)
 260         except Exception:
 261             raise Exception('Invalid private key')
 262
 263         if self.is_mine(address):
 264             raise Exception('Address already in wallet')
 265
 266         # store the originally requested keypair into the imported keys table
 267         self.imported_keys[address] = pw_encode(sec, password )
 268         self.storage.put('imported_keys', self.imported_keys, True)
 269         if self.synchronizer:
 270             self.synchronizer.subscribe_to_addresses([address])
 271         return address
 272
 273     def delete_imported_key(self, addr):
 274         if addr in self.imported_keys:
 275             self.imported_keys.pop(addr)
 276             self.storage.put('imported_keys', self.imported_keys, True)
 277
 278
 279     def make_seed(self):
 280         import mnemonic, ecdsa
 281         entropy = ecdsa.util.randrange( pow(2,160) )
 282         nonce = 0
 283         while True:
 284             ss = "%040x"%(entropy+nonce)
 285             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 286             # we keep only 13 words, that's approximately 139 bits of entropy
 287             words = mnemonic.mn_encode(s)[0:13]
 288             seed = ' '.join(words)
 289             if is_seed(seed):
 290                 break  # this will remove 8 bits of entropy
 291             nonce += 1
 292
 293         return seed
 294
 295
 296     def init_seed(self, seed):
 297         import mnemonic, unicodedata
 298
 299         if self.seed:
 300             raise Exception("a seed exists")
 301
 302         if not seed:
 303             self.seed = self.make_seed()
 304             self.seed_version = SEED_VERSION
 305             return
 306
 307         self.seed_version = SEED_VERSION
 308         self.seed = unicodedata.normalize('NFC', unicode(seed.strip()))
 309         return
 310
 311         # find out what kind of wallet we are
 312         try:
 313             seed.strip().decode('hex')
 314             self.seed_version = 4
 315             self.seed = str(seed)
 316             return
 317         except Exception:
 318             pass
 319
 320         words = seed.split()
 321         try:
 322             mnemonic.mn_decode(words)
 323             uses_electrum_words = True
 324         except Exception:
 325             uses_electrum_words = False
 326
 327         if uses_electrum_words and len(words) != 13:
 328             self.seed_version = 4
 329             self.seed = mnemonic.mn_decode(words)
 330         else:
 331             #assert is_seed(seed)
 332             self.seed_version = SEED_VERSION
 333             self.seed = seed
 334
 335
 336     def save_seed(self, password):
 337         if password:
 338             self.seed = pw_encode( self.seed, password)
 339             self.use_encryption = True
 340         self.storage.put('seed', self.seed, True)
 341         self.storage.put('seed_version', self.seed_version, True)
 342         self.storage.put('use_encryption', self.use_encryption,True)
 343         self.create_accounts(password)
 344
 345
 346     def create_watching_only_wallet(self, params):
 347         K0, c0 = params
 348         if not K0:
 349             return
 350
 351         if not c0:
 352             self.seed_version = 4
 353             self.storage.put('seed_version', self.seed_version, True)
 354             self.create_old_account(K0)
 355             return
 356
 357         cK0 = ""
 358         self.master_public_keys = {
 359             "m/0'/": (c0, K0, cK0),
 360             }
 361         self.storage.put('master_public_keys', self.master_public_keys, True)
 362         self.storage.put('seed_version', self.seed_version, True)
 363         self.create_account('1','Main account')
 364
 365
 366     def create_accounts(self, password):
 367         seed = pw_decode(self.seed, password)
 368
 369         if self.seed_version == 4:
 370             mpk = OldAccount.mpk_from_seed(seed)
 371             self.create_old_account(mpk)
 372         else:
 373             # create default account
 374             self.create_master_keys('1', password)
 375             self.create_account('1','Main account')
 376
 377
 378     def create_master_keys(self, account_type, password):
 379         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(password))
 380         if account_type == '1':
 381             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 382             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 383             self.master_private_keys["m/0'/"] = pw_encode(k0, password)
 384         elif account_type == '2of2':
 385             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 386             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 387             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 388             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 389             self.master_private_keys["m/1'/"] = pw_encode(k1, password)
 390             self.master_private_keys["m/2'/"] = pw_encode(k2, password)
 391         elif account_type == '2of3':
 392             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 393             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 394             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 395             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 396             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 397             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 398             self.master_private_keys["m/3'/"] = pw_encode(k3, password)
 399             self.master_private_keys["m/4'/"] = pw_encode(k4, password)
 400             self.master_private_keys["m/5'/"] = pw_encode(k5, password)
 401
 402         self.storage.put('master_public_keys', self.master_public_keys, True)
 403         self.storage.put('master_private_keys', self.master_private_keys, True)
 404
 405     def has_master_public_keys(self, account_type):
 406         if account_type == '1':
 407             return "m/0'/" in self.master_public_keys
 408         elif account_type == '2of2':
 409             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 410         elif account_type == '2of3':
 411             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 412
 413     def find_root_by_master_key(self, c, K):
 414         for key, v in self.master_public_keys.items():
 415             if key == "m/":continue
 416             cc, KK, _ = v
 417             if (c == cc) and (K == KK):
 418                 return key
 419
 420     def deseed_root(self, seed, password):
 421         # for safety, we ask the user to enter their seed
 422         assert seed == self.get_seed(password)
 423         self.seed = ''
 424         self.storage.put('seed', '', True)
 425
 426
 427     def deseed_branch(self, k):
 428         # check that parent has no seed
 429         # assert self.seed == ''
 430         self.master_private_keys.pop(k)
 431         self.storage.put('master_private_keys', self.master_private_keys, True)
 432
 433
 434     def is_watching_only(self):
 435         return (self.seed == '') and (self.master_private_keys == {})
 436
 437
 438
 439     def account_id(self, account_type, i):
 440         if account_type == '1':
 441             return "m/0'/%d"%i
 442         elif account_type == '2of2':
 443             return "m/1'/%d & m/2'/%d"%(i,i)
 444         elif account_type == '2of3':
 445             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 446         else:
 447             raise Exception('unknown account type')
 448
 449
 450     def num_accounts(self, account_type):
 451         keys = self.accounts.keys()
 452         i = 0
 453         while True:
 454             account_id = self.account_id(account_type, i)
 455             if account_id not in keys: break
 456             i += 1
 457         return i
 458
 459
 460     def new_account_address(self, account_type = '1'):
 461         i = self.num_accounts(account_type)
 462         k = self.account_id(account_type,i)
 463
 464         addr = self.next_addresses.get(k)
 465         if not addr:
 466             account_id, account = self.next_account(account_type)
 467             addr = account.first_address()
 468             self.next_addresses[k] = addr
 469             self.storage.put('next_addresses',self.next_addresses)
 470
 471         return k, addr
 472
 473
 474     def next_account(self, account_type = '1'):
 475
 476         i = self.num_accounts(account_type)
 477         account_id = self.account_id(account_type,i)
 478
 479         if account_type is '1':
 480             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 481             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 482             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 483
 484         elif account_type == '2of2':
 485             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 486             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 487             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 488             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 489             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 490
 491         elif account_type == '2of3':
 492             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 493             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 494             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 495             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 496             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 497             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 498             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 499
 500         return account_id, account
 501
 502
 503     def set_label(self, name, text = None):
 504         changed = False
 505         old_text = self.labels.get(name)
 506         if text:
 507             if old_text != text:
 508                 self.labels[name] = text
 509                 changed = True
 510         else:
 511             if old_text:
 512                 self.labels.pop(name)
 513                 changed = True
 514
 515         if changed:
 516             self.storage.put('labels', self.labels, True)
 517
 518         run_hook('set_label', name, text, changed)
 519         return changed
 520
 521
 522
 523     def create_account(self, account_type = '1', name = None):
 524         k, account = self.next_account(account_type)
 525         if k in self.pending_accounts:
 526             self.pending_accounts.pop(k)
 527             self.storage.put('pending_accounts', self.pending_accounts)
 528
 529         self.accounts[k] = account
 530         self.save_accounts()
 531         if name:
 532             self.set_label(k, name)
 533
 534
 535     def create_old_account(self, mpk):
 536         self.storage.put('master_public_key', mpk, True)
 537         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 538         self.save_accounts()
 539
 540
 541     def save_accounts(self):
 542         d = {}
 543         for k, v in self.accounts.items():
 544             d[k] = v.dump()
 545         self.storage.put('accounts', d, True)
 546
 547
 548
 549     def load_accounts(self):
 550         d = self.storage.get('accounts', {})
 551         self.accounts = {}
 552         for k, v in d.items():
 553             if k == 0:
 554                 v['mpk'] = self.storage.get('master_public_key')
 555                 self.accounts[k] = OldAccount(v)
 556             elif '&' in k:
 557                 self.accounts[k] = BIP32_Account_2of2(v)
 558             else:
 559                 self.accounts[k] = BIP32_Account(v)
 560
 561         self.pending_accounts = self.storage.get('pending_accounts',{})
 562
 563
 564     def delete_pending_account(self, k):
 565         self.pending_accounts.pop(k)
 566         self.storage.put('pending_accounts', self.pending_accounts)
 567
 568     def account_is_pending(self, k):
 569         return k in self.pending_accounts
 570
 571     def create_pending_account(self, acct_type, name):
 572         k, addr = self.new_account_address(acct_type)
 573         self.set_label(k, name)
 574         self.pending_accounts[k] = addr
 575         self.storage.put('pending_accounts', self.pending_accounts)
 576
 577     def get_pending_accounts(self):
 578         return self.pending_accounts.items()
 579
 580
 581     def addresses(self, include_change = True, _next=True):
 582         o = self.get_account_addresses(-1, include_change)
 583         for a in self.accounts.keys():
 584             o += self.get_account_addresses(a, include_change)
 585
 586         if _next:
 587             for addr in self.next_addresses.values():
 588                 if addr not in o:
 589                     o += [addr]
 590         return o
 591
 592
 593     def is_mine(self, address):
 594         return address in self.addresses(True)
 595
 596
 597     def is_change(self, address):
 598         if not self.is_mine(address): return False
 599         if address in self.imported_keys.keys(): return False
 600         acct, s = self.get_address_index(address)
 601         if s is None: return False
 602         return s[0] == 1
 603
 604     def get_master_public_key(self):
 605         if self.seed_version == 4:
 606             return self.storage.get("master_public_key")
 607         else:
 608             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 609             return repr((c, K))
 610
 611     def get_master_private_key(self, account, password):
 612         k = self.master_private_keys.get(account)
 613         if not k: return
 614         master_k = pw_decode( k, password)
 615         master_c, master_K, master_Kc = self.master_public_keys[account]
 616         try:
 617             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 618             assert K.encode('hex') == master_K
 619         except Exception:
 620             raise Exception("Invalid password")
 621         return master_k
 622
 623
 624     def get_address_index(self, address):
 625         if address in self.imported_keys.keys():
 626             return -1, None
 627
 628         for account in self.accounts.keys():
 629             for for_change in [0,1]:
 630                 addresses = self.accounts[account].get_addresses(for_change)
 631                 for addr in addresses:
 632                     if address == addr:
 633                         return account, (for_change, addresses.index(addr))
 634
 635         for k,v in self.next_addresses.items():
 636             if v == address:
 637                 return k, (0,0)
 638
 639         raise Exception("Address not found", address)
 640
 641
 642     def get_roots(self, account):
 643         roots = []
 644         for a in account.split('&'):
 645             s = a.strip()
 646             m = re.match("(m/\d+'/)(\d+)", s)
 647             roots.append( m.group(1) )
 648         return roots
 649
 650     def is_seeded(self, account):
 651         if type(account) is int:
 652             return self.seed is not None
 653
 654         for root in self.get_roots(account):
 655             if root not in self.master_private_keys.keys():
 656                 return False
 657         return True
 658
 659     def rebase_sequence(self, account, sequence):
 660         c, i = sequence
 661         dd = []
 662         for a in account.split('&'):
 663             s = a.strip()
 664             m = re.match("(m/\d+'/)(\d+)", s)
 665             root = m.group(1)
 666             num = int(m.group(2))
 667             dd.append( (root, [num,c,i] ) )
 668         return dd
 669
 670
 671     def get_keyID(self, account, sequence):
 672         if account == 0:
 673             a, b = sequence
 674             mpk = self.storage.get('master_public_key')
 675             return 'old(%s,%d,%d)'%(mpk,a,b)
 676
 677         rs = self.rebase_sequence(account, sequence)
 678         dd = []
 679         for root, public_sequence in rs:
 680             c, K, _ = self.master_public_keys[root]
 681             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 682             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 683         return '&'.join(dd)
 684
 685
 686     def get_seed(self, password):
 687         s = pw_decode(self.seed, password)
 688         if self.seed_version == 4:
 689             seed = s
 690             self.accounts[0].check_seed(seed)
 691         else:
 692             seed = mnemonic_to_seed(s,'').encode('hex')
 693         return seed
 694
 695
 696     def get_mnemonic(self, password):
 697         import mnemonic
 698         s = pw_decode(self.seed, password)
 699         if self.seed_version == 4:
 700             return ' '.join(mnemonic.mn_encode(s))
 701         else:
 702             return s
 703
 704
 705     def get_private_key(self, address, password):
 706         if self.is_watching_only():
 707             return []
 708
 709         # first check the provided password
 710         seed = self.get_seed(password)
 711
 712         out = []
 713         if address in self.imported_keys.keys():
 714             out.append( pw_decode( self.imported_keys[address], password ) )
 715         else:
 716             account, sequence = self.get_address_index(address)
 717             if account == 0:
 718                 pk = self.accounts[account].get_private_key(seed, sequence)
 719                 out.append(pk)
 720                 return out
 721
 722             # assert address == self.accounts[account].get_address(*sequence)
 723             rs = self.rebase_sequence( account, sequence)
 724             for root, public_sequence in rs:
 725
 726                 if root not in self.master_private_keys.keys(): continue
 727                 master_k = self.get_master_private_key(root, password)
 728                 master_c, _, _ = self.master_public_keys[root]
 729                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 730                 out.append(pk)
 731
 732         return out
 733
 734
 735     def add_keypairs_from_wallet(self, tx, keypairs, password):
 736         for txin in tx.inputs:
 737             address = txin['address']
 738             if not self.is_mine(address):
 739                 continue
 740             private_keys = self.get_private_key(address, password)
 741             for sec in private_keys:
 742                 pubkey = public_key_from_private_key(sec)
 743                 keypairs[ pubkey ] = sec
 744                 if address in self.imported_keys.keys():
 745                     txin['redeemPubkey'] = pubkey
 746
 747
 748     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 749         # first check the provided password
 750         seed = self.get_seed(password)
 751
 752         for txin in tx.inputs:
 753             keyid = txin.get('KeyID')
 754             if keyid:
 755
 756                 if self.seed_version == 4:
 757                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 758                     if not m: continue
 759                     mpk = m.group(1)
 760                     if mpk != self.storage.get('master_public_key'): continue
 761                     for_change = int(m.group(2))
 762                     num = int(m.group(3))
 763                     account = self.accounts[0]
 764                     addr = account.get_address(for_change, num)
 765                     txin['address'] = addr # fixme: side effect
 766                     pk = account.get_private_key(seed, (for_change, num))
 767                     pubkey = public_key_from_private_key(pk)
 768                     keypairs[pubkey] = pk
 769                     continue
 770
 771
 772                 roots = []
 773                 for s in keyid.split('&'):
 774                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 775                     if not m: continue
 776                     c = m.group(1)
 777                     K = m.group(2)
 778                     sequence = m.group(3)
 779                     root = self.find_root_by_master_key(c,K)
 780                     if not root: continue
 781                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 782                     root = root + '%d'%sequence[0]
 783                     sequence = sequence[1:]
 784                     roots.append((root,sequence))
 785
 786                 account_id = " & ".join( map(lambda x:x[0], roots) )
 787                 account = self.accounts.get(account_id)
 788                 if not account: continue
 789                 addr = account.get_address(*sequence)
 790                 txin['address'] = addr # fixme: side effect
 791                 pk = self.get_private_key(addr, password)
 792                 for sec in pk:
 793                     pubkey = public_key_from_private_key(sec)
 794                     keypairs[pubkey] = sec
 795
 796
 797
 798     def signrawtransaction(self, tx, input_info, private_keys, password):
 799
 800         # check that the password is correct
 801         seed = self.get_seed(password)
 802
 803         # add input info
 804         tx.add_input_info(input_info)
 805
 806         # add redeem script for coins that are in the wallet
 807         # FIXME: add redeemPubkey too!
 808         unspent_coins = self.get_unspent_coins()
 809         for txin in tx.inputs:
 810             for item in unspent_coins:
 811                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 812                     print_error( "tx input is in unspent coins" )
 813                     txin['scriptPubKey'] = item['scriptPubKey']
 814                     account, sequence = self.get_address_index(item['address'])
 815                     if account != -1:
 816                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 817                         print_error("added redeemScript", txin['redeemScript'])
 818                     break
 819
 820
 821         # build a list of public/private keys
 822         keypairs = {}
 823
 824         # add private keys from parameter
 825         for sec in private_keys:
 826             pubkey = public_key_from_private_key(sec)
 827             keypairs[ pubkey ] = sec
 828
 829         # add private_keys from KeyID
 830         self.add_keypairs_from_KeyID(tx, keypairs, password)
 831
 832         # add private keys from wallet
 833         self.add_keypairs_from_wallet(tx, keypairs, password)
 834         self.sign_transaction(tx, keypairs)
 835
 836
 837     def sign_message(self, address, message, password):
 838         keys = self.get_private_key(address, password)
 839         assert len(keys) == 1
 840         sec = keys[0]
 841         key = regenerate_key(sec)
 842         compressed = is_compressed(sec)
 843         return key.sign_message(message, compressed, address)
 844
 845
 846     def change_gap_limit(self, value):
 847         if value >= self.gap_limit:
 848             self.gap_limit = value
 849             self.storage.put('gap_limit', self.gap_limit, True)
 850             #self.interface.poke('synchronizer')
 851             return True
 852
 853         elif value >= self.min_acceptable_gap():
 854             for key, account in self.accounts.items():
 855                 addresses = account[0]
 856                 k = self.num_unused_trailing_addresses(addresses)
 857                 n = len(addresses) - k + value
 858                 addresses = addresses[0:n]
 859                 self.accounts[key][0] = addresses
 860
 861             self.gap_limit = value
 862             self.storage.put('gap_limit', self.gap_limit, True)
 863             self.save_accounts()
 864             return True
 865         else:
 866             return False
 867
 868     def num_unused_trailing_addresses(self, addresses):
 869         k = 0
 870         for a in addresses[::-1]:
 871             if self.history.get(a):break
 872             k = k + 1
 873         return k
 874
 875     def min_acceptable_gap(self):
 876         # fixme: this assumes wallet is synchronized
 877         n = 0
 878         nmax = 0
 879
 880         for account in self.accounts.values():
 881             addresses = account.get_addresses(0)
 882             k = self.num_unused_trailing_addresses(addresses)
 883             for a in addresses[0:-k]:
 884                 if self.history.get(a):
 885                     n = 0
 886                 else:
 887                     n += 1
 888                     if n > nmax: nmax = n
 889         return nmax + 1
 890
 891
 892     def address_is_old(self, address):
 893         age = -1
 894         h = self.history.get(address, [])
 895         if h == ['*']:
 896             return True
 897         for tx_hash, tx_height in h:
 898             if tx_height == 0:
 899                 tx_age = 0
 900             else:
 901                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 902             if tx_age > age:
 903                 age = tx_age
 904         return age > 2
 905
 906
 907     def synchronize_sequence(self, account, for_change):
 908         limit = self.gap_limit_for_change if for_change else self.gap_limit
 909         new_addresses = []
 910         while True:
 911             addresses = account.get_addresses(for_change)
 912             if len(addresses) < limit:
 913                 address = account.create_new_address(for_change)
 914                 self.history[address] = []
 915                 new_addresses.append( address )
 916                 continue
 917
 918             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 919                 break
 920             else:
 921                 address = account.create_new_address(for_change)
 922                 self.history[address] = []
 923                 new_addresses.append( address )
 924
 925         return new_addresses
 926
 927
 928
 929     def create_pending_accounts(self):
 930         for account_type in ['1','2of2','2of3']:
 931             if not self.has_master_public_keys(account_type):
 932                 continue
 933             k, a = self.new_account_address(account_type)
 934             if self.address_is_old(a):
 935                 print_error( "creating account", a )
 936                 self.create_account(account_type)
 937                 self.next_addresses.pop(k)
 938
 939
 940     def synchronize_account(self, account):
 941         new = []
 942         new += self.synchronize_sequence(account, 0)
 943         new += self.synchronize_sequence(account, 1)
 944         return new
 945
 946
 947     def synchronize(self):
 948         if self.master_public_keys:
 949             self.create_pending_accounts()
 950         new = []
 951         for account in self.accounts.values():
 952             new += self.synchronize_account(account)
 953         if new:
 954             self.save_accounts()
 955             self.storage.put('addr_history', self.history, True)
 956         return new
 957
 958
 959     def is_found(self):
 960         return self.history.values() != [[]] * len(self.history)
 961
 962
 963     def add_contact(self, address, label=None):
 964         self.addressbook.append(address)
 965         self.storage.put('contacts', self.addressbook, True)
 966         if label:
 967             self.set_label(address, label)
 968
 969
 970     def delete_contact(self, addr):
 971         if addr in self.addressbook:
 972             self.addressbook.remove(addr)
 973             self.storage.put('addressbook', self.addressbook, True)
 974
 975
 976     def fill_addressbook(self):
 977         for tx_hash, tx in self.transactions.items():
 978             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 979             if is_send:
 980                 for addr, v in tx.outputs:
 981                     if not self.is_mine(addr) and addr not in self.addressbook:
 982                         self.addressbook.append(addr)
 983         # redo labels
 984         # self.update_tx_labels()
 985
 986     def get_num_tx(self, address):
 987         n = 0
 988         for tx in self.transactions.values():
 989             if address in map(lambda x:x[0], tx.outputs): n += 1
 990         return n
 991
 992
 993     def get_address_flags(self, addr):
 994         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 995         flags += "F" if addr in self.frozen_addresses else "-"
 996         return flags
 997
 998
 999     def get_tx_value(self, tx, account=None):
1000         domain = self.get_account_addresses(account)
1001         return tx.get_value(domain, self.prevout_values)
1002
1003
1004     def update_tx_outputs(self, tx_hash):
1005         tx = self.transactions.get(tx_hash)
1006
1007         for i, (addr, value) in enumerate(tx.outputs):
1008             key = tx_hash+ ':%d'%i
1009             self.prevout_values[key] = value
1010
1011         for item in tx.inputs:
1012             if self.is_mine(item.get('address')):
1013                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1014                 self.spent_outputs.append(key)
1015
1016
1017     def get_addr_balance(self, address):
1018         assert self.is_mine(address)
1019         h = self.history.get(address,[])
1020         if h == ['*']: return 0,0
1021         c = u = 0
1022         received_coins = []   # list of coins received at address
1023
1024         for tx_hash, tx_height in h:
1025             tx = self.transactions.get(tx_hash)
1026             if not tx: continue
1027
1028             for i, (addr, value) in enumerate(tx.outputs):
1029                 if addr == address:
1030                     key = tx_hash + ':%d'%i
1031                     received_coins.append(key)
1032
1033         for tx_hash, tx_height in h:
1034             tx = self.transactions.get(tx_hash)
1035             if not tx: continue
1036             v = 0
1037
1038             for item in tx.inputs:
1039                 addr = item.get('address')
1040                 if addr == address:
1041                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1042                     value = self.prevout_values.get( key )
1043                     if key in received_coins:
1044                         v -= value
1045
1046             for i, (addr, value) in enumerate(tx.outputs):
1047                 key = tx_hash + ':%d'%i
1048                 if addr == address:
1049                     v += value
1050
1051             if tx_height:
1052                 c += v
1053             else:
1054                 u += v
1055         return c, u
1056
1057
1058     def get_account_name(self, k):
1059         if k == 0:
1060             if self.seed_version == 4:
1061                 name = 'Main account'
1062             else:
1063                 name = 'Old account'
1064         else:
1065             default = "Unnamed account"
1066             m = re.match("m/0'/(\d+)", k)
1067             if m:
1068                 num = m.group(1)
1069                 if num == '0':
1070                     default = "Main account"
1071                 else:
1072                     default = "Account %s"%num
1073
1074             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1075             if m:
1076                 num = m.group(1)
1077                 default = "2of2 account %s"%num
1078             name = self.labels.get(k, default)
1079
1080         return name
1081
1082     def get_account_names(self):
1083         accounts = {}
1084         for k, account in self.accounts.items():
1085             accounts[k] = self.get_account_name(k)
1086         if self.imported_keys:
1087             accounts[-1] = 'Imported keys'
1088         return accounts
1089
1090     def get_account_addresses(self, a, include_change=True):
1091         if a is None:
1092             o = self.addresses(True)
1093         elif a == -1:
1094             o = self.imported_keys.keys()
1095         else:
1096             ac = self.accounts[a]
1097             o = ac.get_addresses(0)
1098             if include_change: o += ac.get_addresses(1)
1099         return o
1100
1101     def get_imported_balance(self):
1102         return self.get_balance(self.imported_keys.keys())
1103
1104     def get_account_balance(self, account):
1105         return self.get_balance(self.get_account_addresses(account))
1106
1107     def get_frozen_balance(self):
1108         return self.get_balance(self.frozen_addresses)
1109
1110     def get_balance(self, domain=None):
1111         if domain is None: domain = self.addresses(True)
1112         cc = uu = 0
1113         for addr in domain:
1114             c, u = self.get_addr_balance(addr)
1115             cc += c
1116             uu += u
1117         return cc, uu
1118
1119
1120     def get_unspent_coins(self, domain=None):
1121         coins = []
1122         if domain is None: domain = self.addresses(True)
1123         for addr in domain:
1124             h = self.history.get(addr, [])
1125             if h == ['*']: continue
1126             for tx_hash, tx_height in h:
1127                 tx = self.transactions.get(tx_hash)
1128                 if tx is None: raise Exception("Wallet not synchronized")
1129                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1130                 for o in tx.d.get('outputs'):
1131                     output = o.copy()
1132                     if output.get('address') != addr: continue
1133                     key = tx_hash + ":%d" % output.get('prevout_n')
1134                     if key in self.spent_outputs: continue
1135                     output['prevout_hash'] = tx_hash
1136                     output['height'] = tx_height
1137                     output['coinbase'] = is_coinbase
1138                     coins.append((tx_height, output))
1139
1140         # sort by age
1141         if coins:
1142             coins = sorted(coins)
1143             if coins[-1][0] != 0:
1144                 while coins[0][0] == 0:
1145                     coins = coins[1:] + [ coins[0] ]
1146         return [x[1] for x in coins]
1147
1148
1149     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1150         """ todo: minimize tx size """
1151         total = 0
1152         fee = self.fee if fixed_fee is None else fixed_fee
1153         if domain is None:
1154             domain = self.addresses(True)
1155
1156         for i in self.frozen_addresses:
1157             if i in domain: domain.remove(i)
1158
1159         coins = self.get_unspent_coins(domain)
1160         inputs = []
1161
1162         for item in coins:
1163             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1164                 continue
1165             addr = item.get('address')
1166             v = item.get('value')
1167             total += v
1168             inputs.append(item)
1169             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1170             if total >= amount + fee: break
1171         else:
1172             inputs = []
1173
1174         return inputs, total, fee
1175
1176
1177     def set_fee(self, fee):
1178         if self.fee != fee:
1179             self.fee = fee
1180             self.storage.put('fee_per_kb', self.fee, True)
1181
1182     def estimated_fee(self, inputs):
1183         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1184         fee = self.fee * int(round(estimated_size/1024.))
1185         if fee == 0: fee = self.fee
1186         return fee
1187
1188
1189     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1190         "add change to a transaction"
1191         change_amount = total - ( amount + fee )
1192         if change_amount > DUST_THRESHOLD:
1193             if not change_addr:
1194
1195                 # send change to one of the accounts involved in the tx
1196                 address = inputs[0].get('address')
1197                 account, _ = self.get_address_index(address)
1198
1199                 if not self.use_change or account == -1:
1200                     change_addr = inputs[-1]['address']
1201                 else:
1202                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1203
1204             # Insert the change output at a random position in the outputs
1205             posn = random.randint(0, len(outputs))
1206             outputs[posn:posn] = [( change_addr,  change_amount)]
1207         return outputs
1208
1209
1210     def get_history(self, address):
1211         with self.lock:
1212             return self.history.get(address)
1213
1214
1215     def get_status(self, h):
1216         if not h: return None
1217         if h == ['*']: return '*'
1218         status = ''
1219         for tx_hash, height in h:
1220             status += tx_hash + ':%d:' % height
1221         return hashlib.sha256( status ).digest().encode('hex')
1222
1223
1224     def receive_tx_callback(self, tx_hash, tx, tx_height):
1225
1226         with self.transaction_lock:
1227             self.add_extra_addresses(tx)
1228             if not self.check_new_tx(tx_hash, tx):
1229                 # may happen due to pruning
1230                 print_error("received transaction that is no longer referenced in history", tx_hash)
1231                 return
1232             self.transactions[tx_hash] = tx
1233             self.network.interface.pending_transactions_for_notifications.append(tx)
1234             self.save_transactions()
1235             if self.verifier and tx_height>0:
1236                 self.verifier.add(tx_hash, tx_height)
1237             self.update_tx_outputs(tx_hash)
1238
1239
1240     def save_transactions(self):
1241         tx = {}
1242         for k,v in self.transactions.items():
1243             tx[k] = str(v)
1244         self.storage.put('transactions', tx, True)
1245
1246     def receive_history_callback(self, addr, hist):
1247
1248         if not self.check_new_history(addr, hist):
1249             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1250
1251         with self.lock:
1252             self.history[addr] = hist
1253             self.storage.put('addr_history', self.history, True)
1254
1255         if hist != ['*']:
1256             for tx_hash, tx_height in hist:
1257                 if tx_height>0:
1258                     # add it in case it was previously unconfirmed
1259                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1260
1261
1262     def get_tx_history(self, account=None):
1263         if not self.verifier:
1264             return []
1265
1266         with self.transaction_lock:
1267             history = self.transactions.items()
1268             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1269             result = []
1270
1271             balance = 0
1272             for tx_hash, tx in history:
1273                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1274                 if v is not None: balance += v
1275
1276             c, u = self.get_account_balance(account)
1277
1278             if balance != c+u:
1279                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1280
1281             balance = c + u - balance
1282             for tx_hash, tx in history:
1283                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1284                 if not is_relevant:
1285                     continue
1286                 if value is not None:
1287                     balance += value
1288
1289                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1290                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1291
1292         return result
1293
1294
1295     def get_label(self, tx_hash):
1296         label = self.labels.get(tx_hash)
1297         is_default = (label == '') or (label is None)
1298         if is_default: label = self.get_default_label(tx_hash)
1299         return label, is_default
1300
1301
1302     def get_default_label(self, tx_hash):
1303         tx = self.transactions.get(tx_hash)
1304         default_label = ''
1305         if tx:
1306             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1307             if is_mine:
1308                 for o in tx.outputs:
1309                     o_addr, _ = o
1310                     if not self.is_mine(o_addr):
1311                         try:
1312                             default_label = self.labels[o_addr]
1313                         except KeyError:
1314                             default_label = o_addr
1315                         break
1316                 else:
1317                     default_label = '(internal)'
1318             else:
1319                 for o in tx.outputs:
1320                     o_addr, _ = o
1321                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1322                         break
1323                 else:
1324                     for o in tx.outputs:
1325                         o_addr, _ = o
1326                         if self.is_mine(o_addr):
1327                             break
1328                     else:
1329                         o_addr = None
1330
1331                 if o_addr:
1332                     dest_label = self.labels.get(o_addr)
1333                     try:
1334                         default_label = self.labels[o_addr]
1335                     except KeyError:
1336                         default_label = o_addr
1337
1338         return default_label
1339
1340
1341     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1342         for address, x in outputs:
1343             assert is_valid(address)
1344         amount = sum( map(lambda x:x[1], outputs) )
1345         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1346         if not inputs:
1347             raise ValueError("Not enough funds")
1348         self.add_input_info(inputs)
1349         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1350         return Transaction.from_io(inputs, outputs)
1351
1352
1353     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1354         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1355         keypairs = {}
1356         self.add_keypairs_from_wallet(tx, keypairs, password)
1357         if keypairs:
1358             self.sign_transaction(tx, keypairs)
1359         return tx
1360
1361
1362     def add_input_info(self, inputs):
1363         for txin in inputs:
1364             address = txin['address']
1365             if address in self.imported_keys.keys():
1366                 continue
1367             account, sequence = self.get_address_index(address)
1368             txin['KeyID'] = self.get_keyID(account, sequence)
1369             redeemScript = self.accounts[account].redeem_script(sequence)
1370             if redeemScript:
1371                 txin['redeemScript'] = redeemScript
1372             else:
1373                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1374
1375
1376     def sign_transaction(self, tx, keypairs):
1377         tx.sign(keypairs)
1378         run_hook('sign_transaction', tx)
1379
1380
1381     def sendtx(self, tx):
1382         # synchronous
1383         h = self.send_tx(tx)
1384         self.tx_event.wait()
1385         return self.receive_tx(h, tx)
1386
1387     def send_tx(self, tx):
1388         # asynchronous
1389         self.tx_event.clear()
1390         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1391         return tx.hash()
1392
1393     def on_broadcast(self, i, r):
1394         self.tx_result = r.get('result')
1395         self.tx_event.set()
1396
1397     def receive_tx(self, tx_hash, tx):
1398         out = self.tx_result
1399         if out != tx_hash:
1400             return False, "error: " + out
1401         run_hook('receive_tx', tx, self)
1402         return True, out
1403
1404
1405
1406     def update_password(self, old_password, new_password):
1407         if new_password == '': new_password = None
1408         decoded = self.get_seed(old_password)
1409         self.seed = pw_encode( decoded, new_password)
1410         self.storage.put('seed', self.seed, True)
1411         self.use_encryption = (new_password != None)
1412         self.storage.put('use_encryption', self.use_encryption,True)
1413         for k in self.imported_keys.keys():
1414             a = self.imported_keys[k]
1415             b = pw_decode(a, old_password)
1416             c = pw_encode(b, new_password)
1417             self.imported_keys[k] = c
1418         self.storage.put('imported_keys', self.imported_keys, True)
1419
1420         for k, v in self.master_private_keys.items():
1421             b = pw_decode(v, old_password)
1422             c = pw_encode(b, new_password)
1423             self.master_private_keys[k] = c
1424         self.storage.put('master_private_keys', self.master_private_keys, True)
1425
1426
1427     def freeze(self,addr):
1428         if self.is_mine(addr) and addr not in self.frozen_addresses:
1429             self.frozen_addresses.append(addr)
1430             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1431             return True
1432         else:
1433             return False
1434
1435     def unfreeze(self,addr):
1436         if self.is_mine(addr) and addr in self.frozen_addresses:
1437             self.frozen_addresses.remove(addr)
1438             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1439             return True
1440         else:
1441             return False
1442
1443
1444     def set_verifier(self, verifier):
1445         self.verifier = verifier
1446
1447         # review transactions that are in the history
1448         for addr, hist in self.history.items():
1449             if hist == ['*']: continue
1450             for tx_hash, tx_height in hist:
1451                 if tx_height>0:
1452                     # add it in case it was previously unconfirmed
1453                     self.verifier.add(tx_hash, tx_height)
1454
1455
1456         # if we are on a pruning server, remove unverified transactions
1457         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1458         for tx_hash in self.transactions.keys():
1459             if tx_hash not in vr:
1460                 self.transactions.pop(tx_hash)
1461
1462
1463
1464     def check_new_history(self, addr, hist):
1465
1466         # check that all tx in hist are relevant
1467         if hist != ['*']:
1468             for tx_hash, height in hist:
1469                 tx = self.transactions.get(tx_hash)
1470                 if not tx: continue
1471                 if not tx.has_address(addr):
1472                     return False
1473
1474         # check that we are not "orphaning" a transaction
1475         old_hist = self.history.get(addr,[])
1476         if old_hist == ['*']: return True
1477
1478         for tx_hash, height in old_hist:
1479             if tx_hash in map(lambda x:x[0], hist): continue
1480             found = False
1481             for _addr, _hist in self.history.items():
1482                 if _addr == addr: continue
1483                 if _hist == ['*']: continue
1484                 _tx_hist = map(lambda x:x[0], _hist)
1485                 if tx_hash in _tx_hist:
1486                     found = True
1487                     break
1488
1489             if not found:
1490                 tx = self.transactions.get(tx_hash)
1491                 # tx might not be there
1492                 if not tx: continue
1493
1494                 # already verified?
1495                 if self.verifier.get_height(tx_hash):
1496                     continue
1497                 # unconfirmed tx
1498                 print_error("new history is orphaning transaction:", tx_hash)
1499                 # check that all outputs are not mine, request histories
1500                 ext_requests = []
1501                 for _addr, _v in tx.outputs:
1502                     # assert not self.is_mine(_addr)
1503                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1504
1505                 ext_h = self.network.synchronous_get(ext_requests)
1506                 print_error("sync:", ext_requests, ext_h)
1507                 height = None
1508                 for h in ext_h:
1509                     if h == ['*']: continue
1510                     for item in h:
1511                         if item.get('tx_hash') == tx_hash:
1512                             height = item.get('height')
1513                 if height:
1514                     print_error("found height for", tx_hash, height)
1515                     self.verifier.add(tx_hash, height)
1516                 else:
1517                     print_error("removing orphaned tx from history", tx_hash)
1518                     self.transactions.pop(tx_hash)
1519
1520         return True
1521
1522
1523
1524     def check_new_tx(self, tx_hash, tx):
1525         # 1 check that tx is referenced in addr_history.
1526         addresses = []
1527         for addr, hist in self.history.items():
1528             if hist == ['*']:continue
1529             for txh, height in hist:
1530                 if txh == tx_hash:
1531                     addresses.append(addr)
1532
1533         if not addresses:
1534             return False
1535
1536         # 2 check that referencing addresses are in the tx
1537         for addr in addresses:
1538             if not tx.has_address(addr):
1539                 return False
1540
1541         return True
1542
1543
1544     def start_threads(self, network):
1545         from verifier import TxVerifier
1546         self.network = network
1547         if self.network is not None:
1548             self.verifier = TxVerifier(self.network, self.storage)
1549             self.verifier.start()
1550             self.set_verifier(self.verifier)
1551             self.synchronizer = WalletSynchronizer(self, network)
1552             self.synchronizer.start()
1553         else:
1554             self.verifier = None
1555             self.synchronizer =None
1556
1557     def stop_threads(self):
1558         if self.network:
1559             self.verifier.stop()
1560             self.synchronizer.stop()
1561
1562
1563     def restore(self, callback):
1564         from i18n import _
1565         def wait_for_wallet():
1566             self.set_up_to_date(False)
1567             while not self.is_up_to_date():
1568                 msg = "%s\n%s %d\n%s %.1f"%(
1569                     _("Please wait..."),
1570                     _("Addresses generated:"),
1571                     len(self.addresses(True)),
1572                     _("Kilobytes received:"),
1573                     self.network.interface.bytes_received/1024.)
1574
1575                 apply(callback, (msg,))
1576                 time.sleep(0.1)
1577
1578         def wait_for_network():
1579             while not self.network.is_connected():
1580                 msg = "%s \n" % (_("Connecting..."))
1581                 apply(callback, (msg,))
1582                 time.sleep(0.1)
1583
1584         # wait until we are connected, because the user might have selected another server
1585         if self.network:
1586             wait_for_network()
1587             wait_for_wallet()
1588         else:
1589             self.synchronize()
1590
1591         self.fill_addressbook()
1592
1593
1594
1595
1596 class WalletSynchronizer(threading.Thread):
1597
1598
1599     def __init__(self, wallet, network):
1600         threading.Thread.__init__(self)
1601         self.daemon = True
1602         self.wallet = wallet
1603         self.network = network
1604         self.was_updated = True
1605         self.running = False
1606         self.lock = threading.Lock()
1607         self.queue = Queue.Queue()
1608
1609     def stop(self):
1610         with self.lock: self.running = False
1611
1612     def is_running(self):
1613         with self.lock: return self.running
1614
1615
1616     def subscribe_to_addresses(self, addresses):
1617         messages = []
1618         for addr in addresses:
1619             messages.append(('blockchain.address.subscribe', [addr]))
1620         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1621
1622
1623     def run(self):
1624         with self.lock:
1625             self.running = True
1626
1627         while self.is_running():
1628
1629             if not self.network.is_connected():
1630                 self.network.wait_until_connected()
1631
1632             self.run_interface()
1633
1634
1635     def run_interface(self):
1636
1637         print_error("synchronizer: connected to", self.network.main_server())
1638
1639         requested_tx = []
1640         missing_tx = []
1641         requested_histories = {}
1642
1643         # request any missing transactions
1644         for history in self.wallet.history.values():
1645             if history == ['*']: continue
1646             for tx_hash, tx_height in history:
1647                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1648                     missing_tx.append( (tx_hash, tx_height) )
1649
1650         if missing_tx:
1651             print_error("missing tx", missing_tx)
1652
1653         # subscriptions
1654         self.subscribe_to_addresses(self.wallet.addresses(True))
1655
1656         while self.is_running():
1657             # 1. create new addresses
1658             new_addresses = self.wallet.synchronize()
1659
1660             # request missing addresses
1661             if new_addresses:
1662                 self.subscribe_to_addresses(new_addresses)
1663
1664             # request missing transactions
1665             for tx_hash, tx_height in missing_tx:
1666                 if (tx_hash, tx_height) not in requested_tx:
1667                     self.network.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1668                     requested_tx.append( (tx_hash, tx_height) )
1669             missing_tx = []
1670
1671             # detect if situation has changed
1672             if self.network.is_up_to_date() and self.queue.empty():
1673                 if not self.wallet.is_up_to_date():
1674                     self.wallet.set_up_to_date(True)
1675                     self.was_updated = True
1676             else:
1677                 if self.wallet.is_up_to_date():
1678                     self.wallet.set_up_to_date(False)
1679                     self.was_updated = True
1680
1681             if self.was_updated:
1682                 self.network.trigger_callback('updated')
1683                 self.was_updated = False
1684
1685             # 2. get a response
1686             try:
1687                 r = self.queue.get(block=True, timeout=1)
1688             except Queue.Empty:
1689                 continue
1690
1691             # see if it changed
1692             #if interface != self.network.interface:
1693             #    break
1694
1695             if not r:
1696                 continue
1697
1698             # 3. handle response
1699             method = r['method']
1700             params = r['params']
1701             result = r.get('result')
1702             error = r.get('error')
1703             if error:
1704                 print "error", r
1705                 continue
1706
1707             if method == 'blockchain.address.subscribe':
1708                 addr = params[0]
1709                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1710                     if requested_histories.get(addr) is None:
1711                         self.network.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1712                         requested_histories[addr] = result
1713
1714             elif method == 'blockchain.address.get_history':
1715                 addr = params[0]
1716                 print_error("receiving history", addr, result)
1717                 if result == ['*']:
1718                     assert requested_histories.pop(addr) == '*'
1719                     self.wallet.receive_history_callback(addr, result)
1720                 else:
1721                     hist = []
1722                     # check that txids are unique
1723                     txids = []
1724                     for item in result:
1725                         tx_hash = item['tx_hash']
1726                         if tx_hash not in txids:
1727                             txids.append(tx_hash)
1728                             hist.append( (tx_hash, item['height']) )
1729
1730                     if len(hist) != len(result):
1731                         raise Exception("error: server sent history with non-unique txid", result)
1732
1733                     # check that the status corresponds to what was announced
1734                     rs = requested_histories.pop(addr)
1735                     if self.wallet.get_status(hist) != rs:
1736                         raise Exception("error: status mismatch: %s"%addr)
1737
1738                     # store received history
1739                     self.wallet.receive_history_callback(addr, hist)
1740
1741                     # request transactions that we don't have
1742                     for tx_hash, tx_height in hist:
1743                         if self.wallet.transactions.get(tx_hash) is None:
1744                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1745                                 missing_tx.append( (tx_hash, tx_height) )
1746
1747             elif method == 'blockchain.transaction.get':
1748                 tx_hash = params[0]
1749                 tx_height = params[1]
1750                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1751                 tx = Transaction(result)
1752                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1753                 self.was_updated = True
1754                 requested_tx.remove( (tx_hash, tx_height) )
1755                 print_error("received tx:", tx_hash, len(tx.raw))
1756
1757             else:
1758                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1759
1760             if self.was_updated and not requested_tx:
1761                 self.network.trigger_callback('updated')
1762                 # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1763                 self.network.trigger_callback("new_transaction")
1764                 self.was_updated = False
1765
1766