lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         return self.data.get(key, default)
 129
 130     def put(self, key, value, save = True):
 131
 132         with self.lock:
 133             if value is not None:
 134                 self.data[key] = value
 135             else:
 136                 self.data.pop(key)
 137             if save:
 138                 self.write()
 139
 140     def write(self):
 141         s = repr(self.data)
 142         f = open(self.path,"w")
 143         f.write( s )
 144         f.close()
 145         if 'ANDROID_DATA' not in os.environ:
 146             import stat
 147             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 148
 149
 150 class Wallet:
 151
 152     def __init__(self, storage):
 153
 154         self.storage = storage
 155         self.electrum_version = ELECTRUM_VERSION
 156         self.gap_limit_for_change = 3 # constant
 157
 158         # saved fields
 159         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 160
 161         self.gap_limit             = storage.get('gap_limit', 5)
 162         self.use_change            = storage.get('use_change',True)
 163         self.use_encryption        = storage.get('use_encryption', False)
 164         self.seed                  = storage.get('seed', '')               # encrypted
 165         self.labels                = storage.get('labels', {})
 166         self.frozen_addresses      = storage.get('frozen_addresses',[])
 167         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 168         self.addressbook           = storage.get('contacts', [])
 169
 170         self.imported_keys         = storage.get('imported_keys',{})
 171         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 172
 173         self.fee                   = int(storage.get('fee_per_kb',20000))
 174
 175         self.master_public_keys = storage.get('master_public_keys',{})
 176         self.master_private_keys = storage.get('master_private_keys', {})
 177
 178         self.next_addresses = storage.get('next_addresses',{})
 179
 180         if self.seed_version not in [4, 6]:
 181             msg = "This wallet seed is not supported."
 182             if self.seed_version in [5]:
 183                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 184                 print msg
 185                 sys.exit(1)
 186
 187
 188         self.load_accounts()
 189
 190         self.transactions = {}
 191         tx_list = self.storage.get('transactions',{})
 192         for k,v in tx_list.items():
 193             try:
 194                 tx = Transaction(v)
 195             except Exception:
 196                 print_msg("Warning: Cannot deserialize transactions. skipping")
 197                 continue
 198
 199             self.add_extra_addresses(tx)
 200             self.transactions[k] = tx
 201
 202         for h,tx in self.transactions.items():
 203             if not self.check_new_tx(h, tx):
 204                 print_error("removing unreferenced tx", h)
 205                 self.transactions.pop(h)
 206
 207
 208         # not saved
 209         self.prevout_values = {}     # my own transaction outputs
 210         self.spent_outputs = []
 211
 212         # spv
 213         self.verifier = None
 214
 215         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 216         # interface.is_up_to_date() returns true when all requests have been answered and processed
 217         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 218
 219         self.up_to_date = False
 220         self.lock = threading.Lock()
 221         self.transaction_lock = threading.Lock()
 222         self.tx_event = threading.Event()
 223
 224         for tx_hash, tx in self.transactions.items():
 225             self.update_tx_outputs(tx_hash)
 226
 227
 228     def add_extra_addresses(self, tx):
 229         h = tx.hash()
 230         # find the address corresponding to pay-to-pubkey inputs
 231         tx.add_extra_addresses(self.transactions)
 232         for o in tx.d.get('outputs'):
 233             if o.get('is_pubkey'):
 234                 for tx2 in self.transactions.values():
 235                     tx2.add_extra_addresses({h:tx})
 236
 237
 238
 239
 240     def set_up_to_date(self,b):
 241         with self.lock: self.up_to_date = b
 242
 243
 244     def is_up_to_date(self):
 245         with self.lock: return self.up_to_date
 246
 247
 248     def update(self):
 249         self.up_to_date = False
 250         while not self.is_up_to_date():
 251             time.sleep(0.1)
 252
 253
 254     def import_key(self, sec, password):
 255         # check password
 256         seed = self.get_seed(password)
 257         try:
 258             address = address_from_private_key(sec)
 259         except Exception:
 260             raise Exception('Invalid private key')
 261
 262         if self.is_mine(address):
 263             raise Exception('Address already in wallet')
 264
 265         # store the originally requested keypair into the imported keys table
 266         self.imported_keys[address] = pw_encode(sec, password )
 267         self.storage.put('imported_keys', self.imported_keys, True)
 268         if self.synchronizer:
 269             self.synchronizer.subscribe_to_addresses([address])
 270         return address
 271
 272     def delete_imported_key(self, addr):
 273         if addr in self.imported_keys:
 274             self.imported_keys.pop(addr)
 275             self.storage.put('imported_keys', self.imported_keys, True)
 276
 277
 278     def make_seed(self):
 279         import mnemonic, ecdsa
 280         entropy = ecdsa.util.randrange( pow(2,160) )
 281         nonce = 0
 282         while True:
 283             ss = "%040x"%(entropy+nonce)
 284             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 285             # we keep only 13 words, that's approximately 139 bits of entropy
 286             words = mnemonic.mn_encode(s)[0:13]
 287             seed = ' '.join(words)
 288             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 289                 break  # this removes 12 bits of entropy
 290             nonce += 1
 291
 292         return seed
 293
 294
 295     def init_seed(self, seed):
 296         import mnemonic
 297
 298         if self.seed:
 299             raise Exception("a seed exists")
 300
 301         if not seed:
 302             self.seed = random_seed(128)
 303             self.seed_version = 4
 304             return
 305
 306         #if not seed:
 307         #    self.seed = self.make_seed()
 308         #    self.seed_version = SEED_VERSION
 309         #    return
 310
 311         # find out what kind of wallet we are
 312         try:
 313             seed.strip().decode('hex')
 314             self.seed_version = 4
 315             self.seed = str(seed)
 316             return
 317         except Exception:
 318             pass
 319
 320         words = seed.split()
 321         self.seed_version = 4
 322         self.seed = mnemonic.mn_decode(words)
 323
 324         #try:
 325         #    mnemonic.mn_decode(words)
 326         #    uses_electrum_words = True
 327         #except Exception:
 328         #    uses_electrum_words = False
 329         #
 330         #if uses_electrum_words and len(words) != 13:
 331         #    self.seed_version = 4
 332         #    self.seed = mnemonic.mn_decode(words)
 333         #else:
 334         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 335         #    self.seed_version = SEED_VERSION
 336         #    self.seed = seed
 337
 338
 339     def save_seed(self):
 340         self.storage.put('seed', self.seed, True)
 341         self.storage.put('seed_version', self.seed_version, True)
 342         self.create_accounts()
 343
 344
 345     def create_watching_only_wallet(self, params):
 346         K0, c0 = params
 347         if not K0:
 348             return
 349
 350         if not c0:
 351             self.seed_version = 4
 352             self.storage.put('seed_version', self.seed_version, True)
 353             self.create_old_account(K0)
 354             return
 355
 356         cK0 = ""
 357         self.master_public_keys = {
 358             "m/0'/": (c0, K0, cK0),
 359             }
 360         self.storage.put('master_public_keys', self.master_public_keys, True)
 361         self.storage.put('seed_version', self.seed_version, True)
 362         self.create_account('1','Main account')
 363
 364
 365     def create_accounts(self):
 366         if self.seed_version == 4:
 367             mpk = OldAccount.mpk_from_seed(self.seed)
 368             self.create_old_account(mpk)
 369         else:
 370             # create default account
 371             self.create_master_keys('1')
 372             self.create_account('1','Main account')
 373
 374
 375     def create_master_keys(self, account_type):
 376         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 377         if account_type == '1':
 378             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 379             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 380             self.master_private_keys["m/0'/"] = k0
 381         elif account_type == '2of2':
 382             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 383             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 384             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 385             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 386             self.master_private_keys["m/1'/"] = k1
 387             self.master_private_keys["m/2'/"] = k2
 388         elif account_type == '2of3':
 389             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 390             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 391             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 392             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 393             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 394             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 395             self.master_private_keys["m/3'/"] = k3
 396             self.master_private_keys["m/4'/"] = k4
 397             self.master_private_keys["m/5'/"] = k5
 398
 399         self.storage.put('master_public_keys', self.master_public_keys, True)
 400         self.storage.put('master_private_keys', self.master_private_keys, True)
 401
 402     def has_master_public_keys(self, account_type):
 403         if account_type == '1':
 404             return "m/0'/" in self.master_public_keys
 405         elif account_type == '2of2':
 406             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 407         elif account_type == '2of3':
 408             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 409
 410     def find_root_by_master_key(self, c, K):
 411         for key, v in self.master_public_keys.items():
 412             if key == "m/":continue
 413             cc, KK, _ = v
 414             if (c == cc) and (K == KK):
 415                 return key
 416
 417     def deseed_root(self, seed, password):
 418         # for safety, we ask the user to enter their seed
 419         assert seed == self.get_seed(password)
 420         self.seed = ''
 421         self.storage.put('seed', '', True)
 422
 423
 424     def deseed_branch(self, k):
 425         # check that parent has no seed
 426         assert self.seed == ''
 427         self.master_private_keys.pop(k)
 428         self.storage.put('master_private_keys', self.master_private_keys, True)
 429
 430     def is_watching_only(self):
 431         return (self.seed == '') and (self.master_private_keys == {})
 432
 433
 434
 435     def account_id(self, account_type, i):
 436         if account_type == '1':
 437             return "m/0'/%d"%i
 438         elif account_type == '2of2':
 439             return "m/1'/%d & m/2'/%d"%(i,i)
 440         elif account_type == '2of3':
 441             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 442         else:
 443             raise Exception('unknown account type')
 444
 445
 446     def num_accounts(self, account_type):
 447         keys = self.accounts.keys()
 448         i = 0
 449         while True:
 450             account_id = self.account_id(account_type, i)
 451             if account_id not in keys: break
 452             i += 1
 453         return i
 454
 455
 456     def new_account_address(self, account_type = '1'):
 457         i = self.num_accounts(account_type)
 458         k = self.account_id(account_type,i)
 459
 460         addr = self.next_addresses.get(k)
 461         if not addr:
 462             account_id, account = self.next_account(account_type)
 463             addr = account.first_address()
 464             self.next_addresses[k] = addr
 465             self.storage.put('next_addresses',self.next_addresses)
 466
 467         return k, addr
 468
 469
 470     def next_account(self, account_type = '1'):
 471
 472         i = self.num_accounts(account_type)
 473         account_id = self.account_id(account_type,i)
 474
 475         if account_type is '1':
 476             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 477             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 478             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 479
 480         elif account_type == '2of2':
 481             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 482             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 483             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 484             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 485             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 486
 487         elif account_type == '2of3':
 488             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 489             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 490             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 491             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 492             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 493             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 494             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 495
 496         return account_id, account
 497
 498
 499     def set_label(self, name, text = None):
 500         changed = False
 501         old_text = self.labels.get(name)
 502         if text:
 503             if old_text != text:
 504                 self.labels[name] = text
 505                 changed = True
 506         else:
 507             if old_text:
 508                 self.labels.pop(name)
 509                 changed = True
 510
 511         if changed:
 512             self.storage.put('labels', self.labels, True)
 513
 514         run_hook('set_label', name, text, changed)
 515         return changed
 516
 517
 518
 519     def create_account(self, account_type = '1', name = None):
 520         k, account = self.next_account(account_type)
 521         if k in self.pending_accounts:
 522             self.pending_accounts.pop(k)
 523             self.storage.put('pending_accounts', self.pending_accounts)
 524
 525         self.accounts[k] = account
 526         self.save_accounts()
 527         if name:
 528             self.set_label(k, name)
 529
 530
 531     def create_old_account(self, mpk):
 532         self.storage.put('master_public_key', mpk, True)
 533         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 534         self.save_accounts()
 535
 536
 537     def save_accounts(self):
 538         d = {}
 539         for k, v in self.accounts.items():
 540             d[k] = v.dump()
 541         self.storage.put('accounts', d, True)
 542
 543
 544
 545     def load_accounts(self):
 546         d = self.storage.get('accounts', {})
 547         self.accounts = {}
 548         for k, v in d.items():
 549             if k == 0:
 550                 v['mpk'] = self.storage.get('master_public_key')
 551                 self.accounts[k] = OldAccount(v)
 552             elif '&' in k:
 553                 self.accounts[k] = BIP32_Account_2of2(v)
 554             else:
 555                 self.accounts[k] = BIP32_Account(v)
 556
 557         self.pending_accounts = self.storage.get('pending_accounts',{})
 558
 559
 560     def delete_pending_account(self, k):
 561         self.pending_accounts.pop(k)
 562         self.storage.put('pending_accounts', self.pending_accounts)
 563
 564     def account_is_pending(self, k):
 565         return k in self.pending_accounts
 566
 567     def create_pending_account(self, acct_type, name):
 568         k, addr = self.new_account_address(acct_type)
 569         self.set_label(k, name)
 570         self.pending_accounts[k] = addr
 571         self.storage.put('pending_accounts', self.pending_accounts)
 572
 573     def get_pending_accounts(self):
 574         return self.pending_accounts.items()
 575
 576
 577     def addresses(self, include_change = True, _next=True):
 578         o = self.get_account_addresses(-1, include_change)
 579         for a in self.accounts.keys():
 580             o += self.get_account_addresses(a, include_change)
 581
 582         if _next:
 583             for addr in self.next_addresses.values():
 584                 if addr not in o:
 585                     o += [addr]
 586         return o
 587
 588
 589     def is_mine(self, address):
 590         return address in self.addresses(True)
 591
 592
 593     def is_change(self, address):
 594         if not self.is_mine(address): return False
 595         if address in self.imported_keys.keys(): return False
 596         acct, s = self.get_address_index(address)
 597         if s is None: return False
 598         return s[0] == 1
 599
 600     def get_master_public_key(self):
 601         if self.seed_version == 4:
 602             return self.storage.get("master_public_key")
 603         else:
 604             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 605             return repr((c, K))
 606
 607     def get_master_private_key(self, account, password):
 608         master_k = pw_decode( self.master_private_keys[account], password)
 609         master_c, master_K, master_Kc = self.master_public_keys[account]
 610         try:
 611             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 612             assert K.encode('hex') == master_K
 613         except Exception:
 614             raise Exception("Invalid password")
 615         return master_k
 616
 617
 618     def get_address_index(self, address):
 619         if address in self.imported_keys.keys():
 620             return -1, None
 621
 622         for account in self.accounts.keys():
 623             for for_change in [0,1]:
 624                 addresses = self.accounts[account].get_addresses(for_change)
 625                 for addr in addresses:
 626                     if address == addr:
 627                         return account, (for_change, addresses.index(addr))
 628
 629         for k,v in self.next_addresses.items():
 630             if v == address:
 631                 return k, (0,0)
 632
 633         raise Exception("Address not found", address)
 634
 635
 636     def get_roots(self, account):
 637         roots = []
 638         for a in account.split('&'):
 639             s = a.strip()
 640             m = re.match("(m/\d+'/)(\d+)", s)
 641             roots.append( m.group(1) )
 642         return roots
 643
 644     def is_seeded(self, account):
 645         if type(account) is int:
 646             return self.seed is not None
 647
 648         for root in self.get_roots(account):
 649             if root not in self.master_private_keys.keys():
 650                 return False
 651         return True
 652
 653     def rebase_sequence(self, account, sequence):
 654         c, i = sequence
 655         dd = []
 656         for a in account.split('&'):
 657             s = a.strip()
 658             m = re.match("(m/\d+'/)(\d+)", s)
 659             root = m.group(1)
 660             num = int(m.group(2))
 661             dd.append( (root, [num,c,i] ) )
 662         return dd
 663
 664
 665     def get_keyID(self, account, sequence):
 666         if account == 0:
 667             a, b = sequence
 668             mpk = self.storage.get('master_public_key')
 669             return 'old(%s,%d,%d)'%(mpk,a,b)
 670
 671         rs = self.rebase_sequence(account, sequence)
 672         dd = []
 673         for root, public_sequence in rs:
 674             c, K, _ = self.master_public_keys[root]
 675             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 676             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 677         return '&'.join(dd)
 678
 679
 680
 681     def get_seed(self, password):
 682         s = pw_decode(self.seed, password)
 683         if self.seed_version == 4:
 684             seed = s
 685             self.accounts[0].check_seed(seed)
 686         else:
 687             seed = mnemonic_hash(s)
 688         return seed
 689
 690
 691     def get_mnemonic(self, password):
 692         import mnemonic
 693         s = pw_decode(self.seed, password)
 694         if self.seed_version == 4:
 695             return ' '.join(mnemonic.mn_encode(s))
 696         else:
 697             return s
 698
 699
 700
 701     def get_private_key(self, address, password):
 702         # first check the provided password
 703         seed = self.get_seed(password)
 704
 705         out = []
 706         if address in self.imported_keys.keys():
 707             out.append( pw_decode( self.imported_keys[address], password ) )
 708         else:
 709             account, sequence = self.get_address_index(address)
 710             if account == 0:
 711                 pk = self.accounts[account].get_private_key(seed, sequence)
 712                 out.append(pk)
 713                 return out
 714
 715             # assert address == self.accounts[account].get_address(*sequence)
 716             rs = self.rebase_sequence( account, sequence)
 717             for root, public_sequence in rs:
 718
 719                 if root not in self.master_private_keys.keys(): continue
 720                 master_k = self.get_master_private_key(root, password)
 721                 master_c, _, _ = self.master_public_keys[root]
 722                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 723                 out.append(pk)
 724
 725         return out
 726
 727
 728     def add_keypairs_from_wallet(self, tx, keypairs, password):
 729         for txin in tx.inputs:
 730             address = txin['address']
 731             private_keys = self.get_private_key(address, password)
 732             for sec in private_keys:
 733                 pubkey = public_key_from_private_key(sec)
 734                 keypairs[ pubkey ] = sec
 735                 if address in self.imported_keys.keys():
 736                     txin['redeemPubkey'] = pubkey
 737
 738
 739     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 740         for txin in tx.inputs:
 741             keyid = txin.get('KeyID')
 742             if keyid:
 743
 744                 if self.seed_version==4:
 745                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 746                     if not m: continue
 747                     mpk = m.group(1)
 748                     if mpk != self.storage.get('master_public_key'): continue
 749                     index = int(m.group(2))
 750                     num = int(m.group(3))
 751                     account = self.accounts[0]
 752                     addr = account.get_address(index, num)
 753                     txin['address'] = addr # fixme: side effect
 754                     pk = self.get_private_key(addr, password)
 755                     for sec in pk:
 756                         pubkey = public_key_from_private_key(sec)
 757                         keypairs[pubkey] = sec
 758                     continue
 759
 760
 761                 roots = []
 762                 for s in keyid.split('&'):
 763                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 764                     if not m: continue
 765                     c = m.group(1)
 766                     K = m.group(2)
 767                     sequence = m.group(3)
 768                     root = self.find_root_by_master_key(c,K)
 769                     if not root: continue
 770                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 771                     root = root + '%d'%sequence[0]
 772                     sequence = sequence[1:]
 773                     roots.append((root,sequence))
 774
 775                 account_id = " & ".join( map(lambda x:x[0], roots) )
 776                 account = self.accounts.get(account_id)
 777                 if not account: continue
 778                 addr = account.get_address(*sequence)
 779                 txin['address'] = addr # fixme: side effect
 780                 pk = self.get_private_key(addr, password)
 781                 for sec in pk:
 782                     pubkey = public_key_from_private_key(sec)
 783                     keypairs[pubkey] = sec
 784
 785
 786
 787     def signrawtransaction(self, tx, input_info, private_keys, password):
 788
 789         # check that the password is correct
 790         seed = self.get_seed(password)
 791
 792         # add input info
 793         tx.add_input_info(input_info)
 794
 795         # add redeem script for coins that are in the wallet
 796         # FIXME: add redeemPubkey too!
 797         unspent_coins = self.get_unspent_coins()
 798         for txin in tx.inputs:
 799             for item in unspent_coins:
 800                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 801                     print_error( "tx input is in unspent coins" )
 802                     txin['scriptPubKey'] = item['scriptPubKey']
 803                     account, sequence = self.get_address_index(item['address'])
 804                     if account != -1:
 805                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 806                         print_error("added redeemScript", txin['redeemScript'])
 807                     break
 808
 809
 810         # build a list of public/private keys
 811         keypairs = {}
 812
 813         # add private keys from parameter
 814         for sec in private_keys:
 815             pubkey = public_key_from_private_key(sec)
 816             keypairs[ pubkey ] = sec
 817
 818         # add private_keys from KeyID
 819         self.add_keypairs_from_KeyID(tx, keypairs, password)
 820
 821         # add private keys from wallet
 822         self.add_keypairs_from_wallet(tx, keypairs, password)
 823         self.sign_transaction(tx, keypairs)
 824
 825
 826     def sign_message(self, address, message, password):
 827         keys = self.get_private_key(address, password)
 828         assert len(keys) == 1
 829         sec = keys[0]
 830         key = regenerate_key(sec)
 831         compressed = is_compressed(sec)
 832         return key.sign_message(message, compressed, address)
 833
 834
 835     def change_gap_limit(self, value):
 836         if value >= self.gap_limit:
 837             self.gap_limit = value
 838             self.storage.put('gap_limit', self.gap_limit, True)
 839             #self.interface.poke('synchronizer')
 840             return True
 841
 842         elif value >= self.min_acceptable_gap():
 843             for key, account in self.accounts.items():
 844                 addresses = account[0]
 845                 k = self.num_unused_trailing_addresses(addresses)
 846                 n = len(addresses) - k + value
 847                 addresses = addresses[0:n]
 848                 self.accounts[key][0] = addresses
 849
 850             self.gap_limit = value
 851             self.storage.put('gap_limit', self.gap_limit, True)
 852             self.save_accounts()
 853             return True
 854         else:
 855             return False
 856
 857     def num_unused_trailing_addresses(self, addresses):
 858         k = 0
 859         for a in addresses[::-1]:
 860             if self.history.get(a):break
 861             k = k + 1
 862         return k
 863
 864     def min_acceptable_gap(self):
 865         # fixme: this assumes wallet is synchronized
 866         n = 0
 867         nmax = 0
 868
 869         for account in self.accounts.values():
 870             addresses = account.get_addresses(0)
 871             k = self.num_unused_trailing_addresses(addresses)
 872             for a in addresses[0:-k]:
 873                 if self.history.get(a):
 874                     n = 0
 875                 else:
 876                     n += 1
 877                     if n > nmax: nmax = n
 878         return nmax + 1
 879
 880
 881     def address_is_old(self, address):
 882         age = -1
 883         h = self.history.get(address, [])
 884         if h == ['*']:
 885             return True
 886         for tx_hash, tx_height in h:
 887             if tx_height == 0:
 888                 tx_age = 0
 889             else:
 890                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 891             if tx_age > age:
 892                 age = tx_age
 893         return age > 2
 894
 895
 896     def synchronize_sequence(self, account, for_change):
 897         limit = self.gap_limit_for_change if for_change else self.gap_limit
 898         new_addresses = []
 899         while True:
 900             addresses = account.get_addresses(for_change)
 901             if len(addresses) < limit:
 902                 address = account.create_new_address(for_change)
 903                 self.history[address] = []
 904                 new_addresses.append( address )
 905                 continue
 906
 907             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 908                 break
 909             else:
 910                 address = account.create_new_address(for_change)
 911                 self.history[address] = []
 912                 new_addresses.append( address )
 913
 914         return new_addresses
 915
 916
 917
 918     def create_pending_accounts(self):
 919         for account_type in ['1','2of2','2of3']:
 920             if not self.has_master_public_keys(account_type):
 921                 continue
 922             k, a = self.new_account_address(account_type)
 923             if self.address_is_old(a):
 924                 print_error( "creating account", a )
 925                 self.create_account(account_type)
 926                 self.next_addresses.pop(k)
 927
 928
 929     def synchronize_account(self, account):
 930         new = []
 931         new += self.synchronize_sequence(account, 0)
 932         new += self.synchronize_sequence(account, 1)
 933         return new
 934
 935
 936     def synchronize(self):
 937         if self.master_public_keys:
 938             self.create_pending_accounts()
 939         new = []
 940         for account in self.accounts.values():
 941             new += self.synchronize_account(account)
 942         if new:
 943             self.save_accounts()
 944             self.storage.put('addr_history', self.history, True)
 945         return new
 946
 947
 948     def is_found(self):
 949         return self.history.values() != [[]] * len(self.history)
 950
 951
 952     def add_contact(self, address, label=None):
 953         self.addressbook.append(address)
 954         self.storage.put('contacts', self.addressbook, True)
 955         if label:
 956             self.set_label(address, label)
 957
 958
 959     def delete_contact(self, addr):
 960         if addr in self.addressbook:
 961             self.addressbook.remove(addr)
 962             self.storage.put('addressbook', self.addressbook, True)
 963
 964
 965     def fill_addressbook(self):
 966         for tx_hash, tx in self.transactions.items():
 967             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 968             if is_send:
 969                 for addr, v in tx.outputs:
 970                     if not self.is_mine(addr) and addr not in self.addressbook:
 971                         self.addressbook.append(addr)
 972         # redo labels
 973         # self.update_tx_labels()
 974
 975     def get_num_tx(self, address):
 976         n = 0
 977         for tx in self.transactions.values():
 978             if address in map(lambda x:x[0], tx.outputs): n += 1
 979         return n
 980
 981
 982     def get_address_flags(self, addr):
 983         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 984         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 985         return flags
 986
 987
 988     def get_tx_value(self, tx, account=None):
 989         domain = self.get_account_addresses(account)
 990         return tx.get_value(domain, self.prevout_values)
 991
 992
 993     def update_tx_outputs(self, tx_hash):
 994         tx = self.transactions.get(tx_hash)
 995
 996         for i, (addr, value) in enumerate(tx.outputs):
 997             key = tx_hash+ ':%d'%i
 998             self.prevout_values[key] = value
 999
1000         for item in tx.inputs:
1001             if self.is_mine(item.get('address')):
1002                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1003                 self.spent_outputs.append(key)
1004
1005
1006     def get_addr_balance(self, address):
1007         assert self.is_mine(address)
1008         h = self.history.get(address,[])
1009         if h == ['*']: return 0,0
1010         c = u = 0
1011         received_coins = []   # list of coins received at address
1012
1013         for tx_hash, tx_height in h:
1014             tx = self.transactions.get(tx_hash)
1015             if not tx: continue
1016
1017             for i, (addr, value) in enumerate(tx.outputs):
1018                 if addr == address:
1019                     key = tx_hash + ':%d'%i
1020                     received_coins.append(key)
1021
1022         for tx_hash, tx_height in h:
1023             tx = self.transactions.get(tx_hash)
1024             if not tx: continue
1025             v = 0
1026
1027             for item in tx.inputs:
1028                 addr = item.get('address')
1029                 if addr == address:
1030                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1031                     value = self.prevout_values.get( key )
1032                     if key in received_coins:
1033                         v -= value
1034
1035             for i, (addr, value) in enumerate(tx.outputs):
1036                 key = tx_hash + ':%d'%i
1037                 if addr == address:
1038                     v += value
1039
1040             if tx_height:
1041                 c += v
1042             else:
1043                 u += v
1044         return c, u
1045
1046
1047     def get_account_name(self, k):
1048         if k == 0:
1049             if self.seed_version == 4:
1050                 name = 'Main account'
1051             else:
1052                 name = 'Old account'
1053         else:
1054             default = "Unnamed account"
1055             m = re.match("m/0'/(\d+)", k)
1056             if m:
1057                 num = m.group(1)
1058                 if num == '0':
1059                     default = "Main account"
1060                 else:
1061                     default = "Account %s"%num
1062
1063             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1064             if m:
1065                 num = m.group(1)
1066                 default = "2of2 account %s"%num
1067             name = self.labels.get(k, default)
1068
1069         return name
1070
1071     def get_account_names(self):
1072         accounts = {}
1073         for k, account in self.accounts.items():
1074             accounts[k] = self.get_account_name(k)
1075         if self.imported_keys:
1076             accounts[-1] = 'Imported keys'
1077         return accounts
1078
1079     def get_account_addresses(self, a, include_change=True):
1080         if a is None:
1081             o = self.addresses(True)
1082         elif a == -1:
1083             o = self.imported_keys.keys()
1084         else:
1085             ac = self.accounts[a]
1086             o = ac.get_addresses(0)
1087             if include_change: o += ac.get_addresses(1)
1088         return o
1089
1090     def get_imported_balance(self):
1091         return self.get_balance(self.imported_keys.keys())
1092
1093     def get_account_balance(self, account):
1094         return self.get_balance(self.get_account_addresses(account))
1095
1096     def get_frozen_balance(self):
1097         return self.get_balance(self.frozen_addresses)
1098
1099     def get_balance(self, domain=None):
1100         if domain is None: domain = self.addresses(True)
1101         cc = uu = 0
1102         for addr in domain:
1103             c, u = self.get_addr_balance(addr)
1104             cc += c
1105             uu += u
1106         return cc, uu
1107
1108
1109     def get_unspent_coins(self, domain=None):
1110         coins = []
1111         if domain is None: domain = self.addresses(True)
1112         for addr in domain:
1113             h = self.history.get(addr, [])
1114             if h == ['*']: continue
1115             for tx_hash, tx_height in h:
1116                 tx = self.transactions.get(tx_hash)
1117                 if tx is None: raise Exception("Wallet not synchronized")
1118                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1119                 for o in tx.d.get('outputs'):
1120                     output = o.copy()
1121                     if output.get('address') != addr: continue
1122                     key = tx_hash + ":%d" % output.get('prevout_n')
1123                     if key in self.spent_outputs: continue
1124                     output['prevout_hash'] = tx_hash
1125                     output['height'] = tx_height
1126                     output['coinbase'] = is_coinbase
1127                     coins.append((tx_height, output))
1128
1129         # sort by age
1130         if coins:
1131             coins = sorted(coins)
1132             if coins[-1][0] != 0:
1133                 while coins[0][0] == 0:
1134                     coins = coins[1:] + [ coins[0] ]
1135         return [x[1] for x in coins]
1136
1137
1138
1139     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1140         domain = self.get_account_addresses(account) if account else None
1141         return self.choose_tx_inputs( amount, fixed_fee, domain )
1142
1143
1144     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1145         """ todo: minimize tx size """
1146         total = 0
1147         fee = self.fee if fixed_fee is None else fixed_fee
1148         if domain is None:
1149             domain = self.addresses(True)
1150
1151         for i in self.frozen_addresses:
1152             if i in domain: domain.remove(i)
1153
1154         prioritized = []
1155         for i in self.prioritized_addresses:
1156             if i in domain:
1157                 domain.remove(i)
1158                 prioritized.append(i)
1159
1160         coins = self.get_unspent_coins(domain)
1161         prioritized_coins = self.get_unspent_coins(prioritized)
1162
1163         inputs = []
1164         coins = prioritized_coins + coins
1165
1166         for item in coins:
1167             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1168                 continue
1169             addr = item.get('address')
1170             v = item.get('value')
1171             total += v
1172             inputs.append(item)
1173             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1174             if total >= amount + fee: break
1175         else:
1176             inputs = []
1177
1178         return inputs, total, fee
1179
1180
1181     def set_fee(self, fee):
1182         if self.fee != fee:
1183             self.fee = fee
1184             self.storage.put('fee_per_kb', self.fee, True)
1185
1186     def estimated_fee(self, inputs):
1187         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1188         fee = self.fee * int(round(estimated_size/1024.))
1189         if fee == 0: fee = self.fee
1190         return fee
1191
1192
1193     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1194         "add change to a transaction"
1195         change_amount = total - ( amount + fee )
1196         if change_amount > DUST_THRESHOLD:
1197             if not change_addr:
1198
1199                 # send change to one of the accounts involved in the tx
1200                 address = inputs[0].get('address')
1201                 account, _ = self.get_address_index(address)
1202
1203                 if not self.use_change or account == -1:
1204                     change_addr = inputs[-1]['address']
1205                 else:
1206                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1207
1208             # Insert the change output at a random position in the outputs
1209             posn = random.randint(0, len(outputs))
1210             outputs[posn:posn] = [( change_addr,  change_amount)]
1211         return outputs
1212
1213
1214     def get_history(self, address):
1215         with self.lock:
1216             return self.history.get(address)
1217
1218
1219     def get_status(self, h):
1220         if not h: return None
1221         if h == ['*']: return '*'
1222         status = ''
1223         for tx_hash, height in h:
1224             status += tx_hash + ':%d:' % height
1225         return hashlib.sha256( status ).digest().encode('hex')
1226
1227
1228     def receive_tx_callback(self, tx_hash, tx, tx_height):
1229
1230         with self.transaction_lock:
1231             self.add_extra_addresses(tx)
1232             if not self.check_new_tx(tx_hash, tx):
1233                 # may happen due to pruning
1234                 print_error("received transaction that is no longer referenced in history", tx_hash)
1235                 return
1236             self.transactions[tx_hash] = tx
1237             self.network.interface.pending_transactions_for_notifications.append(tx)
1238             self.save_transactions()
1239             if self.verifier and tx_height>0:
1240                 self.verifier.add(tx_hash, tx_height)
1241             self.update_tx_outputs(tx_hash)
1242
1243
1244     def save_transactions(self):
1245         tx = {}
1246         for k,v in self.transactions.items():
1247             tx[k] = str(v)
1248         self.storage.put('transactions', tx, True)
1249
1250     def receive_history_callback(self, addr, hist):
1251
1252         if not self.check_new_history(addr, hist):
1253             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1254
1255         with self.lock:
1256             self.history[addr] = hist
1257             self.storage.put('addr_history', self.history, True)
1258
1259         if hist != ['*']:
1260             for tx_hash, tx_height in hist:
1261                 if tx_height>0:
1262                     # add it in case it was previously unconfirmed
1263                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1264
1265
1266     def get_tx_history(self, account=None):
1267         if not self.verifier:
1268             return []
1269
1270         with self.transaction_lock:
1271             history = self.transactions.items()
1272             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1273             result = []
1274
1275             balance = 0
1276             for tx_hash, tx in history:
1277                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1278                 if v is not None: balance += v
1279
1280             c, u = self.get_account_balance(account)
1281
1282             if balance != c+u:
1283                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1284
1285             balance = c + u - balance
1286             for tx_hash, tx in history:
1287                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1288                 if not is_relevant:
1289                     continue
1290                 if value is not None:
1291                     balance += value
1292
1293                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1294                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1295
1296         return result
1297
1298
1299     def get_label(self, tx_hash):
1300         label = self.labels.get(tx_hash)
1301         is_default = (label == '') or (label is None)
1302         if is_default: label = self.get_default_label(tx_hash)
1303         return label, is_default
1304
1305
1306     def get_default_label(self, tx_hash):
1307         tx = self.transactions.get(tx_hash)
1308         default_label = ''
1309         if tx:
1310             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1311             if is_mine:
1312                 for o in tx.outputs:
1313                     o_addr, _ = o
1314                     if not self.is_mine(o_addr):
1315                         try:
1316                             default_label = self.labels[o_addr]
1317                         except KeyError:
1318                             default_label = o_addr
1319                         break
1320                 else:
1321                     default_label = '(internal)'
1322             else:
1323                 for o in tx.outputs:
1324                     o_addr, _ = o
1325                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1326                         break
1327                 else:
1328                     for o in tx.outputs:
1329                         o_addr, _ = o
1330                         if self.is_mine(o_addr):
1331                             break
1332                     else:
1333                         o_addr = None
1334
1335                 if o_addr:
1336                     dest_label = self.labels.get(o_addr)
1337                     try:
1338                         default_label = self.labels[o_addr]
1339                     except KeyError:
1340                         default_label = o_addr
1341
1342         return default_label
1343
1344
1345     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1346         for address, x in outputs:
1347             assert is_valid(address)
1348         amount = sum( map(lambda x:x[1], outputs) )
1349         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1350         if not inputs:
1351             raise ValueError("Not enough funds")
1352         self.add_input_info(inputs)
1353         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1354         return Transaction.from_io(inputs, outputs)
1355
1356
1357     def mktx_from_account(self, outputs, password, fee=None, account=None):
1358         domain = self.get_account_addresses(account) if account else None
1359         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1360
1361
1362     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1363         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1364         keypairs = {}
1365         self.add_keypairs_from_wallet(tx, keypairs, password)
1366         if keypairs:
1367             self.sign_transaction(tx, keypairs)
1368         return tx
1369
1370
1371     def add_input_info(self, inputs):
1372         for txin in inputs:
1373             address = txin['address']
1374             if address in self.imported_keys.keys():
1375                 continue
1376             account, sequence = self.get_address_index(address)
1377             txin['KeyID'] = self.get_keyID(account, sequence)
1378             redeemScript = self.accounts[account].redeem_script(sequence)
1379             if redeemScript:
1380                 txin['redeemScript'] = redeemScript
1381             else:
1382                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1383
1384
1385     def sign_transaction(self, tx, keypairs):
1386         tx.sign(keypairs)
1387         run_hook('sign_transaction', tx)
1388
1389
1390     def sendtx(self, tx):
1391         # synchronous
1392         h = self.send_tx(tx)
1393         self.tx_event.wait()
1394         return self.receive_tx(h)
1395
1396     def send_tx(self, tx):
1397         # asynchronous
1398         self.tx_event.clear()
1399         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1400         return tx.hash()
1401
1402     def on_broadcast(self, i, r):
1403         self.tx_result = r.get('result')
1404         self.tx_event.set()
1405
1406     def receive_tx(self,tx_hash):
1407         out = self.tx_result
1408         if out != tx_hash:
1409             return False, "error: " + out
1410         return True, out
1411
1412
1413
1414     def update_password(self, old_password, new_password):
1415         if new_password == '': new_password = None
1416         decoded = self.get_seed(old_password)
1417         self.seed = pw_encode( decoded, new_password)
1418         self.storage.put('seed', self.seed, True)
1419         self.use_encryption = (new_password != None)
1420         self.storage.put('use_encryption', self.use_encryption,True)
1421         for k in self.imported_keys.keys():
1422             a = self.imported_keys[k]
1423             b = pw_decode(a, old_password)
1424             c = pw_encode(b, new_password)
1425             self.imported_keys[k] = c
1426         self.storage.put('imported_keys', self.imported_keys, True)
1427
1428         for k, v in self.master_private_keys.items():
1429             b = pw_decode(v, old_password)
1430             c = pw_encode(b, new_password)
1431             self.master_private_keys[k] = c
1432         self.storage.put('master_private_keys', self.master_private_keys, True)
1433
1434
1435     def freeze(self,addr):
1436         if self.is_mine(addr) and addr not in self.frozen_addresses:
1437             self.unprioritize(addr)
1438             self.frozen_addresses.append(addr)
1439             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1440             return True
1441         else:
1442             return False
1443
1444     def unfreeze(self,addr):
1445         if self.is_mine(addr) and addr in self.frozen_addresses:
1446             self.frozen_addresses.remove(addr)
1447             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1448             return True
1449         else:
1450             return False
1451
1452     def prioritize(self,addr):
1453         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1454             self.unfreeze(addr)
1455             self.prioritized_addresses.append(addr)
1456             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1457             return True
1458         else:
1459             return False
1460
1461     def unprioritize(self,addr):
1462         if self.is_mine(addr) and addr in self.prioritized_addresses:
1463             self.prioritized_addresses.remove(addr)
1464             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1465             return True
1466         else:
1467             return False
1468
1469
1470     def set_verifier(self, verifier):
1471         self.verifier = verifier
1472
1473         # review transactions that are in the history
1474         for addr, hist in self.history.items():
1475             if hist == ['*']: continue
1476             for tx_hash, tx_height in hist:
1477                 if tx_height>0:
1478                     # add it in case it was previously unconfirmed
1479                     self.verifier.add(tx_hash, tx_height)
1480
1481
1482         # if we are on a pruning server, remove unverified transactions
1483         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1484         for tx_hash in self.transactions.keys():
1485             if tx_hash not in vr:
1486                 self.transactions.pop(tx_hash)
1487
1488
1489
1490     def check_new_history(self, addr, hist):
1491
1492         # check that all tx in hist are relevant
1493         if hist != ['*']:
1494             for tx_hash, height in hist:
1495                 tx = self.transactions.get(tx_hash)
1496                 if not tx: continue
1497                 if not tx.has_address(addr):
1498                     return False
1499
1500         # check that we are not "orphaning" a transaction
1501         old_hist = self.history.get(addr,[])
1502         if old_hist == ['*']: return True
1503
1504         for tx_hash, height in old_hist:
1505             if tx_hash in map(lambda x:x[0], hist): continue
1506             found = False
1507             for _addr, _hist in self.history.items():
1508                 if _addr == addr: continue
1509                 if _hist == ['*']: continue
1510                 _tx_hist = map(lambda x:x[0], _hist)
1511                 if tx_hash in _tx_hist:
1512                     found = True
1513                     break
1514
1515             if not found:
1516                 tx = self.transactions.get(tx_hash)
1517                 # tx might not be there
1518                 if not tx: continue
1519
1520                 # already verified?
1521                 if self.verifier.get_height(tx_hash):
1522                     continue
1523                 # unconfirmed tx
1524                 print_error("new history is orphaning transaction:", tx_hash)
1525                 # check that all outputs are not mine, request histories
1526                 ext_requests = []
1527                 for _addr, _v in tx.outputs:
1528                     # assert not self.is_mine(_addr)
1529                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1530
1531                 ext_h = self.network.synchronous_get(ext_requests)
1532                 print_error("sync:", ext_requests, ext_h)
1533                 height = None
1534                 for h in ext_h:
1535                     if h == ['*']: continue
1536                     for item in h:
1537                         if item.get('tx_hash') == tx_hash:
1538                             height = item.get('height')
1539                 if height:
1540                     print_error("found height for", tx_hash, height)
1541                     self.verifier.add(tx_hash, height)
1542                 else:
1543                     print_error("removing orphaned tx from history", tx_hash)
1544                     self.transactions.pop(tx_hash)
1545
1546         return True
1547
1548
1549
1550     def check_new_tx(self, tx_hash, tx):
1551         # 1 check that tx is referenced in addr_history.
1552         addresses = []
1553         for addr, hist in self.history.items():
1554             if hist == ['*']:continue
1555             for txh, height in hist:
1556                 if txh == tx_hash:
1557                     addresses.append(addr)
1558
1559         if not addresses:
1560             return False
1561
1562         # 2 check that referencing addresses are in the tx
1563         for addr in addresses:
1564             if not tx.has_address(addr):
1565                 return False
1566
1567         return True
1568
1569
1570     def start_threads(self, network):
1571         from verifier import TxVerifier
1572         self.network = network
1573         if self.network:
1574             self.verifier = TxVerifier(self.network, self.storage)
1575             self.verifier.start()
1576             self.set_verifier(self.verifier)
1577             self.synchronizer = WalletSynchronizer(self, network)
1578             self.synchronizer.start()
1579         else:
1580             self.verifier = None
1581             self.synchronizer =None
1582
1583     def stop_threads(self):
1584         if self.network:
1585             self.verifier.stop()
1586             self.synchronizer.stop()
1587
1588
1589     def restore(self, callback):
1590         from i18n import _
1591         def wait_for_wallet():
1592             self.set_up_to_date(False)
1593             while not self.is_up_to_date():
1594                 msg = "%s\n%s %d\n%s %.1f"%(
1595                     _("Please wait..."),
1596                     _("Addresses generated:"),
1597                     len(self.addresses(True)),
1598                     _("Kilobytes received:"),
1599                     self.network.interface.bytes_received/1024.)
1600
1601                 apply(callback, (msg,))
1602                 time.sleep(0.1)
1603
1604         def wait_for_network():
1605             while not self.network.is_connected():
1606                 msg = "%s \n" % (_("Connecting..."))
1607                 apply(callback, (msg,))
1608                 time.sleep(0.1)
1609
1610         # wait until we are connected, because the user might have selected another server
1611         if self.network:
1612             wait_for_network()
1613             wait_for_wallet()
1614         else:
1615             self.synchronize()
1616
1617         self.fill_addressbook()
1618
1619
1620
1621
1622 class WalletSynchronizer(threading.Thread):
1623
1624
1625     def __init__(self, wallet, network):
1626         threading.Thread.__init__(self)
1627         self.daemon = True
1628         self.wallet = wallet
1629         self.network = network
1630         self.was_updated = True
1631         self.running = False
1632         self.lock = threading.Lock()
1633         self.queue = Queue.Queue()
1634
1635     def stop(self):
1636         with self.lock: self.running = False
1637
1638     def is_running(self):
1639         with self.lock: return self.running
1640
1641
1642     def subscribe_to_addresses(self, addresses):
1643         messages = []
1644         for addr in addresses:
1645             messages.append(('blockchain.address.subscribe', [addr]))
1646         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1647
1648
1649     def run(self):
1650         with self.lock:
1651             self.running = True
1652
1653         while self.is_running():
1654
1655             if not self.network.is_connected():
1656                 print_error("synchronizer: waiting for interface")
1657                 self.network.wait_until_connected()
1658
1659             self.run_interface(self.network.interface)
1660
1661
1662     def run_interface(self, interface):
1663
1664         print_error("synchronizer: connected to", interface.server)
1665
1666         requested_tx = []
1667         missing_tx = []
1668         requested_histories = {}
1669
1670         # request any missing transactions
1671         for history in self.wallet.history.values():
1672             if history == ['*']: continue
1673             for tx_hash, tx_height in history:
1674                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1675                     missing_tx.append( (tx_hash, tx_height) )
1676
1677         if missing_tx:
1678             print_error("missing tx", missing_tx)
1679
1680         # subscriptions
1681         self.subscribe_to_addresses(self.wallet.addresses(True))
1682
1683         while self.is_running():
1684             # 1. create new addresses
1685             new_addresses = self.wallet.synchronize()
1686
1687             # request missing addresses
1688             if new_addresses:
1689                 self.subscribe_to_addresses(new_addresses)
1690
1691             # request missing transactions
1692             for tx_hash, tx_height in missing_tx:
1693                 if (tx_hash, tx_height) not in requested_tx:
1694                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1695                     requested_tx.append( (tx_hash, tx_height) )
1696             missing_tx = []
1697
1698             # detect if situation has changed
1699             if interface.is_up_to_date() and self.queue.empty():
1700                 if not self.wallet.is_up_to_date():
1701                     self.wallet.set_up_to_date(True)
1702                     self.was_updated = True
1703             else:
1704                 if self.wallet.is_up_to_date():
1705                     self.wallet.set_up_to_date(False)
1706                     self.was_updated = True
1707
1708             if self.was_updated:
1709                 self.wallet.network.trigger_callback('updated')
1710                 self.was_updated = False
1711
1712             # 2. get a response
1713             try:
1714                 r = self.queue.get(block=True, timeout=1)
1715             except Queue.Empty:
1716                 continue
1717
1718             if interface != self.network.interface:
1719                 break
1720
1721             if not r:
1722                 continue
1723
1724             # 3. handle response
1725             method = r['method']
1726             params = r['params']
1727             result = r.get('result')
1728             error = r.get('error')
1729             if error:
1730                 print "error", r
1731                 continue
1732
1733             if method == 'blockchain.address.subscribe':
1734                 addr = params[0]
1735                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1736                     if requested_histories.get(addr) is None:
1737                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1738                         requested_histories[addr] = result
1739
1740             elif method == 'blockchain.address.get_history':
1741                 addr = params[0]
1742                 print_error("receiving history", addr, result)
1743                 if result == ['*']:
1744                     assert requested_histories.pop(addr) == '*'
1745                     self.wallet.receive_history_callback(addr, result)
1746                 else:
1747                     hist = []
1748                     # check that txids are unique
1749                     txids = []
1750                     for item in result:
1751                         tx_hash = item['tx_hash']
1752                         if tx_hash not in txids:
1753                             txids.append(tx_hash)
1754                             hist.append( (tx_hash, item['height']) )
1755
1756                     if len(hist) != len(result):
1757                         raise Exception("error: server sent history with non-unique txid", result)
1758
1759                     # check that the status corresponds to what was announced
1760                     rs = requested_histories.pop(addr)
1761                     if self.wallet.get_status(hist) != rs:
1762                         raise Exception("error: status mismatch: %s"%addr)
1763
1764                     # store received history
1765                     self.wallet.receive_history_callback(addr, hist)
1766
1767                     # request transactions that we don't have
1768                     for tx_hash, tx_height in hist:
1769                         if self.wallet.transactions.get(tx_hash) is None:
1770                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1771                                 missing_tx.append( (tx_hash, tx_height) )
1772
1773             elif method == 'blockchain.transaction.get':
1774                 tx_hash = params[0]
1775                 tx_height = params[1]
1776                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1777                 tx = Transaction(result)
1778                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1779                 self.was_updated = True
1780                 requested_tx.remove( (tx_hash, tx_height) )
1781                 print_error("received tx:", tx_hash, len(tx.raw))
1782
1783             else:
1784                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1785
1786             if self.was_updated and not requested_tx:
1787                 self.wallet.network.trigger_callback('updated')
1788                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1789
1790                 self.was_updated = False