lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         return self.data.get(key, default)
 129
 130     def put(self, key, value, save = True):
 131
 132         with self.lock:
 133             if value is not None:
 134                 self.data[key] = value
 135             else:
 136                 self.data.pop(key)
 137             if save:
 138                 self.write()
 139
 140     def write(self):
 141         s = repr(self.data)
 142         f = open(self.path,"w")
 143         f.write( s )
 144         f.close()
 145         if 'ANDROID_DATA' not in os.environ:
 146             import stat
 147             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 148
 149
 150 class Wallet:
 151
 152     def __init__(self, storage):
 153
 154         self.storage = storage
 155         self.electrum_version = ELECTRUM_VERSION
 156         self.gap_limit_for_change = 3 # constant
 157
 158         # saved fields
 159         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 160
 161         self.gap_limit             = storage.get('gap_limit', 5)
 162         self.use_change            = storage.get('use_change',True)
 163         self.use_encryption        = storage.get('use_encryption', False)
 164         self.seed                  = storage.get('seed', '')               # encrypted
 165         self.labels                = storage.get('labels', {})
 166         self.frozen_addresses      = storage.get('frozen_addresses',[])
 167         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 168         self.addressbook           = storage.get('contacts', [])
 169
 170         self.imported_keys         = storage.get('imported_keys',{})
 171         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 172
 173         self.fee                   = int(storage.get('fee_per_kb',20000))
 174
 175         self.master_public_keys = storage.get('master_public_keys',{})
 176         self.master_private_keys = storage.get('master_private_keys', {})
 177
 178         self.next_addresses = storage.get('next_addresses',{})
 179
 180         if self.seed_version not in [4, 6]:
 181             msg = "This wallet seed is not supported."
 182             if self.seed_version in [5]:
 183                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 184                 print msg
 185                 sys.exit(1)
 186
 187
 188         self.load_accounts()
 189
 190         self.transactions = {}
 191         tx_list = self.storage.get('transactions',{})
 192         for k,v in tx_list.items():
 193             try:
 194                 tx = Transaction(v)
 195             except Exception:
 196                 print_msg("Warning: Cannot deserialize transactions. skipping")
 197                 continue
 198
 199             self.add_extra_addresses(tx)
 200             self.transactions[k] = tx
 201
 202         for h,tx in self.transactions.items():
 203             if not self.check_new_tx(h, tx):
 204                 print_error("removing unreferenced tx", h)
 205                 self.transactions.pop(h)
 206
 207
 208         # not saved
 209         self.prevout_values = {}     # my own transaction outputs
 210         self.spent_outputs = []
 211
 212         # spv
 213         self.verifier = None
 214
 215         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 216         # interface.is_up_to_date() returns true when all requests have been answered and processed
 217         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 218
 219         self.up_to_date = False
 220         self.lock = threading.Lock()
 221         self.transaction_lock = threading.Lock()
 222         self.tx_event = threading.Event()
 223
 224         for tx_hash, tx in self.transactions.items():
 225             self.update_tx_outputs(tx_hash)
 226
 227
 228     def add_extra_addresses(self, tx):
 229         h = tx.hash()
 230         # find the address corresponding to pay-to-pubkey inputs
 231         tx.add_extra_addresses(self.transactions)
 232         for o in tx.d.get('outputs'):
 233             if o.get('is_pubkey'):
 234                 for tx2 in self.transactions.values():
 235                     tx2.add_extra_addresses({h:tx})
 236
 237
 238
 239
 240     def set_up_to_date(self,b):
 241         with self.lock: self.up_to_date = b
 242
 243
 244     def is_up_to_date(self):
 245         with self.lock: return self.up_to_date
 246
 247
 248     def update(self):
 249         self.up_to_date = False
 250         while not self.is_up_to_date():
 251             time.sleep(0.1)
 252
 253
 254     def import_key(self, sec, password):
 255         # check password
 256         seed = self.get_seed(password)
 257         try:
 258             address = address_from_private_key(sec)
 259         except Exception:
 260             raise Exception('Invalid private key')
 261
 262         if self.is_mine(address):
 263             raise Exception('Address already in wallet')
 264
 265         # store the originally requested keypair into the imported keys table
 266         self.imported_keys[address] = pw_encode(sec, password )
 267         self.storage.put('imported_keys', self.imported_keys, True)
 268         if self.synchronizer:
 269             self.synchronizer.subscribe_to_addresses([address])
 270         return address
 271
 272     def delete_imported_key(self, addr):
 273         if addr in self.imported_keys:
 274             self.imported_keys.pop(addr)
 275             self.storage.put('imported_keys', self.imported_keys, True)
 276
 277
 278     def make_seed(self):
 279         import mnemonic, ecdsa
 280         entropy = ecdsa.util.randrange( pow(2,160) )
 281         nonce = 0
 282         while True:
 283             ss = "%040x"%(entropy+nonce)
 284             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 285             # we keep only 13 words, that's approximately 139 bits of entropy
 286             words = mnemonic.mn_encode(s)[0:13]
 287             seed = ' '.join(words)
 288             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 289                 break  # this removes 12 bits of entropy
 290             nonce += 1
 291
 292         return seed
 293
 294
 295     def init_seed(self, seed):
 296         import mnemonic
 297
 298         if self.seed:
 299             raise Exception("a seed exists")
 300
 301         if not seed:
 302             self.seed = random_seed(128)
 303             self.seed_version = 4
 304             return
 305
 306         #if not seed:
 307         #    self.seed = self.make_seed()
 308         #    self.seed_version = SEED_VERSION
 309         #    return
 310
 311         # find out what kind of wallet we are
 312         try:
 313             seed.strip().decode('hex')
 314             self.seed_version = 4
 315             self.seed = str(seed)
 316             return
 317         except Exception:
 318             pass
 319
 320         words = seed.split()
 321         self.seed_version = 4
 322         self.seed = mnemonic.mn_decode(words)
 323
 324         #try:
 325         #    mnemonic.mn_decode(words)
 326         #    uses_electrum_words = True
 327         #except Exception:
 328         #    uses_electrum_words = False
 329         #
 330         #if uses_electrum_words and len(words) != 13:
 331         #    self.seed_version = 4
 332         #    self.seed = mnemonic.mn_decode(words)
 333         #else:
 334         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 335         #    self.seed_version = SEED_VERSION
 336         #    self.seed = seed
 337
 338
 339     def save_seed(self):
 340         self.storage.put('seed', self.seed, True)
 341         self.storage.put('seed_version', self.seed_version, True)
 342         self.create_accounts()
 343
 344
 345     def create_watching_only_wallet(self, params):
 346         K0, c0 = params
 347         if not K0:
 348             return
 349
 350         if not c0:
 351             self.seed_version = 4
 352             self.storage.put('seed_version', self.seed_version, True)
 353             self.create_old_account(K0)
 354             return
 355
 356         cK0 = ""
 357         self.master_public_keys = {
 358             "m/0'/": (c0, K0, cK0),
 359             }
 360         self.storage.put('master_public_keys', self.master_public_keys, True)
 361         self.storage.put('seed_version', self.seed_version, True)
 362         self.create_account('1','Main account')
 363
 364
 365     def create_accounts(self):
 366         if self.seed_version == 4:
 367             mpk = OldAccount.mpk_from_seed(self.seed)
 368             self.create_old_account(mpk)
 369         else:
 370             # create default account
 371             self.create_master_keys('1')
 372             self.create_account('1','Main account')
 373
 374
 375     def create_master_keys(self, account_type):
 376         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 377         if account_type == '1':
 378             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 379             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 380             self.master_private_keys["m/0'/"] = k0
 381         elif account_type == '2of2':
 382             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 383             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 384             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 385             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 386             self.master_private_keys["m/1'/"] = k1
 387             self.master_private_keys["m/2'/"] = k2
 388         elif account_type == '2of3':
 389             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 390             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 391             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 392             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 393             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 394             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 395             self.master_private_keys["m/3'/"] = k3
 396             self.master_private_keys["m/4'/"] = k4
 397             self.master_private_keys["m/5'/"] = k5
 398
 399         self.storage.put('master_public_keys', self.master_public_keys, True)
 400         self.storage.put('master_private_keys', self.master_private_keys, True)
 401
 402     def has_master_public_keys(self, account_type):
 403         if account_type == '1':
 404             return "m/0'/" in self.master_public_keys
 405         elif account_type == '2of2':
 406             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 407         elif account_type == '2of3':
 408             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 409
 410     def find_root_by_master_key(self, c, K):
 411         for key, v in self.master_public_keys.items():
 412             if key == "m/":continue
 413             cc, KK, _ = v
 414             if (c == cc) and (K == KK):
 415                 return key
 416
 417     def deseed_root(self, seed, password):
 418         # for safety, we ask the user to enter their seed
 419         assert seed == self.get_seed(password)
 420         self.seed = ''
 421         self.storage.put('seed', '', True)
 422
 423
 424     def deseed_branch(self, k):
 425         # check that parent has no seed
 426         assert self.seed == ''
 427         self.master_private_keys.pop(k)
 428         self.storage.put('master_private_keys', self.master_private_keys, True)
 429
 430     def is_watching_only(self):
 431         return (self.seed == '') and (self.master_private_keys == {})
 432
 433
 434
 435     def account_id(self, account_type, i):
 436         if account_type == '1':
 437             return "m/0'/%d"%i
 438         elif account_type == '2of2':
 439             return "m/1'/%d & m/2'/%d"%(i,i)
 440         elif account_type == '2of3':
 441             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 442         else:
 443             raise Exception('unknown account type')
 444
 445
 446     def num_accounts(self, account_type):
 447         keys = self.accounts.keys()
 448         i = 0
 449         while True:
 450             account_id = self.account_id(account_type, i)
 451             if account_id not in keys: break
 452             i += 1
 453         return i
 454
 455
 456     def new_account_address(self, account_type = '1'):
 457         i = self.num_accounts(account_type)
 458         k = self.account_id(account_type,i)
 459
 460         addr = self.next_addresses.get(k)
 461         if not addr:
 462             account_id, account = self.next_account(account_type)
 463             addr = account.first_address()
 464             self.next_addresses[k] = addr
 465             self.storage.put('next_addresses',self.next_addresses)
 466
 467         return k, addr
 468
 469
 470     def next_account(self, account_type = '1'):
 471
 472         i = self.num_accounts(account_type)
 473         account_id = self.account_id(account_type,i)
 474
 475         if account_type is '1':
 476             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 477             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 478             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 479
 480         elif account_type == '2of2':
 481             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 482             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 483             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 484             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 485             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 486
 487         elif account_type == '2of3':
 488             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 489             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 490             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 491             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 492             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 493             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 494             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 495
 496         return account_id, account
 497
 498
 499     def set_label(self, name, text = None):
 500         changed = False
 501         old_text = self.labels.get(name)
 502         if text:
 503             if old_text != text:
 504                 self.labels[name] = text
 505                 changed = True
 506         else:
 507             if old_text:
 508                 self.labels.pop(name)
 509                 changed = True
 510
 511         if changed:
 512             self.storage.put('labels', self.labels, True)
 513
 514         run_hook('set_label', name, text, changed)
 515         return changed
 516
 517
 518
 519     def create_account(self, account_type = '1', name = None):
 520         k, account = self.next_account(account_type)
 521         if k in self.pending_accounts:
 522             self.pending_accounts.pop(k)
 523             self.storage.put('pending_accounts', self.pending_accounts)
 524
 525         self.accounts[k] = account
 526         self.save_accounts()
 527         if name:
 528             self.set_label(k, name)
 529
 530
 531     def create_old_account(self, mpk):
 532         self.storage.put('master_public_key', mpk, True)
 533         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 534         self.save_accounts()
 535
 536
 537     def save_accounts(self):
 538         d = {}
 539         for k, v in self.accounts.items():
 540             d[k] = v.dump()
 541         self.storage.put('accounts', d, True)
 542
 543
 544
 545     def load_accounts(self):
 546         d = self.storage.get('accounts', {})
 547         self.accounts = {}
 548         for k, v in d.items():
 549             if k == 0:
 550                 v['mpk'] = self.storage.get('master_public_key')
 551                 self.accounts[k] = OldAccount(v)
 552             elif '&' in k:
 553                 self.accounts[k] = BIP32_Account_2of2(v)
 554             else:
 555                 self.accounts[k] = BIP32_Account(v)
 556
 557         self.pending_accounts = self.storage.get('pending_accounts',{})
 558
 559
 560     def delete_pending_account(self, k):
 561         self.pending_accounts.pop(k)
 562         self.storage.put('pending_accounts', self.pending_accounts)
 563
 564     def account_is_pending(self, k):
 565         return k in self.pending_accounts
 566
 567     def create_pending_account(self, acct_type, name):
 568         k, addr = self.new_account_address(acct_type)
 569         self.set_label(k, name)
 570         self.pending_accounts[k] = addr
 571         self.storage.put('pending_accounts', self.pending_accounts)
 572
 573     def get_pending_accounts(self):
 574         return self.pending_accounts.items()
 575
 576
 577     def addresses(self, include_change = True, _next=True):
 578         o = self.get_account_addresses(-1, include_change)
 579         for a in self.accounts.keys():
 580             o += self.get_account_addresses(a, include_change)
 581
 582         if _next:
 583             for addr in self.next_addresses.values():
 584                 if addr not in o:
 585                     o += [addr]
 586         return o
 587
 588
 589     def is_mine(self, address):
 590         return address in self.addresses(True)
 591
 592
 593     def is_change(self, address):
 594         if not self.is_mine(address): return False
 595         if address in self.imported_keys.keys(): return False
 596         acct, s = self.get_address_index(address)
 597         if s is None: return False
 598         return s[0] == 1
 599
 600     def get_master_public_key(self):
 601         if self.seed_version == 4:
 602             return self.storage.get("master_public_key")
 603         else:
 604             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 605             return repr((c, K))
 606
 607     def get_master_private_key(self, account, password):
 608         master_k = pw_decode( self.master_private_keys[account], password)
 609         master_c, master_K, master_Kc = self.master_public_keys[account]
 610         try:
 611             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 612             assert K.encode('hex') == master_K
 613         except Exception:
 614             raise Exception("Invalid password")
 615         return master_k
 616
 617
 618     def get_address_index(self, address):
 619         if address in self.imported_keys.keys():
 620             return -1, None
 621
 622         for account in self.accounts.keys():
 623             for for_change in [0,1]:
 624                 addresses = self.accounts[account].get_addresses(for_change)
 625                 for addr in addresses:
 626                     if address == addr:
 627                         return account, (for_change, addresses.index(addr))
 628
 629         for k,v in self.next_addresses.items():
 630             if v == address:
 631                 return k, (0,0)
 632
 633         raise Exception("Address not found", address)
 634
 635
 636     def get_roots(self, account):
 637         roots = []
 638         for a in account.split('&'):
 639             s = a.strip()
 640             m = re.match("(m/\d+'/)(\d+)", s)
 641             roots.append( m.group(1) )
 642         return roots
 643
 644     def is_seeded(self, account):
 645         if type(account) is int:
 646             return self.seed is not None
 647
 648         for root in self.get_roots(account):
 649             if root not in self.master_private_keys.keys():
 650                 return False
 651         return True
 652
 653     def rebase_sequence(self, account, sequence):
 654         c, i = sequence
 655         dd = []
 656         for a in account.split('&'):
 657             s = a.strip()
 658             m = re.match("(m/\d+'/)(\d+)", s)
 659             root = m.group(1)
 660             num = int(m.group(2))
 661             dd.append( (root, [num,c,i] ) )
 662         return dd
 663
 664
 665     def get_keyID(self, account, sequence):
 666         if account == 0:
 667             a, b = sequence
 668             mpk = self.storage.get('master_public_key')
 669             return 'old(%s,%d,%d)'%(mpk,a,b)
 670
 671         rs = self.rebase_sequence(account, sequence)
 672         dd = []
 673         for root, public_sequence in rs:
 674             c, K, _ = self.master_public_keys[root]
 675             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 676             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 677         return '&'.join(dd)
 678
 679
 680
 681     def get_seed(self, password):
 682         s = pw_decode(self.seed, password)
 683         if self.seed_version == 4:
 684             seed = s
 685             self.accounts[0].check_seed(seed)
 686         else:
 687             seed = mnemonic_hash(s)
 688         return seed
 689
 690
 691     def get_mnemonic(self, password):
 692         import mnemonic
 693         s = pw_decode(self.seed, password)
 694         if self.seed_version == 4:
 695             return ' '.join(mnemonic.mn_encode(s))
 696         else:
 697             return s
 698
 699
 700
 701     def get_private_key(self, address, password):
 702         if self.is_watching_only():
 703             return []
 704
 705         # first check the provided password
 706         seed = self.get_seed(password)
 707
 708         out = []
 709         if address in self.imported_keys.keys():
 710             out.append( pw_decode( self.imported_keys[address], password ) )
 711         else:
 712             account, sequence = self.get_address_index(address)
 713             if account == 0:
 714                 pk = self.accounts[account].get_private_key(seed, sequence)
 715                 out.append(pk)
 716                 return out
 717
 718             # assert address == self.accounts[account].get_address(*sequence)
 719             rs = self.rebase_sequence( account, sequence)
 720             for root, public_sequence in rs:
 721
 722                 if root not in self.master_private_keys.keys(): continue
 723                 master_k = self.get_master_private_key(root, password)
 724                 master_c, _, _ = self.master_public_keys[root]
 725                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 726                 out.append(pk)
 727
 728         return out
 729
 730
 731     def add_keypairs_from_wallet(self, tx, keypairs, password):
 732         for txin in tx.inputs:
 733             address = txin['address']
 734             private_keys = self.get_private_key(address, password)
 735             for sec in private_keys:
 736                 pubkey = public_key_from_private_key(sec)
 737                 keypairs[ pubkey ] = sec
 738                 if address in self.imported_keys.keys():
 739                     txin['redeemPubkey'] = pubkey
 740
 741
 742     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 743         for txin in tx.inputs:
 744             keyid = txin.get('KeyID')
 745             if keyid:
 746
 747                 if self.seed_version==4:
 748                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 749                     if not m: continue
 750                     mpk = m.group(1)
 751                     if mpk != self.storage.get('master_public_key'): continue
 752                     index = int(m.group(2))
 753                     num = int(m.group(3))
 754                     account = self.accounts[0]
 755                     addr = account.get_address(index, num)
 756                     txin['address'] = addr # fixme: side effect
 757                     pk = self.get_private_key(addr, password)
 758                     for sec in pk:
 759                         pubkey = public_key_from_private_key(sec)
 760                         keypairs[pubkey] = sec
 761                     continue
 762
 763
 764                 roots = []
 765                 for s in keyid.split('&'):
 766                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 767                     if not m: continue
 768                     c = m.group(1)
 769                     K = m.group(2)
 770                     sequence = m.group(3)
 771                     root = self.find_root_by_master_key(c,K)
 772                     if not root: continue
 773                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 774                     root = root + '%d'%sequence[0]
 775                     sequence = sequence[1:]
 776                     roots.append((root,sequence))
 777
 778                 account_id = " & ".join( map(lambda x:x[0], roots) )
 779                 account = self.accounts.get(account_id)
 780                 if not account: continue
 781                 addr = account.get_address(*sequence)
 782                 txin['address'] = addr # fixme: side effect
 783                 pk = self.get_private_key(addr, password)
 784                 for sec in pk:
 785                     pubkey = public_key_from_private_key(sec)
 786                     keypairs[pubkey] = sec
 787
 788
 789
 790     def signrawtransaction(self, tx, input_info, private_keys, password):
 791
 792         # check that the password is correct
 793         seed = self.get_seed(password)
 794
 795         # add input info
 796         tx.add_input_info(input_info)
 797
 798         # add redeem script for coins that are in the wallet
 799         # FIXME: add redeemPubkey too!
 800         unspent_coins = self.get_unspent_coins()
 801         for txin in tx.inputs:
 802             for item in unspent_coins:
 803                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 804                     print_error( "tx input is in unspent coins" )
 805                     txin['scriptPubKey'] = item['scriptPubKey']
 806                     account, sequence = self.get_address_index(item['address'])
 807                     if account != -1:
 808                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 809                         print_error("added redeemScript", txin['redeemScript'])
 810                     break
 811
 812
 813         # build a list of public/private keys
 814         keypairs = {}
 815
 816         # add private keys from parameter
 817         for sec in private_keys:
 818             pubkey = public_key_from_private_key(sec)
 819             keypairs[ pubkey ] = sec
 820
 821         # add private_keys from KeyID
 822         self.add_keypairs_from_KeyID(tx, keypairs, password)
 823
 824         # add private keys from wallet
 825         self.add_keypairs_from_wallet(tx, keypairs, password)
 826         self.sign_transaction(tx, keypairs)
 827
 828
 829     def sign_message(self, address, message, password):
 830         keys = self.get_private_key(address, password)
 831         assert len(keys) == 1
 832         sec = keys[0]
 833         key = regenerate_key(sec)
 834         compressed = is_compressed(sec)
 835         return key.sign_message(message, compressed, address)
 836
 837
 838     def change_gap_limit(self, value):
 839         if value >= self.gap_limit:
 840             self.gap_limit = value
 841             self.storage.put('gap_limit', self.gap_limit, True)
 842             #self.interface.poke('synchronizer')
 843             return True
 844
 845         elif value >= self.min_acceptable_gap():
 846             for key, account in self.accounts.items():
 847                 addresses = account[0]
 848                 k = self.num_unused_trailing_addresses(addresses)
 849                 n = len(addresses) - k + value
 850                 addresses = addresses[0:n]
 851                 self.accounts[key][0] = addresses
 852
 853             self.gap_limit = value
 854             self.storage.put('gap_limit', self.gap_limit, True)
 855             self.save_accounts()
 856             return True
 857         else:
 858             return False
 859
 860     def num_unused_trailing_addresses(self, addresses):
 861         k = 0
 862         for a in addresses[::-1]:
 863             if self.history.get(a):break
 864             k = k + 1
 865         return k
 866
 867     def min_acceptable_gap(self):
 868         # fixme: this assumes wallet is synchronized
 869         n = 0
 870         nmax = 0
 871
 872         for account in self.accounts.values():
 873             addresses = account.get_addresses(0)
 874             k = self.num_unused_trailing_addresses(addresses)
 875             for a in addresses[0:-k]:
 876                 if self.history.get(a):
 877                     n = 0
 878                 else:
 879                     n += 1
 880                     if n > nmax: nmax = n
 881         return nmax + 1
 882
 883
 884     def address_is_old(self, address):
 885         age = -1
 886         h = self.history.get(address, [])
 887         if h == ['*']:
 888             return True
 889         for tx_hash, tx_height in h:
 890             if tx_height == 0:
 891                 tx_age = 0
 892             else:
 893                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 894             if tx_age > age:
 895                 age = tx_age
 896         return age > 2
 897
 898
 899     def synchronize_sequence(self, account, for_change):
 900         limit = self.gap_limit_for_change if for_change else self.gap_limit
 901         new_addresses = []
 902         while True:
 903             addresses = account.get_addresses(for_change)
 904             if len(addresses) < limit:
 905                 address = account.create_new_address(for_change)
 906                 self.history[address] = []
 907                 new_addresses.append( address )
 908                 continue
 909
 910             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 911                 break
 912             else:
 913                 address = account.create_new_address(for_change)
 914                 self.history[address] = []
 915                 new_addresses.append( address )
 916
 917         return new_addresses
 918
 919
 920
 921     def create_pending_accounts(self):
 922         for account_type in ['1','2of2','2of3']:
 923             if not self.has_master_public_keys(account_type):
 924                 continue
 925             k, a = self.new_account_address(account_type)
 926             if self.address_is_old(a):
 927                 print_error( "creating account", a )
 928                 self.create_account(account_type)
 929                 self.next_addresses.pop(k)
 930
 931
 932     def synchronize_account(self, account):
 933         new = []
 934         new += self.synchronize_sequence(account, 0)
 935         new += self.synchronize_sequence(account, 1)
 936         return new
 937
 938
 939     def synchronize(self):
 940         if self.master_public_keys:
 941             self.create_pending_accounts()
 942         new = []
 943         for account in self.accounts.values():
 944             new += self.synchronize_account(account)
 945         if new:
 946             self.save_accounts()
 947             self.storage.put('addr_history', self.history, True)
 948         return new
 949
 950
 951     def is_found(self):
 952         return self.history.values() != [[]] * len(self.history)
 953
 954
 955     def add_contact(self, address, label=None):
 956         self.addressbook.append(address)
 957         self.storage.put('contacts', self.addressbook, True)
 958         if label:
 959             self.set_label(address, label)
 960
 961
 962     def delete_contact(self, addr):
 963         if addr in self.addressbook:
 964             self.addressbook.remove(addr)
 965             self.storage.put('addressbook', self.addressbook, True)
 966
 967
 968     def fill_addressbook(self):
 969         for tx_hash, tx in self.transactions.items():
 970             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 971             if is_send:
 972                 for addr, v in tx.outputs:
 973                     if not self.is_mine(addr) and addr not in self.addressbook:
 974                         self.addressbook.append(addr)
 975         # redo labels
 976         # self.update_tx_labels()
 977
 978     def get_num_tx(self, address):
 979         n = 0
 980         for tx in self.transactions.values():
 981             if address in map(lambda x:x[0], tx.outputs): n += 1
 982         return n
 983
 984
 985     def get_address_flags(self, addr):
 986         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 987         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 988         return flags
 989
 990
 991     def get_tx_value(self, tx, account=None):
 992         domain = self.get_account_addresses(account)
 993         return tx.get_value(domain, self.prevout_values)
 994
 995
 996     def update_tx_outputs(self, tx_hash):
 997         tx = self.transactions.get(tx_hash)
 998
 999         for i, (addr, value) in enumerate(tx.outputs):
1000             key = tx_hash+ ':%d'%i
1001             self.prevout_values[key] = value
1002
1003         for item in tx.inputs:
1004             if self.is_mine(item.get('address')):
1005                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1006                 self.spent_outputs.append(key)
1007
1008
1009     def get_addr_balance(self, address):
1010         assert self.is_mine(address)
1011         h = self.history.get(address,[])
1012         if h == ['*']: return 0,0
1013         c = u = 0
1014         received_coins = []   # list of coins received at address
1015
1016         for tx_hash, tx_height in h:
1017             tx = self.transactions.get(tx_hash)
1018             if not tx: continue
1019
1020             for i, (addr, value) in enumerate(tx.outputs):
1021                 if addr == address:
1022                     key = tx_hash + ':%d'%i
1023                     received_coins.append(key)
1024
1025         for tx_hash, tx_height in h:
1026             tx = self.transactions.get(tx_hash)
1027             if not tx: continue
1028             v = 0
1029
1030             for item in tx.inputs:
1031                 addr = item.get('address')
1032                 if addr == address:
1033                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1034                     value = self.prevout_values.get( key )
1035                     if key in received_coins:
1036                         v -= value
1037
1038             for i, (addr, value) in enumerate(tx.outputs):
1039                 key = tx_hash + ':%d'%i
1040                 if addr == address:
1041                     v += value
1042
1043             if tx_height:
1044                 c += v
1045             else:
1046                 u += v
1047         return c, u
1048
1049
1050     def get_account_name(self, k):
1051         if k == 0:
1052             if self.seed_version == 4:
1053                 name = 'Main account'
1054             else:
1055                 name = 'Old account'
1056         else:
1057             default = "Unnamed account"
1058             m = re.match("m/0'/(\d+)", k)
1059             if m:
1060                 num = m.group(1)
1061                 if num == '0':
1062                     default = "Main account"
1063                 else:
1064                     default = "Account %s"%num
1065
1066             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1067             if m:
1068                 num = m.group(1)
1069                 default = "2of2 account %s"%num
1070             name = self.labels.get(k, default)
1071
1072         return name
1073
1074     def get_account_names(self):
1075         accounts = {}
1076         for k, account in self.accounts.items():
1077             accounts[k] = self.get_account_name(k)
1078         if self.imported_keys:
1079             accounts[-1] = 'Imported keys'
1080         return accounts
1081
1082     def get_account_addresses(self, a, include_change=True):
1083         if a is None:
1084             o = self.addresses(True)
1085         elif a == -1:
1086             o = self.imported_keys.keys()
1087         else:
1088             ac = self.accounts[a]
1089             o = ac.get_addresses(0)
1090             if include_change: o += ac.get_addresses(1)
1091         return o
1092
1093     def get_imported_balance(self):
1094         return self.get_balance(self.imported_keys.keys())
1095
1096     def get_account_balance(self, account):
1097         return self.get_balance(self.get_account_addresses(account))
1098
1099     def get_frozen_balance(self):
1100         return self.get_balance(self.frozen_addresses)
1101
1102     def get_balance(self, domain=None):
1103         if domain is None: domain = self.addresses(True)
1104         cc = uu = 0
1105         for addr in domain:
1106             c, u = self.get_addr_balance(addr)
1107             cc += c
1108             uu += u
1109         return cc, uu
1110
1111
1112     def get_unspent_coins(self, domain=None):
1113         coins = []
1114         if domain is None: domain = self.addresses(True)
1115         for addr in domain:
1116             h = self.history.get(addr, [])
1117             if h == ['*']: continue
1118             for tx_hash, tx_height in h:
1119                 tx = self.transactions.get(tx_hash)
1120                 if tx is None: raise Exception("Wallet not synchronized")
1121                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1122                 for o in tx.d.get('outputs'):
1123                     output = o.copy()
1124                     if output.get('address') != addr: continue
1125                     key = tx_hash + ":%d" % output.get('prevout_n')
1126                     if key in self.spent_outputs: continue
1127                     output['prevout_hash'] = tx_hash
1128                     output['height'] = tx_height
1129                     output['coinbase'] = is_coinbase
1130                     coins.append((tx_height, output))
1131
1132         # sort by age
1133         if coins:
1134             coins = sorted(coins)
1135             if coins[-1][0] != 0:
1136                 while coins[0][0] == 0:
1137                     coins = coins[1:] + [ coins[0] ]
1138         return [x[1] for x in coins]
1139
1140
1141
1142     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1143         domain = self.get_account_addresses(account) if account else None
1144         return self.choose_tx_inputs( amount, fixed_fee, domain )
1145
1146
1147     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1148         """ todo: minimize tx size """
1149         total = 0
1150         fee = self.fee if fixed_fee is None else fixed_fee
1151         if domain is None:
1152             domain = self.addresses(True)
1153
1154         for i in self.frozen_addresses:
1155             if i in domain: domain.remove(i)
1156
1157         prioritized = []
1158         for i in self.prioritized_addresses:
1159             if i in domain:
1160                 domain.remove(i)
1161                 prioritized.append(i)
1162
1163         coins = self.get_unspent_coins(domain)
1164         prioritized_coins = self.get_unspent_coins(prioritized)
1165
1166         inputs = []
1167         coins = prioritized_coins + coins
1168
1169         for item in coins:
1170             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1171                 continue
1172             addr = item.get('address')
1173             v = item.get('value')
1174             total += v
1175             inputs.append(item)
1176             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1177             if total >= amount + fee: break
1178         else:
1179             inputs = []
1180
1181         return inputs, total, fee
1182
1183
1184     def set_fee(self, fee):
1185         if self.fee != fee:
1186             self.fee = fee
1187             self.storage.put('fee_per_kb', self.fee, True)
1188
1189     def estimated_fee(self, inputs):
1190         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1191         fee = self.fee * int(round(estimated_size/1024.))
1192         if fee == 0: fee = self.fee
1193         return fee
1194
1195
1196     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1197         "add change to a transaction"
1198         change_amount = total - ( amount + fee )
1199         if change_amount > DUST_THRESHOLD:
1200             if not change_addr:
1201
1202                 # send change to one of the accounts involved in the tx
1203                 address = inputs[0].get('address')
1204                 account, _ = self.get_address_index(address)
1205
1206                 if not self.use_change or account == -1:
1207                     change_addr = inputs[-1]['address']
1208                 else:
1209                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1210
1211             # Insert the change output at a random position in the outputs
1212             posn = random.randint(0, len(outputs))
1213             outputs[posn:posn] = [( change_addr,  change_amount)]
1214         return outputs
1215
1216
1217     def get_history(self, address):
1218         with self.lock:
1219             return self.history.get(address)
1220
1221
1222     def get_status(self, h):
1223         if not h: return None
1224         if h == ['*']: return '*'
1225         status = ''
1226         for tx_hash, height in h:
1227             status += tx_hash + ':%d:' % height
1228         return hashlib.sha256( status ).digest().encode('hex')
1229
1230
1231     def receive_tx_callback(self, tx_hash, tx, tx_height):
1232
1233         with self.transaction_lock:
1234             self.add_extra_addresses(tx)
1235             if not self.check_new_tx(tx_hash, tx):
1236                 # may happen due to pruning
1237                 print_error("received transaction that is no longer referenced in history", tx_hash)
1238                 return
1239             self.transactions[tx_hash] = tx
1240             self.network.interface.pending_transactions_for_notifications.append(tx)
1241             self.save_transactions()
1242             if self.verifier and tx_height>0:
1243                 self.verifier.add(tx_hash, tx_height)
1244             self.update_tx_outputs(tx_hash)
1245
1246
1247     def save_transactions(self):
1248         tx = {}
1249         for k,v in self.transactions.items():
1250             tx[k] = str(v)
1251         self.storage.put('transactions', tx, True)
1252
1253     def receive_history_callback(self, addr, hist):
1254
1255         if not self.check_new_history(addr, hist):
1256             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1257
1258         with self.lock:
1259             self.history[addr] = hist
1260             self.storage.put('addr_history', self.history, True)
1261
1262         if hist != ['*']:
1263             for tx_hash, tx_height in hist:
1264                 if tx_height>0:
1265                     # add it in case it was previously unconfirmed
1266                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1267
1268
1269     def get_tx_history(self, account=None):
1270         if not self.verifier:
1271             return []
1272
1273         with self.transaction_lock:
1274             history = self.transactions.items()
1275             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1276             result = []
1277
1278             balance = 0
1279             for tx_hash, tx in history:
1280                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1281                 if v is not None: balance += v
1282
1283             c, u = self.get_account_balance(account)
1284
1285             if balance != c+u:
1286                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1287
1288             balance = c + u - balance
1289             for tx_hash, tx in history:
1290                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1291                 if not is_relevant:
1292                     continue
1293                 if value is not None:
1294                     balance += value
1295
1296                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1297                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1298
1299         return result
1300
1301
1302     def get_label(self, tx_hash):
1303         label = self.labels.get(tx_hash)
1304         is_default = (label == '') or (label is None)
1305         if is_default: label = self.get_default_label(tx_hash)
1306         return label, is_default
1307
1308
1309     def get_default_label(self, tx_hash):
1310         tx = self.transactions.get(tx_hash)
1311         default_label = ''
1312         if tx:
1313             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1314             if is_mine:
1315                 for o in tx.outputs:
1316                     o_addr, _ = o
1317                     if not self.is_mine(o_addr):
1318                         try:
1319                             default_label = self.labels[o_addr]
1320                         except KeyError:
1321                             default_label = o_addr
1322                         break
1323                 else:
1324                     default_label = '(internal)'
1325             else:
1326                 for o in tx.outputs:
1327                     o_addr, _ = o
1328                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1329                         break
1330                 else:
1331                     for o in tx.outputs:
1332                         o_addr, _ = o
1333                         if self.is_mine(o_addr):
1334                             break
1335                     else:
1336                         o_addr = None
1337
1338                 if o_addr:
1339                     dest_label = self.labels.get(o_addr)
1340                     try:
1341                         default_label = self.labels[o_addr]
1342                     except KeyError:
1343                         default_label = o_addr
1344
1345         return default_label
1346
1347
1348     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1349         for address, x in outputs:
1350             assert is_valid(address)
1351         amount = sum( map(lambda x:x[1], outputs) )
1352         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1353         if not inputs:
1354             raise ValueError("Not enough funds")
1355         self.add_input_info(inputs)
1356         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1357         return Transaction.from_io(inputs, outputs)
1358
1359
1360     def mktx_from_account(self, outputs, password, fee=None, account=None):
1361         domain = self.get_account_addresses(account) if account else None
1362         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1363
1364
1365     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1366         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1367         keypairs = {}
1368         self.add_keypairs_from_wallet(tx, keypairs, password)
1369         if keypairs:
1370             self.sign_transaction(tx, keypairs)
1371         return tx
1372
1373
1374     def add_input_info(self, inputs):
1375         for txin in inputs:
1376             address = txin['address']
1377             if address in self.imported_keys.keys():
1378                 continue
1379             account, sequence = self.get_address_index(address)
1380             txin['KeyID'] = self.get_keyID(account, sequence)
1381             redeemScript = self.accounts[account].redeem_script(sequence)
1382             if redeemScript:
1383                 txin['redeemScript'] = redeemScript
1384             else:
1385                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1386
1387
1388     def sign_transaction(self, tx, keypairs):
1389         tx.sign(keypairs)
1390         run_hook('sign_transaction', tx)
1391
1392
1393     def sendtx(self, tx):
1394         # synchronous
1395         h = self.send_tx(tx)
1396         self.tx_event.wait()
1397         return self.receive_tx(h)
1398
1399     def send_tx(self, tx):
1400         # asynchronous
1401         self.tx_event.clear()
1402         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1403         return tx.hash()
1404
1405     def on_broadcast(self, i, r):
1406         self.tx_result = r.get('result')
1407         self.tx_event.set()
1408
1409     def receive_tx(self,tx_hash):
1410         out = self.tx_result
1411         if out != tx_hash:
1412             return False, "error: " + out
1413         return True, out
1414
1415
1416
1417     def update_password(self, old_password, new_password):
1418         if new_password == '': new_password = None
1419         decoded = self.get_seed(old_password)
1420         self.seed = pw_encode( decoded, new_password)
1421         self.storage.put('seed', self.seed, True)
1422         self.use_encryption = (new_password != None)
1423         self.storage.put('use_encryption', self.use_encryption,True)
1424         for k in self.imported_keys.keys():
1425             a = self.imported_keys[k]
1426             b = pw_decode(a, old_password)
1427             c = pw_encode(b, new_password)
1428             self.imported_keys[k] = c
1429         self.storage.put('imported_keys', self.imported_keys, True)
1430
1431         for k, v in self.master_private_keys.items():
1432             b = pw_decode(v, old_password)
1433             c = pw_encode(b, new_password)
1434             self.master_private_keys[k] = c
1435         self.storage.put('master_private_keys', self.master_private_keys, True)
1436
1437
1438     def freeze(self,addr):
1439         if self.is_mine(addr) and addr not in self.frozen_addresses:
1440             self.unprioritize(addr)
1441             self.frozen_addresses.append(addr)
1442             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1443             return True
1444         else:
1445             return False
1446
1447     def unfreeze(self,addr):
1448         if self.is_mine(addr) and addr in self.frozen_addresses:
1449             self.frozen_addresses.remove(addr)
1450             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1451             return True
1452         else:
1453             return False
1454
1455     def prioritize(self,addr):
1456         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1457             self.unfreeze(addr)
1458             self.prioritized_addresses.append(addr)
1459             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1460             return True
1461         else:
1462             return False
1463
1464     def unprioritize(self,addr):
1465         if self.is_mine(addr) and addr in self.prioritized_addresses:
1466             self.prioritized_addresses.remove(addr)
1467             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1468             return True
1469         else:
1470             return False
1471
1472
1473     def set_verifier(self, verifier):
1474         self.verifier = verifier
1475
1476         # review transactions that are in the history
1477         for addr, hist in self.history.items():
1478             if hist == ['*']: continue
1479             for tx_hash, tx_height in hist:
1480                 if tx_height>0:
1481                     # add it in case it was previously unconfirmed
1482                     self.verifier.add(tx_hash, tx_height)
1483
1484
1485         # if we are on a pruning server, remove unverified transactions
1486         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1487         for tx_hash in self.transactions.keys():
1488             if tx_hash not in vr:
1489                 self.transactions.pop(tx_hash)
1490
1491
1492
1493     def check_new_history(self, addr, hist):
1494
1495         # check that all tx in hist are relevant
1496         if hist != ['*']:
1497             for tx_hash, height in hist:
1498                 tx = self.transactions.get(tx_hash)
1499                 if not tx: continue
1500                 if not tx.has_address(addr):
1501                     return False
1502
1503         # check that we are not "orphaning" a transaction
1504         old_hist = self.history.get(addr,[])
1505         if old_hist == ['*']: return True
1506
1507         for tx_hash, height in old_hist:
1508             if tx_hash in map(lambda x:x[0], hist): continue
1509             found = False
1510             for _addr, _hist in self.history.items():
1511                 if _addr == addr: continue
1512                 if _hist == ['*']: continue
1513                 _tx_hist = map(lambda x:x[0], _hist)
1514                 if tx_hash in _tx_hist:
1515                     found = True
1516                     break
1517
1518             if not found:
1519                 tx = self.transactions.get(tx_hash)
1520                 # tx might not be there
1521                 if not tx: continue
1522
1523                 # already verified?
1524                 if self.verifier.get_height(tx_hash):
1525                     continue
1526                 # unconfirmed tx
1527                 print_error("new history is orphaning transaction:", tx_hash)
1528                 # check that all outputs are not mine, request histories
1529                 ext_requests = []
1530                 for _addr, _v in tx.outputs:
1531                     # assert not self.is_mine(_addr)
1532                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1533
1534                 ext_h = self.network.synchronous_get(ext_requests)
1535                 print_error("sync:", ext_requests, ext_h)
1536                 height = None
1537                 for h in ext_h:
1538                     if h == ['*']: continue
1539                     for item in h:
1540                         if item.get('tx_hash') == tx_hash:
1541                             height = item.get('height')
1542                 if height:
1543                     print_error("found height for", tx_hash, height)
1544                     self.verifier.add(tx_hash, height)
1545                 else:
1546                     print_error("removing orphaned tx from history", tx_hash)
1547                     self.transactions.pop(tx_hash)
1548
1549         return True
1550
1551
1552
1553     def check_new_tx(self, tx_hash, tx):
1554         # 1 check that tx is referenced in addr_history.
1555         addresses = []
1556         for addr, hist in self.history.items():
1557             if hist == ['*']:continue
1558             for txh, height in hist:
1559                 if txh == tx_hash:
1560                     addresses.append(addr)
1561
1562         if not addresses:
1563             return False
1564
1565         # 2 check that referencing addresses are in the tx
1566         for addr in addresses:
1567             if not tx.has_address(addr):
1568                 return False
1569
1570         return True
1571
1572
1573     def start_threads(self, network):
1574         from verifier import TxVerifier
1575         self.network = network
1576         if self.network:
1577             self.verifier = TxVerifier(self.network, self.storage)
1578             self.verifier.start()
1579             self.set_verifier(self.verifier)
1580             self.synchronizer = WalletSynchronizer(self, network)
1581             self.synchronizer.start()
1582         else:
1583             self.verifier = None
1584             self.synchronizer =None
1585
1586     def stop_threads(self):
1587         if self.network:
1588             self.verifier.stop()
1589             self.synchronizer.stop()
1590
1591
1592     def restore(self, callback):
1593         from i18n import _
1594         def wait_for_wallet():
1595             self.set_up_to_date(False)
1596             while not self.is_up_to_date():
1597                 msg = "%s\n%s %d\n%s %.1f"%(
1598                     _("Please wait..."),
1599                     _("Addresses generated:"),
1600                     len(self.addresses(True)),
1601                     _("Kilobytes received:"),
1602                     self.network.interface.bytes_received/1024.)
1603
1604                 apply(callback, (msg,))
1605                 time.sleep(0.1)
1606
1607         def wait_for_network():
1608             while not self.network.is_connected():
1609                 msg = "%s \n" % (_("Connecting..."))
1610                 apply(callback, (msg,))
1611                 time.sleep(0.1)
1612
1613         # wait until we are connected, because the user might have selected another server
1614         if self.network:
1615             wait_for_network()
1616             wait_for_wallet()
1617         else:
1618             self.synchronize()
1619
1620         self.fill_addressbook()
1621
1622
1623
1624
1625 class WalletSynchronizer(threading.Thread):
1626
1627
1628     def __init__(self, wallet, network):
1629         threading.Thread.__init__(self)
1630         self.daemon = True
1631         self.wallet = wallet
1632         self.network = network
1633         self.was_updated = True
1634         self.running = False
1635         self.lock = threading.Lock()
1636         self.queue = Queue.Queue()
1637
1638     def stop(self):
1639         with self.lock: self.running = False
1640
1641     def is_running(self):
1642         with self.lock: return self.running
1643
1644
1645     def subscribe_to_addresses(self, addresses):
1646         messages = []
1647         for addr in addresses:
1648             messages.append(('blockchain.address.subscribe', [addr]))
1649         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1650
1651
1652     def run(self):
1653         with self.lock:
1654             self.running = True
1655
1656         while self.is_running():
1657
1658             if not self.network.is_connected():
1659                 print_error("synchronizer: waiting for interface")
1660                 self.network.wait_until_connected()
1661
1662             self.run_interface(self.network.interface)
1663
1664
1665     def run_interface(self, interface):
1666
1667         print_error("synchronizer: connected to", interface.server)
1668
1669         requested_tx = []
1670         missing_tx = []
1671         requested_histories = {}
1672
1673         # request any missing transactions
1674         for history in self.wallet.history.values():
1675             if history == ['*']: continue
1676             for tx_hash, tx_height in history:
1677                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1678                     missing_tx.append( (tx_hash, tx_height) )
1679
1680         if missing_tx:
1681             print_error("missing tx", missing_tx)
1682
1683         # subscriptions
1684         self.subscribe_to_addresses(self.wallet.addresses(True))
1685
1686         while self.is_running():
1687             # 1. create new addresses
1688             new_addresses = self.wallet.synchronize()
1689
1690             # request missing addresses
1691             if new_addresses:
1692                 self.subscribe_to_addresses(new_addresses)
1693
1694             # request missing transactions
1695             for tx_hash, tx_height in missing_tx:
1696                 if (tx_hash, tx_height) not in requested_tx:
1697                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1698                     requested_tx.append( (tx_hash, tx_height) )
1699             missing_tx = []
1700
1701             # detect if situation has changed
1702             if interface.is_up_to_date() and self.queue.empty():
1703                 if not self.wallet.is_up_to_date():
1704                     self.wallet.set_up_to_date(True)
1705                     self.was_updated = True
1706             else:
1707                 if self.wallet.is_up_to_date():
1708                     self.wallet.set_up_to_date(False)
1709                     self.was_updated = True
1710
1711             if self.was_updated:
1712                 self.wallet.network.trigger_callback('updated')
1713                 self.was_updated = False
1714
1715             # 2. get a response
1716             try:
1717                 r = self.queue.get(block=True, timeout=1)
1718             except Queue.Empty:
1719                 continue
1720
1721             if interface != self.network.interface:
1722                 break
1723
1724             if not r:
1725                 continue
1726
1727             # 3. handle response
1728             method = r['method']
1729             params = r['params']
1730             result = r.get('result')
1731             error = r.get('error')
1732             if error:
1733                 print "error", r
1734                 continue
1735
1736             if method == 'blockchain.address.subscribe':
1737                 addr = params[0]
1738                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1739                     if requested_histories.get(addr) is None:
1740                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1741                         requested_histories[addr] = result
1742
1743             elif method == 'blockchain.address.get_history':
1744                 addr = params[0]
1745                 print_error("receiving history", addr, result)
1746                 if result == ['*']:
1747                     assert requested_histories.pop(addr) == '*'
1748                     self.wallet.receive_history_callback(addr, result)
1749                 else:
1750                     hist = []
1751                     # check that txids are unique
1752                     txids = []
1753                     for item in result:
1754                         tx_hash = item['tx_hash']
1755                         if tx_hash not in txids:
1756                             txids.append(tx_hash)
1757                             hist.append( (tx_hash, item['height']) )
1758
1759                     if len(hist) != len(result):
1760                         raise Exception("error: server sent history with non-unique txid", result)
1761
1762                     # check that the status corresponds to what was announced
1763                     rs = requested_histories.pop(addr)
1764                     if self.wallet.get_status(hist) != rs:
1765                         raise Exception("error: status mismatch: %s"%addr)
1766
1767                     # store received history
1768                     self.wallet.receive_history_callback(addr, hist)
1769
1770                     # request transactions that we don't have
1771                     for tx_hash, tx_height in hist:
1772                         if self.wallet.transactions.get(tx_hash) is None:
1773                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1774                                 missing_tx.append( (tx_hash, tx_height) )
1775
1776             elif method == 'blockchain.transaction.get':
1777                 tx_hash = params[0]
1778                 tx_height = params[1]
1779                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1780                 tx = Transaction(result)
1781                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1782                 self.was_updated = True
1783                 requested_tx.remove( (tx_hash, tx_height) )
1784                 print_error("received tx:", tx_hash, len(tx.raw))
1785
1786             else:
1787                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1788
1789             if self.was_updated and not requested_tx:
1790                 self.wallet.network.trigger_callback('updated')
1791                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1792
1793                 self.was_updated = False