lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40
  41 # AES encryption
  42 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  43 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  44
  45 def pw_encode(s, password):
  46     if password:
  47         secret = Hash(password)
  48         return EncodeAES(secret, s)
  49     else:
  50         return s
  51
  52 def pw_decode(s, password):
  53     if password is not None:
  54         secret = Hash(password)
  55         try:
  56             d = DecodeAES(secret, s)
  57         except:
  58             raise BaseException('Invalid password')
  59         return d
  60     else:
  61         return s
  62
  63
  64
  65
  66
  67 from version import *
  68
  69
  70 class WalletStorage:
  71
  72     def __init__(self, config):
  73         self.lock = threading.Lock()
  74         self.data = {}
  75         self.file_exists = False
  76         self.path = self.init_path(config)
  77         print_error( "wallet path", self.path )
  78         if self.path:
  79             self.read(self.path)
  80
  81
  82     def init_path(self, config):
  83         """Set the path of the wallet."""
  84
  85         # command line -w option
  86         path = config.get('wallet_path')
  87         if path:
  88             return path
  89
  90         # path in config file
  91         path = config.get('default_wallet_path')
  92         if path:
  93             return path
  94
  95         # default path
  96         dirpath = os.path.join(config.path, "wallets")
  97         if not os.path.exists(dirpath):
  98             os.mkdir(dirpath)
  99
 100         new_path = os.path.join(config.path, "wallets", "default_wallet")
 101
 102         # default path in pre 1.9 versions
 103         old_path = os.path.join(config.path, "electrum.dat")
 104         if os.path.exists(old_path) and not os.path.exists(new_path):
 105             os.rename(old_path, new_path)
 106
 107         return new_path
 108
 109
 110     def read(self, path):
 111         """Read the contents of the wallet file."""
 112         try:
 113             with open(self.path, "r") as f:
 114                 data = f.read()
 115         except IOError:
 116             return
 117         try:
 118             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 119         except:
 120             raise IOError("Cannot read wallet file.")
 121
 122         self.data = d
 123         self.file_exists = True
 124
 125
 126     def get(self, key, default=None):
 127         return self.data.get(key, default)
 128
 129     def put(self, key, value, save = True):
 130
 131         with self.lock:
 132             if value is not None:
 133                 self.data[key] = value
 134             else:
 135                 self.data.pop[key]
 136             if save:
 137                 self.write()
 138
 139     def write(self):
 140         s = repr(self.data)
 141         f = open(self.path,"w")
 142         f.write( s )
 143         f.close()
 144         if 'ANDROID_DATA' not in os.environ:
 145             import stat
 146             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 147
 148
 149 class Wallet:
 150
 151     def __init__(self, storage):
 152
 153         self.storage = storage
 154         self.electrum_version = ELECTRUM_VERSION
 155         self.gap_limit_for_change = 3 # constant
 156
 157         # saved fields
 158         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 159
 160         self.gap_limit             = storage.get('gap_limit', 5)
 161         self.use_change            = storage.get('use_change',True)
 162         self.use_encryption        = storage.get('use_encryption', False)
 163         self.seed                  = storage.get('seed', '')               # encrypted
 164         self.labels                = storage.get('labels', {})
 165         self.frozen_addresses      = storage.get('frozen_addresses',[])
 166         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 167         self.addressbook           = storage.get('contacts', [])
 168
 169         self.imported_keys         = storage.get('imported_keys',{})
 170         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 171
 172         self.fee                   = int(storage.get('fee_per_kb',20000))
 173
 174         self.master_public_keys = storage.get('master_public_keys',{})
 175         self.master_private_keys = storage.get('master_private_keys', {})
 176
 177         self.next_addresses = storage.get('next_addresses',{})
 178
 179         if self.seed_version not in [4, 6]:
 180             msg = "This wallet seed is not supported."
 181             if self.seed_version in [5]:
 182                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 183                 print msg
 184                 sys.exit(1)
 185
 186
 187         self.load_accounts()
 188
 189         self.transactions = {}
 190         tx_list = self.storage.get('transactions',{})
 191         for k,v in tx_list.items():
 192             try:
 193                 tx = Transaction(v)
 194             except:
 195                 print_msg("Warning: Cannot deserialize transactions. skipping")
 196                 continue
 197
 198             self.add_extra_addresses(tx)
 199             self.transactions[k] = tx
 200
 201         for h,tx in self.transactions.items():
 202             if not self.check_new_tx(h, tx):
 203                 print_error("removing unreferenced tx", h)
 204                 self.transactions.pop(h)
 205
 206
 207         # not saved
 208         self.prevout_values = {}     # my own transaction outputs
 209         self.spent_outputs = []
 210
 211         # spv
 212         self.verifier = None
 213
 214         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 215         # interface.is_up_to_date() returns true when all requests have been answered and processed
 216         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 217
 218         self.up_to_date = False
 219         self.lock = threading.Lock()
 220         self.transaction_lock = threading.Lock()
 221         self.tx_event = threading.Event()
 222
 223         for tx_hash, tx in self.transactions.items():
 224             self.update_tx_outputs(tx_hash)
 225
 226
 227     def add_extra_addresses(self, tx):
 228         h = tx.hash()
 229         # find the address corresponding to pay-to-pubkey inputs
 230         tx.add_extra_addresses(self.transactions)
 231         for o in tx.d.get('outputs'):
 232             if o.get('is_pubkey'):
 233                 for tx2 in self.transactions.values():
 234                     tx2.add_extra_addresses({h:tx})
 235
 236
 237
 238
 239     def set_up_to_date(self,b):
 240         with self.lock: self.up_to_date = b
 241
 242
 243     def is_up_to_date(self):
 244         with self.lock: return self.up_to_date
 245
 246
 247     def update(self):
 248         self.up_to_date = False
 249         while not self.is_up_to_date():
 250             time.sleep(0.1)
 251
 252
 253     def import_key(self, sec, password):
 254         # check password
 255         seed = self.get_seed(password)
 256         try:
 257             address = address_from_private_key(sec)
 258         except:
 259             raise BaseException('Invalid private key')
 260
 261         if self.is_mine(address):
 262             raise BaseException('Address already in wallet')
 263
 264         # store the originally requested keypair into the imported keys table
 265         self.imported_keys[address] = pw_encode(sec, password )
 266         self.storage.put('imported_keys', self.imported_keys, True)
 267         if self.synchronizer:
 268             self.synchronizer.subscribe_to_addresses([address])
 269         return address
 270
 271     def delete_imported_key(self, addr):
 272         if addr in self.imported_keys:
 273             self.imported_keys.pop(addr)
 274             self.storage.put('imported_keys', self.imported_keys, True)
 275
 276
 277     def make_seed(self):
 278         import mnemonic, ecdsa
 279         entropy = ecdsa.util.randrange( pow(2,160) )
 280         nonce = 0
 281         while True:
 282             ss = "%040x"%(entropy+nonce)
 283             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 284             # we keep only 13 words, that's approximately 139 bits of entropy
 285             words = mnemonic.mn_encode(s)[0:13]
 286             seed = ' '.join(words)
 287             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 288                 break  # this removes 12 bits of entropy
 289             nonce += 1
 290
 291         return seed
 292
 293
 294     def init_seed(self, seed):
 295         import mnemonic
 296
 297         if self.seed:
 298             raise BaseException("a seed exists")
 299
 300         if not seed:
 301             self.seed = random_seed(128)
 302             self.seed_version = 4
 303             return
 304
 305         #if not seed:
 306         #    self.seed = self.make_seed()
 307         #    self.seed_version = SEED_VERSION
 308         #    return
 309
 310         # find out what kind of wallet we are
 311         try:
 312             seed.decode('hex')
 313             self.seed_version = 4
 314             self.seed = str(seed)
 315             return
 316         except:
 317             pass
 318
 319         words = seed.split()
 320         self.seed_version = 4
 321         self.seed = mnemonic.mn_decode(words)
 322
 323         #try:
 324         #    mnemonic.mn_decode(words)
 325         #    uses_electrum_words = True
 326         #except:
 327         #    uses_electrum_words = False
 328         #
 329         #if uses_electrum_words and len(words) != 13:
 330         #    self.seed_version = 4
 331         #    self.seed = mnemonic.mn_decode(words)
 332         #else:
 333         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 334         #    self.seed_version = SEED_VERSION
 335         #    self.seed = seed
 336
 337
 338     def save_seed(self):
 339         self.storage.put('seed', self.seed, True)
 340         self.storage.put('seed_version', self.seed_version, True)
 341
 342     def create_watching_only_wallet(self, params):
 343         K0, c0 = params
 344         if not K0:
 345             return
 346
 347         if not c0:
 348             self.seed_version = 4
 349             self.create_old_account(K0)
 350             return
 351
 352         cK0 = ""
 353         self.master_public_keys = {
 354             "m/0'/": (c0, K0, cK0),
 355             }
 356         self.storage.put('master_public_keys', self.master_public_keys, True)
 357         self.create_account('1','Main account')
 358
 359
 360     def create_accounts(self):
 361         if self.seed_version == 4:
 362             mpk = OldAccount.mpk_from_seed(self.seed)
 363             self.create_old_account(mpk)
 364         else:
 365             # create default account
 366             self.create_master_keys('1')
 367             self.create_account('1','Main account')
 368
 369
 370     def create_master_keys(self, account_type):
 371         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 372         if account_type == '1':
 373             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 374             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 375             self.master_private_keys["m/0'/"] = k0
 376         elif account_type == '2of2':
 377             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 378             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 379             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 380             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 381             self.master_private_keys["m/1'/"] = k1
 382             self.master_private_keys["m/2'/"] = k2
 383         elif account_type == '2of3':
 384             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 385             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 386             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 387             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 388             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 389             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 390             self.master_private_keys["m/3'/"] = k3
 391             self.master_private_keys["m/4'/"] = k4
 392             self.master_private_keys["m/5'/"] = k5
 393
 394         self.storage.put('master_public_keys', self.master_public_keys, True)
 395         self.storage.put('master_private_keys', self.master_private_keys, True)
 396
 397     def has_master_public_keys(self, account_type):
 398         if account_type == '1':
 399             return "m/0'/" in self.master_public_keys
 400         elif account_type == '2of2':
 401             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 402         elif account_type == '2of3':
 403             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 404
 405     def find_root_by_master_key(self, c, K):
 406         for key, v in self.master_public_keys.items():
 407             if key == "m/":continue
 408             cc, KK, _ = v
 409             if (c == cc) and (K == KK):
 410                 return key
 411
 412     def deseed_root(self, seed, password):
 413         # for safety, we ask the user to enter their seed
 414         assert seed == self.get_seed(password)
 415         self.seed = ''
 416         self.storage.put('seed', '', True)
 417
 418
 419     def deseed_branch(self, k):
 420         # check that parent has no seed
 421         assert self.seed == ''
 422         self.master_private_keys.pop(k)
 423         self.storage.put('master_private_keys', self.master_private_keys, True)
 424
 425     def is_watching_only(self):
 426         return (self.seed == '') and (self.master_private_keys == {})
 427
 428
 429
 430     def account_id(self, account_type, i):
 431         if account_type == '1':
 432             return "m/0'/%d"%i
 433         elif account_type == '2of2':
 434             return "m/1'/%d & m/2'/%d"%(i,i)
 435         elif account_type == '2of3':
 436             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 437         else:
 438             raise BaseException('unknown account type')
 439
 440
 441     def num_accounts(self, account_type):
 442         keys = self.accounts.keys()
 443         i = 0
 444         while True:
 445             account_id = self.account_id(account_type, i)
 446             if account_id not in keys: break
 447             i += 1
 448         return i
 449
 450
 451     def new_account_address(self, account_type = '1'):
 452         i = self.num_accounts(account_type)
 453         k = self.account_id(account_type,i)
 454
 455         addr = self.next_addresses.get(k)
 456         if not addr:
 457             account_id, account = self.next_account(account_type)
 458             addr = account.first_address()
 459             self.next_addresses[k] = addr
 460             self.storage.put('next_addresses',self.next_addresses)
 461
 462         return k, addr
 463
 464
 465     def next_account(self, account_type = '1'):
 466
 467         i = self.num_accounts(account_type)
 468         account_id = self.account_id(account_type,i)
 469
 470         if account_type is '1':
 471             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 472             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 473             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 474
 475         elif account_type == '2of2':
 476             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 477             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 478             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 479             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 480             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 481
 482         elif account_type == '2of3':
 483             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 484             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 485             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 486             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 487             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 488             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 489             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 490
 491         return account_id, account
 492
 493
 494     def set_label(self, name, text = None):
 495         changed = False
 496         old_text = self.labels.get(name)
 497         if text:
 498             if old_text != text:
 499                 self.labels[name] = text
 500                 changed = True
 501         else:
 502             if old_text:
 503                 self.labels.pop(name)
 504                 changed = True
 505
 506         if changed:
 507             self.storage.put('labels', self.labels, True)
 508
 509         run_hook('set_label', name, text, changed)
 510         return changed
 511
 512
 513
 514     def create_account(self, account_type = '1', name = None):
 515         k, account = self.next_account(account_type)
 516         if k in self.pending_accounts:
 517             self.pending_accounts.pop(k)
 518             self.storage.put('pending_accounts', self.pending_accounts)
 519
 520         self.accounts[k] = account
 521         self.save_accounts()
 522         if name:
 523             self.set_label(k, name)
 524
 525
 526     def create_old_account(self, mpk):
 527         self.storage.put('master_public_key', mpk, True)
 528         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 529         self.save_accounts()
 530
 531
 532     def save_accounts(self):
 533         d = {}
 534         for k, v in self.accounts.items():
 535             d[k] = v.dump()
 536         self.storage.put('accounts', d, True)
 537
 538
 539
 540     def load_accounts(self):
 541         d = self.storage.get('accounts', {})
 542         self.accounts = {}
 543         for k, v in d.items():
 544             if k == 0:
 545                 v['mpk'] = self.storage.get('master_public_key')
 546                 self.accounts[k] = OldAccount(v)
 547             elif '&' in k:
 548                 self.accounts[k] = BIP32_Account_2of2(v)
 549             else:
 550                 self.accounts[k] = BIP32_Account(v)
 551
 552         self.pending_accounts = self.storage.get('pending_accounts',{})
 553
 554
 555     def delete_pending_account(self, k):
 556         self.pending_accounts.pop(k)
 557         self.storage.put('pending_accounts', self.pending_accounts)
 558
 559     def account_is_pending(self, k):
 560         return k in self.pending_accounts
 561
 562     def create_pending_account(self, acct_type, name):
 563         k, addr = self.new_account_address(acct_type)
 564         self.set_label(k, name)
 565         self.pending_accounts[k] = addr
 566         self.storage.put('pending_accounts', self.pending_accounts)
 567
 568     def get_pending_accounts(self):
 569         return self.pending_accounts.items()
 570
 571
 572     def addresses(self, include_change = True, _next=True):
 573         o = self.get_account_addresses(-1, include_change)
 574         for a in self.accounts.keys():
 575             o += self.get_account_addresses(a, include_change)
 576
 577         if _next:
 578             for addr in self.next_addresses.values():
 579                 if addr not in o:
 580                     o += [addr]
 581         return o
 582
 583
 584     def is_mine(self, address):
 585         return address in self.addresses(True)
 586
 587
 588     def is_change(self, address):
 589         if not self.is_mine(address): return False
 590         if address in self.imported_keys.keys(): return False
 591         acct, s = self.get_address_index(address)
 592         if s is None: return False
 593         return s[0] == 1
 594
 595     def get_master_public_key(self):
 596         if self.seed_version == 4:
 597             return self.storage.get("master_public_key")
 598         else:
 599             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 600             return repr((c, K))
 601
 602     def get_master_private_key(self, account, password):
 603         master_k = pw_decode( self.master_private_keys[account], password)
 604         master_c, master_K, master_Kc = self.master_public_keys[account]
 605         try:
 606             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 607             assert K.encode('hex') == master_K
 608         except:
 609             raise BaseException("Invalid password")
 610         return master_k
 611
 612
 613     def get_address_index(self, address):
 614         if address in self.imported_keys.keys():
 615             return -1, None
 616
 617         for account in self.accounts.keys():
 618             for for_change in [0,1]:
 619                 addresses = self.accounts[account].get_addresses(for_change)
 620                 for addr in addresses:
 621                     if address == addr:
 622                         return account, (for_change, addresses.index(addr))
 623
 624         for k,v in self.next_addresses.items():
 625             if v == address:
 626                 return k, (0,0)
 627
 628         raise BaseException("Address not found", address)
 629
 630
 631     def get_roots(self, account):
 632         roots = []
 633         for a in account.split('&'):
 634             s = a.strip()
 635             m = re.match("(m/\d+'/)(\d+)", s)
 636             roots.append( m.group(1) )
 637         return roots
 638
 639     def is_seeded(self, account):
 640         if type(account) is int:
 641             return self.seed is not None
 642
 643         for root in self.get_roots(account):
 644             if root not in self.master_private_keys.keys():
 645                 return False
 646         return True
 647
 648     def rebase_sequence(self, account, sequence):
 649         c, i = sequence
 650         dd = []
 651         for a in account.split('&'):
 652             s = a.strip()
 653             m = re.match("(m/\d+'/)(\d+)", s)
 654             root = m.group(1)
 655             num = int(m.group(2))
 656             dd.append( (root, [num,c,i] ) )
 657         return dd
 658
 659
 660     def get_keyID(self, account, sequence):
 661         if account == 0:
 662             a, b = sequence
 663             mpk = self.storage.get('master_public_key')
 664             return 'old(%s,%d,%d)'%(mpk,a,b)
 665
 666         rs = self.rebase_sequence(account, sequence)
 667         dd = []
 668         for root, public_sequence in rs:
 669             c, K, _ = self.master_public_keys[root]
 670             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 671             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 672         return '&'.join(dd)
 673
 674
 675
 676     def get_seed(self, password):
 677         s = pw_decode(self.seed, password)
 678         if self.seed_version == 4:
 679             seed = s
 680         else:
 681             seed = mnemonic_hash(s)
 682         #todo:  #self.sequences[0].check_seed(seed)
 683         return seed
 684
 685
 686     def get_mnemonic(self, password):
 687         import mnemonic
 688         s = pw_decode(self.seed, password)
 689         if self.seed_version == 4:
 690             return ' '.join(mnemonic.mn_encode(s))
 691         else:
 692             return s
 693
 694
 695
 696     def get_private_key(self, address, password):
 697         out = []
 698         if address in self.imported_keys.keys():
 699             out.append( pw_decode( self.imported_keys[address], password ) )
 700         else:
 701             account, sequence = self.get_address_index(address)
 702             if account == 0:
 703                 seed = self.get_seed(password)
 704                 pk = self.accounts[account].get_private_key(seed, sequence)
 705                 out.append(pk)
 706                 return out
 707
 708             # assert address == self.accounts[account].get_address(*sequence)
 709             rs = self.rebase_sequence( account, sequence)
 710             for root, public_sequence in rs:
 711
 712                 if root not in self.master_private_keys.keys(): continue
 713                 master_k = self.get_master_private_key(root, password)
 714                 master_c, _, _ = self.master_public_keys[root]
 715                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 716                 out.append(pk)
 717
 718         return out
 719
 720
 721     def add_keypairs_from_wallet(self, tx, keypairs, password):
 722         for txin in tx.inputs:
 723             address = txin['address']
 724             private_keys = self.get_private_key(address, password)
 725             for sec in private_keys:
 726                 pubkey = public_key_from_private_key(sec)
 727                 keypairs[ pubkey ] = sec
 728                 if address in self.imported_keys.keys():
 729                     txin['redeemPubkey'] = pubkey
 730
 731
 732     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 733         for txin in tx.inputs:
 734             keyid = txin.get('KeyID')
 735             if keyid:
 736
 737                 if self.seed_version==4:
 738                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 739                     if not m: continue
 740                     mpk = m.group(1)
 741                     if mpk != self.storage.get('master_public_key'): continue
 742                     index = int(m.group(2))
 743                     num = int(m.group(3))
 744                     account = self.accounts[0]
 745                     addr = account.get_address(index, num)
 746                     txin['address'] = addr # fixme: side effect
 747                     pk = self.get_private_key(addr, password)
 748                     for sec in pk:
 749                         pubkey = public_key_from_private_key(sec)
 750                         keypairs[pubkey] = sec
 751                     continue
 752
 753
 754                 roots = []
 755                 for s in keyid.split('&'):
 756                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 757                     if not m: continue
 758                     c = m.group(1)
 759                     K = m.group(2)
 760                     sequence = m.group(3)
 761                     root = self.find_root_by_master_key(c,K)
 762                     if not root: continue
 763                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 764                     root = root + '%d'%sequence[0]
 765                     sequence = sequence[1:]
 766                     roots.append((root,sequence))
 767
 768                 account_id = " & ".join( map(lambda x:x[0], roots) )
 769                 account = self.accounts.get(account_id)
 770                 if not account: continue
 771                 addr = account.get_address(*sequence)
 772                 txin['address'] = addr # fixme: side effect
 773                 pk = self.get_private_key(addr, password)
 774                 for sec in pk:
 775                     pubkey = public_key_from_private_key(sec)
 776                     keypairs[pubkey] = sec
 777
 778
 779
 780     def signrawtransaction(self, tx, input_info, private_keys, password):
 781
 782         # check that the password is correct
 783         seed = self.get_seed(password)
 784
 785         # add input info
 786         tx.add_input_info(input_info)
 787
 788         # add redeem script for coins that are in the wallet
 789         # FIXME: add redeemPubkey too!
 790         unspent_coins = self.get_unspent_coins()
 791         for txin in tx.inputs:
 792             for item in unspent_coins:
 793                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 794                     print_error( "tx input is in unspent coins" )
 795                     txin['scriptPubKey'] = item['scriptPubKey']
 796                     account, sequence = self.get_address_index(item['address'])
 797                     if account != -1:
 798                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 799                         print_error("added redeemScript", txin['redeemScript'])
 800                     break
 801
 802
 803         # build a list of public/private keys
 804         keypairs = {}
 805
 806         # add private keys from parameter
 807         for sec in private_keys:
 808             pubkey = public_key_from_private_key(sec)
 809             keypairs[ pubkey ] = sec
 810
 811         # add private_keys from KeyID
 812         self.add_keypairs_from_KeyID(tx, keypairs, password)
 813
 814         # add private keys from wallet
 815         self.add_keypairs_from_wallet(tx, keypairs, password)
 816         self.sign_transaction(tx, keypairs)
 817
 818
 819     def sign_message(self, address, message, password):
 820         keys = self.get_private_key(address, password)
 821         assert len(keys) == 1
 822         sec = keys[0]
 823         key = regenerate_key(sec)
 824         compressed = is_compressed(sec)
 825         return key.sign_message(message, compressed, address)
 826
 827
 828     def change_gap_limit(self, value):
 829         if value >= self.gap_limit:
 830             self.gap_limit = value
 831             self.storage.put('gap_limit', self.gap_limit, True)
 832             #self.interface.poke('synchronizer')
 833             return True
 834
 835         elif value >= self.min_acceptable_gap():
 836             for key, account in self.accounts.items():
 837                 addresses = account[0]
 838                 k = self.num_unused_trailing_addresses(addresses)
 839                 n = len(addresses) - k + value
 840                 addresses = addresses[0:n]
 841                 self.accounts[key][0] = addresses
 842
 843             self.gap_limit = value
 844             self.storage.put('gap_limit', self.gap_limit, True)
 845             self.save_accounts()
 846             return True
 847         else:
 848             return False
 849
 850     def num_unused_trailing_addresses(self, addresses):
 851         k = 0
 852         for a in addresses[::-1]:
 853             if self.history.get(a):break
 854             k = k + 1
 855         return k
 856
 857     def min_acceptable_gap(self):
 858         # fixme: this assumes wallet is synchronized
 859         n = 0
 860         nmax = 0
 861
 862         for account in self.accounts.values():
 863             addresses = account.get_addresses(0)
 864             k = self.num_unused_trailing_addresses(addresses)
 865             for a in addresses[0:-k]:
 866                 if self.history.get(a):
 867                     n = 0
 868                 else:
 869                     n += 1
 870                     if n > nmax: nmax = n
 871         return nmax + 1
 872
 873
 874     def address_is_old(self, address):
 875         age = -1
 876         h = self.history.get(address, [])
 877         if h == ['*']:
 878             return True
 879         for tx_hash, tx_height in h:
 880             if tx_height == 0:
 881                 tx_age = 0
 882             else:
 883                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 884             if tx_age > age:
 885                 age = tx_age
 886         return age > 2
 887
 888
 889     def synchronize_sequence(self, account, for_change):
 890         limit = self.gap_limit_for_change if for_change else self.gap_limit
 891         new_addresses = []
 892         while True:
 893             addresses = account.get_addresses(for_change)
 894             if len(addresses) < limit:
 895                 address = account.create_new_address(for_change)
 896                 self.history[address] = []
 897                 new_addresses.append( address )
 898                 continue
 899
 900             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 901                 break
 902             else:
 903                 address = account.create_new_address(for_change)
 904                 self.history[address] = []
 905                 new_addresses.append( address )
 906
 907         return new_addresses
 908
 909
 910
 911     def create_pending_accounts(self):
 912         for account_type in ['1','2of2','2of3']:
 913             if not self.has_master_public_keys(account_type):
 914                 continue
 915             k, a = self.new_account_address(account_type)
 916             if self.address_is_old(a):
 917                 print_error( "creating account", a )
 918                 self.create_account(account_type)
 919                 self.next_addresses.pop(k)
 920
 921
 922     def synchronize_account(self, account):
 923         new = []
 924         new += self.synchronize_sequence(account, 0)
 925         new += self.synchronize_sequence(account, 1)
 926         return new
 927
 928
 929     def synchronize(self):
 930         if self.master_public_keys:
 931             self.create_pending_accounts()
 932         new = []
 933         for account in self.accounts.values():
 934             new += self.synchronize_account(account)
 935         if new:
 936             self.save_accounts()
 937             self.storage.put('addr_history', self.history, True)
 938         return new
 939
 940
 941     def is_found(self):
 942         return self.history.values() != [[]] * len(self.history)
 943
 944
 945     def add_contact(self, address, label=None):
 946         self.addressbook.append(address)
 947         self.storage.put('contacts', self.addressbook, True)
 948         if label:
 949             self.set_label(address, label)
 950
 951
 952     def delete_contact(self, addr):
 953         if addr in self.addressbook:
 954             self.addressbook.remove(addr)
 955             self.storage.put('addressbook', self.addressbook, True)
 956
 957
 958     def fill_addressbook(self):
 959         for tx_hash, tx in self.transactions.items():
 960             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 961             if is_send:
 962                 for addr, v in tx.outputs:
 963                     if not self.is_mine(addr) and addr not in self.addressbook:
 964                         self.addressbook.append(addr)
 965         # redo labels
 966         # self.update_tx_labels()
 967
 968     def get_num_tx(self, address):
 969         n = 0
 970         for tx in self.transactions.values():
 971             if address in map(lambda x:x[0], tx.outputs): n += 1
 972         return n
 973
 974
 975     def get_address_flags(self, addr):
 976         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 977         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 978         return flags
 979
 980
 981     def get_tx_value(self, tx, account=None):
 982         domain = self.get_account_addresses(account)
 983         return tx.get_value(domain, self.prevout_values)
 984
 985
 986     def update_tx_outputs(self, tx_hash):
 987         tx = self.transactions.get(tx_hash)
 988
 989         for i, (addr, value) in enumerate(tx.outputs):
 990             key = tx_hash+ ':%d'%i
 991             self.prevout_values[key] = value
 992
 993         for item in tx.inputs:
 994             if self.is_mine(item.get('address')):
 995                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 996                 self.spent_outputs.append(key)
 997
 998
 999     def get_addr_balance(self, address):
1000         assert self.is_mine(address)
1001         h = self.history.get(address,[])
1002         if h == ['*']: return 0,0
1003         c = u = 0
1004         received_coins = []   # list of coins received at address
1005
1006         for tx_hash, tx_height in h:
1007             tx = self.transactions.get(tx_hash)
1008             if not tx: continue
1009
1010             for i, (addr, value) in enumerate(tx.outputs):
1011                 if addr == address:
1012                     key = tx_hash + ':%d'%i
1013                     received_coins.append(key)
1014
1015         for tx_hash, tx_height in h:
1016             tx = self.transactions.get(tx_hash)
1017             if not tx: continue
1018             v = 0
1019
1020             for item in tx.inputs:
1021                 addr = item.get('address')
1022                 if addr == address:
1023                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1024                     value = self.prevout_values.get( key )
1025                     if key in received_coins:
1026                         v -= value
1027
1028             for i, (addr, value) in enumerate(tx.outputs):
1029                 key = tx_hash + ':%d'%i
1030                 if addr == address:
1031                     v += value
1032
1033             if tx_height:
1034                 c += v
1035             else:
1036                 u += v
1037         return c, u
1038
1039
1040     def get_account_name(self, k):
1041         if k == 0:
1042             if self.seed_version == 4:
1043                 name = 'Main account'
1044             else:
1045                 name = 'Old account'
1046         else:
1047             default = "Unnamed account"
1048             m = re.match("m/0'/(\d+)", k)
1049             if m:
1050                 num = m.group(1)
1051                 if num == '0':
1052                     default = "Main account"
1053                 else:
1054                     default = "Account %s"%num
1055
1056             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1057             if m:
1058                 num = m.group(1)
1059                 default = "2of2 account %s"%num
1060             name = self.labels.get(k, default)
1061
1062         return name
1063
1064     def get_account_names(self):
1065         accounts = {}
1066         for k, account in self.accounts.items():
1067             accounts[k] = self.get_account_name(k)
1068         if self.imported_keys:
1069             accounts[-1] = 'Imported keys'
1070         return accounts
1071
1072     def get_account_addresses(self, a, include_change=True):
1073         if a is None:
1074             o = self.addresses(True)
1075         elif a == -1:
1076             o = self.imported_keys.keys()
1077         else:
1078             ac = self.accounts[a]
1079             o = ac.get_addresses(0)
1080             if include_change: o += ac.get_addresses(1)
1081         return o
1082
1083     def get_imported_balance(self):
1084         return self.get_balance(self.imported_keys.keys())
1085
1086     def get_account_balance(self, account):
1087         return self.get_balance(self.get_account_addresses(account))
1088
1089     def get_frozen_balance(self):
1090         return self.get_balance(self.frozen_addresses)
1091
1092     def get_balance(self, domain=None):
1093         if domain is None: domain = self.addresses(True)
1094         cc = uu = 0
1095         for addr in domain:
1096             c, u = self.get_addr_balance(addr)
1097             cc += c
1098             uu += u
1099         return cc, uu
1100
1101
1102     def get_unspent_coins(self, domain=None):
1103         coins = []
1104         if domain is None: domain = self.addresses(True)
1105         for addr in domain:
1106             h = self.history.get(addr, [])
1107             if h == ['*']: continue
1108             for tx_hash, tx_height in h:
1109                 tx = self.transactions.get(tx_hash)
1110                 if tx is None: raise BaseException("Wallet not synchronized")
1111                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1112                 for output in tx.d.get('outputs'):
1113                     if output.get('address') != addr: continue
1114                     key = tx_hash + ":%d" % output.get('prevout_n')
1115                     if key in self.spent_outputs: continue
1116                     output['prevout_hash'] = tx_hash
1117                     output['height'] = tx_height
1118                     output['coinbase'] = is_coinbase
1119                     coins.append((tx_height, output))
1120
1121         # sort by age
1122         if coins:
1123             coins = sorted(coins)
1124             if coins[-1][0] != 0:
1125                 while coins[0][0] == 0:
1126                     coins = coins[1:] + [ coins[0] ]
1127         return [x[1] for x in coins]
1128
1129
1130
1131     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1132         domain = self.get_account_addresses(account) if account else None
1133         return self.choose_tx_inputs( amount, fixed_fee, domain )
1134
1135
1136     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1137         """ todo: minimize tx size """
1138         total = 0
1139         fee = self.fee if fixed_fee is None else fixed_fee
1140         if domain is None:
1141             domain = self.addresses(True)
1142
1143         for i in self.frozen_addresses:
1144             if i in domain: domain.remove(i)
1145
1146         prioritized = []
1147         for i in self.prioritized_addresses:
1148             if i in domain:
1149                 domain.remove(i)
1150                 prioritized.append(i)
1151
1152         coins = self.get_unspent_coins(domain)
1153         prioritized_coins = self.get_unspent_coins(prioritized)
1154
1155         inputs = []
1156         coins = prioritized_coins + coins
1157
1158         for item in coins:
1159             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1160                 continue
1161             addr = item.get('address')
1162             v = item.get('value')
1163             total += v
1164             inputs.append(item)
1165             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1166             if total >= amount + fee: break
1167         else:
1168             inputs = []
1169
1170         return inputs, total, fee
1171
1172
1173     def set_fee(self, fee):
1174         if self.fee != fee:
1175             self.fee = fee
1176             self.storage.put('fee_per_kb', self.fee, True)
1177
1178     def estimated_fee(self, inputs):
1179         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1180         fee = self.fee * int(round(estimated_size/1024.))
1181         if fee == 0: fee = self.fee
1182         return fee
1183
1184
1185     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1186         "add change to a transaction"
1187         change_amount = total - ( amount + fee )
1188         if change_amount != 0:
1189             if not change_addr:
1190
1191                 # send change to one of the accounts involved in the tx
1192                 address = inputs[0].get('address')
1193                 account, _ = self.get_address_index(address)
1194
1195                 if not self.use_change or account == -1:
1196                     change_addr = inputs[-1]['address']
1197                 else:
1198                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1199
1200             # Insert the change output at a random position in the outputs
1201             posn = random.randint(0, len(outputs))
1202             outputs[posn:posn] = [( change_addr,  change_amount)]
1203         return outputs
1204
1205
1206     def get_history(self, address):
1207         with self.lock:
1208             return self.history.get(address)
1209
1210
1211     def get_status(self, h):
1212         if not h: return None
1213         if h == ['*']: return '*'
1214         status = ''
1215         for tx_hash, height in h:
1216             status += tx_hash + ':%d:' % height
1217         return hashlib.sha256( status ).digest().encode('hex')
1218
1219
1220     def receive_tx_callback(self, tx_hash, tx, tx_height):
1221
1222         with self.transaction_lock:
1223             self.add_extra_addresses(tx)
1224             if not self.check_new_tx(tx_hash, tx):
1225                 # may happen due to pruning
1226                 print_error("received transaction that is no longer referenced in history", tx_hash)
1227                 return
1228             self.transactions[tx_hash] = tx
1229             self.network.interface.pending_transactions_for_notifications.append(tx)
1230             self.save_transactions()
1231             if self.verifier and tx_height>0:
1232                 self.verifier.add(tx_hash, tx_height)
1233             self.update_tx_outputs(tx_hash)
1234
1235
1236     def save_transactions(self):
1237         tx = {}
1238         for k,v in self.transactions.items():
1239             tx[k] = str(v)
1240         self.storage.put('transactions', tx, True)
1241
1242     def receive_history_callback(self, addr, hist):
1243
1244         if not self.check_new_history(addr, hist):
1245             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1246
1247         with self.lock:
1248             self.history[addr] = hist
1249             self.storage.put('addr_history', self.history, True)
1250
1251         if hist != ['*']:
1252             for tx_hash, tx_height in hist:
1253                 if tx_height>0:
1254                     # add it in case it was previously unconfirmed
1255                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1256
1257
1258     def get_tx_history(self, account=None):
1259         with self.transaction_lock:
1260             history = self.transactions.items()
1261             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1262             result = []
1263
1264             balance = 0
1265             for tx_hash, tx in history:
1266                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1267                 if v is not None: balance += v
1268
1269             c, u = self.get_account_balance(account)
1270
1271             if balance != c+u:
1272                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1273
1274             balance = c + u - balance
1275             for tx_hash, tx in history:
1276                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1277                 if not is_relevant:
1278                     continue
1279                 if value is not None:
1280                     balance += value
1281
1282                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1283                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1284
1285         return result
1286
1287
1288     def get_label(self, tx_hash):
1289         label = self.labels.get(tx_hash)
1290         is_default = (label == '') or (label is None)
1291         if is_default: label = self.get_default_label(tx_hash)
1292         return label, is_default
1293
1294
1295     def get_default_label(self, tx_hash):
1296         tx = self.transactions.get(tx_hash)
1297         default_label = ''
1298         if tx:
1299             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1300             if is_mine:
1301                 for o in tx.outputs:
1302                     o_addr, _ = o
1303                     if not self.is_mine(o_addr):
1304                         try:
1305                             default_label = self.labels[o_addr]
1306                         except KeyError:
1307                             default_label = o_addr
1308                         break
1309                 else:
1310                     default_label = '(internal)'
1311             else:
1312                 for o in tx.outputs:
1313                     o_addr, _ = o
1314                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1315                         break
1316                 else:
1317                     for o in tx.outputs:
1318                         o_addr, _ = o
1319                         if self.is_mine(o_addr):
1320                             break
1321                     else:
1322                         o_addr = None
1323
1324                 if o_addr:
1325                     dest_label = self.labels.get(o_addr)
1326                     try:
1327                         default_label = self.labels[o_addr]
1328                     except KeyError:
1329                         default_label = o_addr
1330
1331         return default_label
1332
1333
1334     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1335         for address, x in outputs:
1336             assert is_valid(address)
1337         amount = sum( map(lambda x:x[1], outputs) )
1338         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1339         if not inputs:
1340             raise ValueError("Not enough funds")
1341         self.add_input_info(inputs)
1342         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1343         return Transaction.from_io(inputs, outputs)
1344
1345
1346     def mktx_from_account(self, outputs, password, fee=None, account=None):
1347         domain = self.get_account_addresses(account) if account else None
1348         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1349
1350
1351     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1352         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1353         keypairs = {}
1354         self.add_keypairs_from_wallet(tx, keypairs, password)
1355         if keypairs:
1356             self.sign_transaction(tx, keypairs)
1357         return tx
1358
1359
1360     def add_input_info(self, inputs):
1361         for txin in inputs:
1362             address = txin['address']
1363             if address in self.imported_keys.keys():
1364                 continue
1365             account, sequence = self.get_address_index(address)
1366             txin['KeyID'] = self.get_keyID(account, sequence)
1367             redeemScript = self.accounts[account].redeem_script(sequence)
1368             if redeemScript:
1369                 txin['redeemScript'] = redeemScript
1370             else:
1371                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1372
1373
1374     def sign_transaction(self, tx, keypairs):
1375         tx.sign(keypairs)
1376         run_hook('sign_transaction', tx)
1377
1378
1379     def sendtx(self, tx):
1380         # synchronous
1381         h = self.send_tx(tx)
1382         self.tx_event.wait()
1383         return self.receive_tx(h)
1384
1385     def send_tx(self, tx):
1386         # asynchronous
1387         self.tx_event.clear()
1388         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1389         return tx.hash()
1390
1391     def on_broadcast(self, i, r):
1392         self.tx_result = r.get('result')
1393         self.tx_event.set()
1394
1395     def receive_tx(self,tx_hash):
1396         out = self.tx_result
1397         if out != tx_hash:
1398             return False, "error: " + out
1399         return True, out
1400
1401
1402
1403     def update_password(self, old_password, new_password):
1404         if new_password == '': new_password = None
1405         # this will throw an exception if unicode cannot be converted
1406         decoded = pw_decode(self.seed, old_password)
1407         self.seed = pw_encode( decoded, new_password)
1408         self.storage.put('seed', self.seed, True)
1409         self.use_encryption = (new_password != None)
1410         self.storage.put('use_encryption', self.use_encryption,True)
1411         for k in self.imported_keys.keys():
1412             a = self.imported_keys[k]
1413             b = pw_decode(a, old_password)
1414             c = pw_encode(b, new_password)
1415             self.imported_keys[k] = c
1416         self.storage.put('imported_keys', self.imported_keys, True)
1417
1418         for k, v in self.master_private_keys.items():
1419             b = pw_decode(v, old_password)
1420             c = pw_encode(b, new_password)
1421             self.master_private_keys[k] = c
1422         self.storage.put('master_private_keys', self.master_private_keys, True)
1423
1424
1425     def freeze(self,addr):
1426         if self.is_mine(addr) and addr not in self.frozen_addresses:
1427             self.unprioritize(addr)
1428             self.frozen_addresses.append(addr)
1429             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1430             return True
1431         else:
1432             return False
1433
1434     def unfreeze(self,addr):
1435         if self.is_mine(addr) and addr in self.frozen_addresses:
1436             self.frozen_addresses.remove(addr)
1437             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1438             return True
1439         else:
1440             return False
1441
1442     def prioritize(self,addr):
1443         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1444             self.unfreeze(addr)
1445             self.prioritized_addresses.append(addr)
1446             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1447             return True
1448         else:
1449             return False
1450
1451     def unprioritize(self,addr):
1452         if self.is_mine(addr) and addr in self.prioritized_addresses:
1453             self.prioritized_addresses.remove(addr)
1454             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1455             return True
1456         else:
1457             return False
1458
1459
1460     def set_verifier(self, verifier):
1461         self.verifier = verifier
1462
1463         # review transactions that are in the history
1464         for addr, hist in self.history.items():
1465             if hist == ['*']: continue
1466             for tx_hash, tx_height in hist:
1467                 if tx_height>0:
1468                     # add it in case it was previously unconfirmed
1469                     self.verifier.add(tx_hash, tx_height)
1470
1471
1472         # if we are on a pruning server, remove unverified transactions
1473         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1474         for tx_hash in self.transactions.keys():
1475             if tx_hash not in vr:
1476                 self.transactions.pop(tx_hash)
1477
1478
1479
1480     def check_new_history(self, addr, hist):
1481
1482         # check that all tx in hist are relevant
1483         if hist != ['*']:
1484             for tx_hash, height in hist:
1485                 tx = self.transactions.get(tx_hash)
1486                 if not tx: continue
1487                 if not tx.has_address(addr):
1488                     return False
1489
1490         # check that we are not "orphaning" a transaction
1491         old_hist = self.history.get(addr,[])
1492         if old_hist == ['*']: return True
1493
1494         for tx_hash, height in old_hist:
1495             if tx_hash in map(lambda x:x[0], hist): continue
1496             found = False
1497             for _addr, _hist in self.history.items():
1498                 if _addr == addr: continue
1499                 if _hist == ['*']: continue
1500                 _tx_hist = map(lambda x:x[0], _hist)
1501                 if tx_hash in _tx_hist:
1502                     found = True
1503                     break
1504
1505             if not found:
1506                 tx = self.transactions.get(tx_hash)
1507                 # tx might not be there
1508                 if not tx: continue
1509
1510                 # already verified?
1511                 if self.verifier.get_height(tx_hash):
1512                     continue
1513                 # unconfirmed tx
1514                 print_error("new history is orphaning transaction:", tx_hash)
1515                 # check that all outputs are not mine, request histories
1516                 ext_requests = []
1517                 for _addr, _v in tx.outputs:
1518                     # assert not self.is_mine(_addr)
1519                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1520
1521                 ext_h = self.network.synchronous_get(ext_requests)
1522                 print_error("sync:", ext_requests, ext_h)
1523                 height = None
1524                 for h in ext_h:
1525                     if h == ['*']: continue
1526                     for item in h:
1527                         if item.get('tx_hash') == tx_hash:
1528                             height = item.get('height')
1529                 if height:
1530                     print_error("found height for", tx_hash, height)
1531                     self.verifier.add(tx_hash, height)
1532                 else:
1533                     print_error("removing orphaned tx from history", tx_hash)
1534                     self.transactions.pop(tx_hash)
1535
1536         return True
1537
1538
1539
1540     def check_new_tx(self, tx_hash, tx):
1541         # 1 check that tx is referenced in addr_history.
1542         addresses = []
1543         for addr, hist in self.history.items():
1544             if hist == ['*']:continue
1545             for txh, height in hist:
1546                 if txh == tx_hash:
1547                     addresses.append(addr)
1548
1549         if not addresses:
1550             return False
1551
1552         # 2 check that referencing addresses are in the tx
1553         for addr in addresses:
1554             if not tx.has_address(addr):
1555                 return False
1556
1557         return True
1558
1559
1560     def start_threads(self, network):
1561         from verifier import TxVerifier
1562         self.network = network
1563         self.verifier = TxVerifier(self.network, self.storage)
1564         self.verifier.start()
1565         self.set_verifier(self.verifier)
1566         self.synchronizer = WalletSynchronizer(self, network)
1567         self.synchronizer.start()
1568
1569     def stop_threads(self):
1570         self.verifier.stop()
1571         self.synchronizer.stop()
1572
1573
1574
1575     def restore(self, callback):
1576         from i18n import _
1577         def wait_for_wallet():
1578             self.set_up_to_date(False)
1579             while not self.is_up_to_date():
1580                 msg = "%s\n%s %d\n%s %.1f"%(
1581                     _("Please wait..."),
1582                     _("Addresses generated:"),
1583                     len(self.addresses(True)),_("Kilobytes received:"),
1584                     self.network.interface.bytes_received/1024.)
1585
1586                 apply(callback, (msg,))
1587                 time.sleep(0.1)
1588
1589         def wait_for_network():
1590             while not self.network.is_connected():
1591                 msg = "%s \n" % (_("Connecting..."))
1592                 apply(callback, (msg,))
1593                 time.sleep(0.1)
1594
1595         # wait until we are connected, because the user might have selected another server
1596         wait_for_network()
1597
1598
1599         self.create_accounts()
1600         wait_for_wallet()
1601
1602
1603
1604
1605 class WalletSynchronizer(threading.Thread):
1606
1607
1608     def __init__(self, wallet, network):
1609         threading.Thread.__init__(self)
1610         self.daemon = True
1611         self.wallet = wallet
1612         self.network = network
1613         self.was_updated = True
1614         self.running = False
1615         self.lock = threading.Lock()
1616         self.queue = Queue.Queue()
1617
1618     def stop(self):
1619         with self.lock: self.running = False
1620
1621     def is_running(self):
1622         with self.lock: return self.running
1623
1624
1625     def subscribe_to_addresses(self, addresses):
1626         messages = []
1627         for addr in addresses:
1628             messages.append(('blockchain.address.subscribe', [addr]))
1629         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1630
1631
1632     def run(self):
1633         with self.lock:
1634             self.running = True
1635
1636         while self.is_running():
1637
1638             if not self.network.is_connected():
1639                 print_error("synchronizer: waiting for interface")
1640                 self.network.wait_until_connected()
1641
1642             self.run_interface(self.network.interface)
1643
1644
1645     def run_interface(self, interface):
1646
1647         print_error("synchronizer: connected to", interface.server)
1648
1649         requested_tx = []
1650         missing_tx = []
1651         requested_histories = {}
1652
1653         # request any missing transactions
1654         for history in self.wallet.history.values():
1655             if history == ['*']: continue
1656             for tx_hash, tx_height in history:
1657                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1658                     missing_tx.append( (tx_hash, tx_height) )
1659
1660         if missing_tx:
1661             print_error("missing tx", missing_tx)
1662
1663         # subscriptions
1664         self.subscribe_to_addresses(self.wallet.addresses(True))
1665
1666         while self.is_running():
1667             # 1. create new addresses
1668             new_addresses = self.wallet.synchronize()
1669
1670             # request missing addresses
1671             if new_addresses:
1672                 self.subscribe_to_addresses(new_addresses)
1673
1674             # request missing transactions
1675             for tx_hash, tx_height in missing_tx:
1676                 if (tx_hash, tx_height) not in requested_tx:
1677                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1678                     requested_tx.append( (tx_hash, tx_height) )
1679             missing_tx = []
1680
1681             # detect if situation has changed
1682             if interface.is_up_to_date() and self.queue.empty():
1683                 if not self.wallet.is_up_to_date():
1684                     self.wallet.set_up_to_date(True)
1685                     self.was_updated = True
1686             else:
1687                 if self.wallet.is_up_to_date():
1688                     self.wallet.set_up_to_date(False)
1689                     self.was_updated = True
1690
1691             if self.was_updated:
1692                 self.wallet.network.trigger_callback('updated')
1693                 self.was_updated = False
1694
1695             # 2. get a response
1696             try:
1697                 r = self.queue.get(block=True, timeout=1)
1698             except Queue.Empty:
1699                 continue
1700
1701             if interface != self.network.interface:
1702                 break
1703
1704             if not r:
1705                 continue
1706
1707             # 3. handle response
1708             method = r['method']
1709             params = r['params']
1710             result = r.get('result')
1711             error = r.get('error')
1712             if error:
1713                 print "error", r
1714                 continue
1715
1716             if method == 'blockchain.address.subscribe':
1717                 addr = params[0]
1718                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1719                     if requested_histories.get(addr) is None:
1720                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1721                         requested_histories[addr] = result
1722
1723             elif method == 'blockchain.address.get_history':
1724                 addr = params[0]
1725                 print_error("receiving history", addr, result)
1726                 if result == ['*']:
1727                     assert requested_histories.pop(addr) == '*'
1728                     self.wallet.receive_history_callback(addr, result)
1729                 else:
1730                     hist = []
1731                     # check that txids are unique
1732                     txids = []
1733                     for item in result:
1734                         tx_hash = item['tx_hash']
1735                         if tx_hash not in txids:
1736                             txids.append(tx_hash)
1737                             hist.append( (tx_hash, item['height']) )
1738
1739                     if len(hist) != len(result):
1740                         raise BaseException("error: server sent history with non-unique txid", result)
1741
1742                     # check that the status corresponds to what was announced
1743                     rs = requested_histories.pop(addr)
1744                     if self.wallet.get_status(hist) != rs:
1745                         raise BaseException("error: status mismatch: %s"%addr)
1746
1747                     # store received history
1748                     self.wallet.receive_history_callback(addr, hist)
1749
1750                     # request transactions that we don't have
1751                     for tx_hash, tx_height in hist:
1752                         if self.wallet.transactions.get(tx_hash) is None:
1753                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1754                                 missing_tx.append( (tx_hash, tx_height) )
1755
1756             elif method == 'blockchain.transaction.get':
1757                 tx_hash = params[0]
1758                 tx_height = params[1]
1759                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1760                 tx = Transaction(result)
1761                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1762                 self.was_updated = True
1763                 requested_tx.remove( (tx_hash, tx_height) )
1764                 print_error("received tx:", tx_hash, len(tx.raw))
1765
1766             else:
1767                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1768
1769             if self.was_updated and not requested_tx:
1770                 self.wallet.network.trigger_callback('updated')
1771                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1772
1773                 self.was_updated = False