lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40
  41 # AES encryption
  42 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  43 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  44
  45 def pw_encode(s, password):
  46     if password:
  47         secret = Hash(password)
  48         return EncodeAES(secret, s)
  49     else:
  50         return s
  51
  52 def pw_decode(s, password):
  53     if password is not None:
  54         secret = Hash(password)
  55         try:
  56             d = DecodeAES(secret, s)
  57         except:
  58             raise BaseException('Invalid password')
  59         return d
  60     else:
  61         return s
  62
  63
  64
  65
  66
  67 from version import *
  68
  69
  70 class WalletStorage:
  71
  72     def __init__(self, config):
  73         self.lock = threading.Lock()
  74         self.data = {}
  75         self.file_exists = False
  76         self.path = self.init_path(config)
  77         print_error( "wallet path", self.path )
  78         if self.path:
  79             self.read(self.path)
  80
  81
  82     def init_path(self, config):
  83         """Set the path of the wallet."""
  84
  85         # command line -w option
  86         path = config.get('wallet_path')
  87         if path:
  88             return path
  89
  90         # path in config file
  91         path = config.get('default_wallet_path')
  92         if path:
  93             return path
  94
  95         # default path
  96         dirpath = os.path.join(config.path, "wallets")
  97         if not os.path.exists(dirpath):
  98             os.mkdir(dirpath)
  99
 100         new_path = os.path.join(config.path, "wallets", "default_wallet")
 101
 102         # default path in pre 1.9 versions
 103         old_path = os.path.join(config.path, "electrum.dat")
 104         if os.path.exists(old_path) and not os.path.exists(new_path):
 105             os.rename(old_path, new_path)
 106
 107         return new_path
 108
 109
 110     def read(self, path):
 111         """Read the contents of the wallet file."""
 112         try:
 113             with open(self.path, "r") as f:
 114                 data = f.read()
 115         except IOError:
 116             return
 117         try:
 118             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 119         except:
 120             raise IOError("Cannot read wallet file.")
 121
 122         self.data = d
 123         self.file_exists = True
 124
 125
 126     def get(self, key, default=None):
 127         return self.data.get(key, default)
 128
 129     def put(self, key, value, save = True):
 130
 131         with self.lock:
 132             if value is not None:
 133                 self.data[key] = value
 134             else:
 135                 self.data.pop(key)
 136             if save:
 137                 self.write()
 138
 139     def write(self):
 140         s = repr(self.data)
 141         f = open(self.path,"w")
 142         f.write( s )
 143         f.close()
 144         if 'ANDROID_DATA' not in os.environ:
 145             import stat
 146             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 147
 148
 149 class Wallet:
 150
 151     def __init__(self, storage):
 152
 153         self.storage = storage
 154         self.electrum_version = ELECTRUM_VERSION
 155         self.gap_limit_for_change = 3 # constant
 156
 157         # saved fields
 158         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 159
 160         self.gap_limit             = storage.get('gap_limit', 5)
 161         self.use_change            = storage.get('use_change',True)
 162         self.use_encryption        = storage.get('use_encryption', False)
 163         self.seed                  = storage.get('seed', '')               # encrypted
 164         self.labels                = storage.get('labels', {})
 165         self.frozen_addresses      = storage.get('frozen_addresses',[])
 166         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 167         self.addressbook           = storage.get('contacts', [])
 168
 169         self.imported_keys         = storage.get('imported_keys',{})
 170         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 171
 172         self.fee                   = int(storage.get('fee_per_kb',20000))
 173
 174         self.master_public_keys = storage.get('master_public_keys',{})
 175         self.master_private_keys = storage.get('master_private_keys', {})
 176
 177         self.next_addresses = storage.get('next_addresses',{})
 178
 179         if self.seed_version not in [4, 6]:
 180             msg = "This wallet seed is not supported."
 181             if self.seed_version in [5]:
 182                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 183                 print msg
 184                 sys.exit(1)
 185
 186
 187         self.load_accounts()
 188
 189         self.transactions = {}
 190         tx_list = self.storage.get('transactions',{})
 191         for k,v in tx_list.items():
 192             try:
 193                 tx = Transaction(v)
 194             except:
 195                 print_msg("Warning: Cannot deserialize transactions. skipping")
 196                 continue
 197
 198             self.add_extra_addresses(tx)
 199             self.transactions[k] = tx
 200
 201         for h,tx in self.transactions.items():
 202             if not self.check_new_tx(h, tx):
 203                 print_error("removing unreferenced tx", h)
 204                 self.transactions.pop(h)
 205
 206
 207         # not saved
 208         self.prevout_values = {}     # my own transaction outputs
 209         self.spent_outputs = []
 210
 211         # spv
 212         self.verifier = None
 213
 214         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 215         # interface.is_up_to_date() returns true when all requests have been answered and processed
 216         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 217
 218         self.up_to_date = False
 219         self.lock = threading.Lock()
 220         self.transaction_lock = threading.Lock()
 221         self.tx_event = threading.Event()
 222
 223         for tx_hash, tx in self.transactions.items():
 224             self.update_tx_outputs(tx_hash)
 225
 226
 227     def add_extra_addresses(self, tx):
 228         h = tx.hash()
 229         # find the address corresponding to pay-to-pubkey inputs
 230         tx.add_extra_addresses(self.transactions)
 231         for o in tx.d.get('outputs'):
 232             if o.get('is_pubkey'):
 233                 for tx2 in self.transactions.values():
 234                     tx2.add_extra_addresses({h:tx})
 235
 236
 237
 238
 239     def set_up_to_date(self,b):
 240         with self.lock: self.up_to_date = b
 241
 242
 243     def is_up_to_date(self):
 244         with self.lock: return self.up_to_date
 245
 246
 247     def update(self):
 248         self.up_to_date = False
 249         while not self.is_up_to_date():
 250             time.sleep(0.1)
 251
 252
 253     def import_key(self, sec, password):
 254         # check password
 255         seed = self.get_seed(password)
 256         try:
 257             address = address_from_private_key(sec)
 258         except:
 259             raise BaseException('Invalid private key')
 260
 261         if self.is_mine(address):
 262             raise BaseException('Address already in wallet')
 263
 264         # store the originally requested keypair into the imported keys table
 265         self.imported_keys[address] = pw_encode(sec, password )
 266         self.storage.put('imported_keys', self.imported_keys, True)
 267         if self.synchronizer:
 268             self.synchronizer.subscribe_to_addresses([address])
 269         return address
 270
 271     def delete_imported_key(self, addr):
 272         if addr in self.imported_keys:
 273             self.imported_keys.pop(addr)
 274             self.storage.put('imported_keys', self.imported_keys, True)
 275
 276
 277     def make_seed(self):
 278         import mnemonic, ecdsa
 279         entropy = ecdsa.util.randrange( pow(2,160) )
 280         nonce = 0
 281         while True:
 282             ss = "%040x"%(entropy+nonce)
 283             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 284             # we keep only 13 words, that's approximately 139 bits of entropy
 285             words = mnemonic.mn_encode(s)[0:13]
 286             seed = ' '.join(words)
 287             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 288                 break  # this removes 12 bits of entropy
 289             nonce += 1
 290
 291         return seed
 292
 293
 294     def init_seed(self, seed):
 295         import mnemonic
 296
 297         if self.seed:
 298             raise BaseException("a seed exists")
 299
 300         if not seed:
 301             self.seed = random_seed(128)
 302             self.seed_version = 4
 303             return
 304
 305         #if not seed:
 306         #    self.seed = self.make_seed()
 307         #    self.seed_version = SEED_VERSION
 308         #    return
 309
 310         # find out what kind of wallet we are
 311         try:
 312             seed.decode('hex')
 313             self.seed_version = 4
 314             self.seed = str(seed)
 315             return
 316         except:
 317             pass
 318
 319         words = seed.split()
 320         self.seed_version = 4
 321         self.seed = mnemonic.mn_decode(words)
 322
 323         #try:
 324         #    mnemonic.mn_decode(words)
 325         #    uses_electrum_words = True
 326         #except:
 327         #    uses_electrum_words = False
 328         #
 329         #if uses_electrum_words and len(words) != 13:
 330         #    self.seed_version = 4
 331         #    self.seed = mnemonic.mn_decode(words)
 332         #else:
 333         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 334         #    self.seed_version = SEED_VERSION
 335         #    self.seed = seed
 336
 337
 338     def save_seed(self):
 339         self.storage.put('seed', self.seed, True)
 340         self.storage.put('seed_version', self.seed_version, True)
 341
 342     def create_watching_only_wallet(self, params):
 343         K0, c0 = params
 344         if not K0:
 345             return
 346
 347         if not c0:
 348             self.seed_version = 4
 349             self.create_old_account(K0)
 350             return
 351
 352         cK0 = ""
 353         self.master_public_keys = {
 354             "m/0'/": (c0, K0, cK0),
 355             }
 356         self.storage.put('master_public_keys', self.master_public_keys, True)
 357         self.create_account('1','Main account')
 358
 359
 360     def create_accounts(self):
 361         if self.seed_version == 4:
 362             mpk = OldAccount.mpk_from_seed(self.seed)
 363             self.create_old_account(mpk)
 364         else:
 365             # create default account
 366             self.create_master_keys('1')
 367             self.create_account('1','Main account')
 368
 369
 370     def create_master_keys(self, account_type):
 371         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 372         if account_type == '1':
 373             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 374             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 375             self.master_private_keys["m/0'/"] = k0
 376         elif account_type == '2of2':
 377             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 378             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 379             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 380             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 381             self.master_private_keys["m/1'/"] = k1
 382             self.master_private_keys["m/2'/"] = k2
 383         elif account_type == '2of3':
 384             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 385             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 386             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 387             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 388             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 389             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 390             self.master_private_keys["m/3'/"] = k3
 391             self.master_private_keys["m/4'/"] = k4
 392             self.master_private_keys["m/5'/"] = k5
 393
 394         self.storage.put('master_public_keys', self.master_public_keys, True)
 395         self.storage.put('master_private_keys', self.master_private_keys, True)
 396
 397     def has_master_public_keys(self, account_type):
 398         if account_type == '1':
 399             return "m/0'/" in self.master_public_keys
 400         elif account_type == '2of2':
 401             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 402         elif account_type == '2of3':
 403             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 404
 405     def find_root_by_master_key(self, c, K):
 406         for key, v in self.master_public_keys.items():
 407             if key == "m/":continue
 408             cc, KK, _ = v
 409             if (c == cc) and (K == KK):
 410                 return key
 411
 412     def deseed_root(self, seed, password):
 413         # for safety, we ask the user to enter their seed
 414         assert seed == self.get_seed(password)
 415         self.seed = ''
 416         self.storage.put('seed', '', True)
 417
 418
 419     def deseed_branch(self, k):
 420         # check that parent has no seed
 421         assert self.seed == ''
 422         self.master_private_keys.pop(k)
 423         self.storage.put('master_private_keys', self.master_private_keys, True)
 424
 425     def is_watching_only(self):
 426         return (self.seed == '') and (self.master_private_keys == {})
 427
 428
 429
 430     def account_id(self, account_type, i):
 431         if account_type == '1':
 432             return "m/0'/%d"%i
 433         elif account_type == '2of2':
 434             return "m/1'/%d & m/2'/%d"%(i,i)
 435         elif account_type == '2of3':
 436             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 437         else:
 438             raise BaseException('unknown account type')
 439
 440
 441     def num_accounts(self, account_type):
 442         keys = self.accounts.keys()
 443         i = 0
 444         while True:
 445             account_id = self.account_id(account_type, i)
 446             if account_id not in keys: break
 447             i += 1
 448         return i
 449
 450
 451     def new_account_address(self, account_type = '1'):
 452         i = self.num_accounts(account_type)
 453         k = self.account_id(account_type,i)
 454
 455         addr = self.next_addresses.get(k)
 456         if not addr:
 457             account_id, account = self.next_account(account_type)
 458             addr = account.first_address()
 459             self.next_addresses[k] = addr
 460             self.storage.put('next_addresses',self.next_addresses)
 461
 462         return k, addr
 463
 464
 465     def next_account(self, account_type = '1'):
 466
 467         i = self.num_accounts(account_type)
 468         account_id = self.account_id(account_type,i)
 469
 470         if account_type is '1':
 471             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 472             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 473             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 474
 475         elif account_type == '2of2':
 476             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 477             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 478             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 479             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 480             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 481
 482         elif account_type == '2of3':
 483             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 484             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 485             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 486             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 487             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 488             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 489             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 490
 491         return account_id, account
 492
 493
 494     def set_label(self, name, text = None):
 495         changed = False
 496         old_text = self.labels.get(name)
 497         if text:
 498             if old_text != text:
 499                 self.labels[name] = text
 500                 changed = True
 501         else:
 502             if old_text:
 503                 self.labels.pop(name)
 504                 changed = True
 505
 506         if changed:
 507             self.storage.put('labels', self.labels, True)
 508
 509         run_hook('set_label', name, text, changed)
 510         return changed
 511
 512
 513
 514     def create_account(self, account_type = '1', name = None):
 515         k, account = self.next_account(account_type)
 516         if k in self.pending_accounts:
 517             self.pending_accounts.pop(k)
 518             self.storage.put('pending_accounts', self.pending_accounts)
 519
 520         self.accounts[k] = account
 521         self.save_accounts()
 522         if name:
 523             self.set_label(k, name)
 524
 525
 526     def create_old_account(self, mpk):
 527         self.storage.put('master_public_key', mpk, True)
 528         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 529         self.save_accounts()
 530
 531
 532     def save_accounts(self):
 533         d = {}
 534         for k, v in self.accounts.items():
 535             d[k] = v.dump()
 536         self.storage.put('accounts', d, True)
 537
 538
 539
 540     def load_accounts(self):
 541         d = self.storage.get('accounts', {})
 542         self.accounts = {}
 543         for k, v in d.items():
 544             if k == 0:
 545                 v['mpk'] = self.storage.get('master_public_key')
 546                 self.accounts[k] = OldAccount(v)
 547             elif '&' in k:
 548                 self.accounts[k] = BIP32_Account_2of2(v)
 549             else:
 550                 self.accounts[k] = BIP32_Account(v)
 551
 552         self.pending_accounts = self.storage.get('pending_accounts',{})
 553
 554
 555     def delete_pending_account(self, k):
 556         self.pending_accounts.pop(k)
 557         self.storage.put('pending_accounts', self.pending_accounts)
 558
 559     def account_is_pending(self, k):
 560         return k in self.pending_accounts
 561
 562     def create_pending_account(self, acct_type, name):
 563         k, addr = self.new_account_address(acct_type)
 564         self.set_label(k, name)
 565         self.pending_accounts[k] = addr
 566         self.storage.put('pending_accounts', self.pending_accounts)
 567
 568     def get_pending_accounts(self):
 569         return self.pending_accounts.items()
 570
 571
 572     def addresses(self, include_change = True, _next=True):
 573         o = self.get_account_addresses(-1, include_change)
 574         for a in self.accounts.keys():
 575             o += self.get_account_addresses(a, include_change)
 576
 577         if _next:
 578             for addr in self.next_addresses.values():
 579                 if addr not in o:
 580                     o += [addr]
 581         return o
 582
 583
 584     def is_mine(self, address):
 585         return address in self.addresses(True)
 586
 587
 588     def is_change(self, address):
 589         if not self.is_mine(address): return False
 590         if address in self.imported_keys.keys(): return False
 591         acct, s = self.get_address_index(address)
 592         if s is None: return False
 593         return s[0] == 1
 594
 595     def get_master_public_key(self):
 596         if self.seed_version == 4:
 597             return self.storage.get("master_public_key")
 598         else:
 599             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 600             return repr((c, K))
 601
 602     def get_master_private_key(self, account, password):
 603         master_k = pw_decode( self.master_private_keys[account], password)
 604         master_c, master_K, master_Kc = self.master_public_keys[account]
 605         try:
 606             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 607             assert K.encode('hex') == master_K
 608         except:
 609             raise BaseException("Invalid password")
 610         return master_k
 611
 612
 613     def get_address_index(self, address):
 614         if address in self.imported_keys.keys():
 615             return -1, None
 616
 617         for account in self.accounts.keys():
 618             for for_change in [0,1]:
 619                 addresses = self.accounts[account].get_addresses(for_change)
 620                 for addr in addresses:
 621                     if address == addr:
 622                         return account, (for_change, addresses.index(addr))
 623
 624         for k,v in self.next_addresses.items():
 625             if v == address:
 626                 return k, (0,0)
 627
 628         raise BaseException("Address not found", address)
 629
 630
 631     def get_roots(self, account):
 632         roots = []
 633         for a in account.split('&'):
 634             s = a.strip()
 635             m = re.match("(m/\d+'/)(\d+)", s)
 636             roots.append( m.group(1) )
 637         return roots
 638
 639     def is_seeded(self, account):
 640         if type(account) is int:
 641             return self.seed is not None
 642
 643         for root in self.get_roots(account):
 644             if root not in self.master_private_keys.keys():
 645                 return False
 646         return True
 647
 648     def rebase_sequence(self, account, sequence):
 649         c, i = sequence
 650         dd = []
 651         for a in account.split('&'):
 652             s = a.strip()
 653             m = re.match("(m/\d+'/)(\d+)", s)
 654             root = m.group(1)
 655             num = int(m.group(2))
 656             dd.append( (root, [num,c,i] ) )
 657         return dd
 658
 659
 660     def get_keyID(self, account, sequence):
 661         if account == 0:
 662             a, b = sequence
 663             mpk = self.storage.get('master_public_key')
 664             return 'old(%s,%d,%d)'%(mpk,a,b)
 665
 666         rs = self.rebase_sequence(account, sequence)
 667         dd = []
 668         for root, public_sequence in rs:
 669             c, K, _ = self.master_public_keys[root]
 670             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 671             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 672         return '&'.join(dd)
 673
 674
 675
 676     def get_seed(self, password):
 677         s = pw_decode(self.seed, password)
 678         if self.seed_version == 4:
 679             seed = s
 680         else:
 681             seed = mnemonic_hash(s)
 682         #todo:  #self.sequences[0].check_seed(seed)
 683         return seed
 684
 685
 686     def get_mnemonic(self, password):
 687         import mnemonic
 688         s = pw_decode(self.seed, password)
 689         if self.seed_version == 4:
 690             return ' '.join(mnemonic.mn_encode(s))
 691         else:
 692             return s
 693
 694
 695
 696     def get_private_key(self, address, password):
 697         out = []
 698         if address in self.imported_keys.keys():
 699             out.append( pw_decode( self.imported_keys[address], password ) )
 700         else:
 701             account, sequence = self.get_address_index(address)
 702             if account == 0:
 703                 seed = self.get_seed(password)
 704                 pk = self.accounts[account].get_private_key(seed, sequence)
 705                 out.append(pk)
 706                 return out
 707
 708             # assert address == self.accounts[account].get_address(*sequence)
 709             rs = self.rebase_sequence( account, sequence)
 710             for root, public_sequence in rs:
 711
 712                 if root not in self.master_private_keys.keys(): continue
 713                 master_k = self.get_master_private_key(root, password)
 714                 master_c, _, _ = self.master_public_keys[root]
 715                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 716                 out.append(pk)
 717
 718         return out
 719
 720
 721     def add_keypairs_from_wallet(self, tx, keypairs, password):
 722         for txin in tx.inputs:
 723             address = txin['address']
 724             private_keys = self.get_private_key(address, password)
 725             for sec in private_keys:
 726                 pubkey = public_key_from_private_key(sec)
 727                 keypairs[ pubkey ] = sec
 728                 if address in self.imported_keys.keys():
 729                     txin['redeemPubkey'] = pubkey
 730
 731
 732     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 733         for txin in tx.inputs:
 734             keyid = txin.get('KeyID')
 735             if keyid:
 736
 737                 if self.seed_version==4:
 738                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 739                     if not m: continue
 740                     mpk = m.group(1)
 741                     if mpk != self.storage.get('master_public_key'): continue
 742                     index = int(m.group(2))
 743                     num = int(m.group(3))
 744                     account = self.accounts[0]
 745                     addr = account.get_address(index, num)
 746                     txin['address'] = addr # fixme: side effect
 747                     pk = self.get_private_key(addr, password)
 748                     for sec in pk:
 749                         pubkey = public_key_from_private_key(sec)
 750                         keypairs[pubkey] = sec
 751                     continue
 752
 753
 754                 roots = []
 755                 for s in keyid.split('&'):
 756                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 757                     if not m: continue
 758                     c = m.group(1)
 759                     K = m.group(2)
 760                     sequence = m.group(3)
 761                     root = self.find_root_by_master_key(c,K)
 762                     if not root: continue
 763                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 764                     root = root + '%d'%sequence[0]
 765                     sequence = sequence[1:]
 766                     roots.append((root,sequence))
 767
 768                 account_id = " & ".join( map(lambda x:x[0], roots) )
 769                 account = self.accounts.get(account_id)
 770                 if not account: continue
 771                 addr = account.get_address(*sequence)
 772                 txin['address'] = addr # fixme: side effect
 773                 pk = self.get_private_key(addr, password)
 774                 for sec in pk:
 775                     pubkey = public_key_from_private_key(sec)
 776                     keypairs[pubkey] = sec
 777
 778
 779
 780     def signrawtransaction(self, tx, input_info, private_keys, password):
 781
 782         # check that the password is correct
 783         seed = self.get_seed(password)
 784
 785         # add input info
 786         tx.add_input_info(input_info)
 787
 788         # add redeem script for coins that are in the wallet
 789         # FIXME: add redeemPubkey too!
 790         unspent_coins = self.get_unspent_coins()
 791         for txin in tx.inputs:
 792             for item in unspent_coins:
 793                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 794                     print_error( "tx input is in unspent coins" )
 795                     txin['scriptPubKey'] = item['scriptPubKey']
 796                     account, sequence = self.get_address_index(item['address'])
 797                     if account != -1:
 798                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 799                         print_error("added redeemScript", txin['redeemScript'])
 800                     break
 801
 802
 803         # build a list of public/private keys
 804         keypairs = {}
 805
 806         # add private keys from parameter
 807         for sec in private_keys:
 808             pubkey = public_key_from_private_key(sec)
 809             keypairs[ pubkey ] = sec
 810
 811         # add private_keys from KeyID
 812         self.add_keypairs_from_KeyID(tx, keypairs, password)
 813
 814         # add private keys from wallet
 815         self.add_keypairs_from_wallet(tx, keypairs, password)
 816         self.sign_transaction(tx, keypairs)
 817
 818
 819     def sign_message(self, address, message, password):
 820         keys = self.get_private_key(address, password)
 821         assert len(keys) == 1
 822         sec = keys[0]
 823         key = regenerate_key(sec)
 824         compressed = is_compressed(sec)
 825         return key.sign_message(message, compressed, address)
 826
 827
 828     def change_gap_limit(self, value):
 829         if value >= self.gap_limit:
 830             self.gap_limit = value
 831             self.storage.put('gap_limit', self.gap_limit, True)
 832             #self.interface.poke('synchronizer')
 833             return True
 834
 835         elif value >= self.min_acceptable_gap():
 836             for key, account in self.accounts.items():
 837                 addresses = account[0]
 838                 k = self.num_unused_trailing_addresses(addresses)
 839                 n = len(addresses) - k + value
 840                 addresses = addresses[0:n]
 841                 self.accounts[key][0] = addresses
 842
 843             self.gap_limit = value
 844             self.storage.put('gap_limit', self.gap_limit, True)
 845             self.save_accounts()
 846             return True
 847         else:
 848             return False
 849
 850     def num_unused_trailing_addresses(self, addresses):
 851         k = 0
 852         for a in addresses[::-1]:
 853             if self.history.get(a):break
 854             k = k + 1
 855         return k
 856
 857     def min_acceptable_gap(self):
 858         # fixme: this assumes wallet is synchronized
 859         n = 0
 860         nmax = 0
 861
 862         for account in self.accounts.values():
 863             addresses = account.get_addresses(0)
 864             k = self.num_unused_trailing_addresses(addresses)
 865             for a in addresses[0:-k]:
 866                 if self.history.get(a):
 867                     n = 0
 868                 else:
 869                     n += 1
 870                     if n > nmax: nmax = n
 871         return nmax + 1
 872
 873
 874     def address_is_old(self, address):
 875         age = -1
 876         h = self.history.get(address, [])
 877         if h == ['*']:
 878             return True
 879         for tx_hash, tx_height in h:
 880             if tx_height == 0:
 881                 tx_age = 0
 882             else:
 883                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 884             if tx_age > age:
 885                 age = tx_age
 886         return age > 2
 887
 888
 889     def synchronize_sequence(self, account, for_change):
 890         limit = self.gap_limit_for_change if for_change else self.gap_limit
 891         new_addresses = []
 892         while True:
 893             addresses = account.get_addresses(for_change)
 894             if len(addresses) < limit:
 895                 address = account.create_new_address(for_change)
 896                 self.history[address] = []
 897                 new_addresses.append( address )
 898                 continue
 899
 900             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 901                 break
 902             else:
 903                 address = account.create_new_address(for_change)
 904                 self.history[address] = []
 905                 new_addresses.append( address )
 906
 907         return new_addresses
 908
 909
 910
 911     def create_pending_accounts(self):
 912         for account_type in ['1','2of2','2of3']:
 913             if not self.has_master_public_keys(account_type):
 914                 continue
 915             k, a = self.new_account_address(account_type)
 916             if self.address_is_old(a):
 917                 print_error( "creating account", a )
 918                 self.create_account(account_type)
 919                 self.next_addresses.pop(k)
 920
 921
 922     def synchronize_account(self, account):
 923         new = []
 924         new += self.synchronize_sequence(account, 0)
 925         new += self.synchronize_sequence(account, 1)
 926         return new
 927
 928
 929     def synchronize(self):
 930         if self.master_public_keys:
 931             self.create_pending_accounts()
 932         new = []
 933         for account in self.accounts.values():
 934             new += self.synchronize_account(account)
 935         if new:
 936             self.save_accounts()
 937             self.storage.put('addr_history', self.history, True)
 938         return new
 939
 940
 941     def is_found(self):
 942         return self.history.values() != [[]] * len(self.history)
 943
 944
 945     def add_contact(self, address, label=None):
 946         self.addressbook.append(address)
 947         self.storage.put('contacts', self.addressbook, True)
 948         if label:
 949             self.set_label(address, label)
 950
 951
 952     def delete_contact(self, addr):
 953         if addr in self.addressbook:
 954             self.addressbook.remove(addr)
 955             self.storage.put('addressbook', self.addressbook, True)
 956
 957
 958     def fill_addressbook(self):
 959         for tx_hash, tx in self.transactions.items():
 960             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 961             if is_send:
 962                 for addr, v in tx.outputs:
 963                     if not self.is_mine(addr) and addr not in self.addressbook:
 964                         self.addressbook.append(addr)
 965         # redo labels
 966         # self.update_tx_labels()
 967
 968     def get_num_tx(self, address):
 969         n = 0
 970         for tx in self.transactions.values():
 971             if address in map(lambda x:x[0], tx.outputs): n += 1
 972         return n
 973
 974
 975     def get_address_flags(self, addr):
 976         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 977         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 978         return flags
 979
 980
 981     def get_tx_value(self, tx, account=None):
 982         domain = self.get_account_addresses(account)
 983         return tx.get_value(domain, self.prevout_values)
 984
 985
 986     def update_tx_outputs(self, tx_hash):
 987         tx = self.transactions.get(tx_hash)
 988
 989         for i, (addr, value) in enumerate(tx.outputs):
 990             key = tx_hash+ ':%d'%i
 991             self.prevout_values[key] = value
 992
 993         for item in tx.inputs:
 994             if self.is_mine(item.get('address')):
 995                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 996                 self.spent_outputs.append(key)
 997
 998
 999     def get_addr_balance(self, address):
1000         assert self.is_mine(address)
1001         h = self.history.get(address,[])
1002         if h == ['*']: return 0,0
1003         c = u = 0
1004         received_coins = []   # list of coins received at address
1005
1006         for tx_hash, tx_height in h:
1007             tx = self.transactions.get(tx_hash)
1008             if not tx: continue
1009
1010             for i, (addr, value) in enumerate(tx.outputs):
1011                 if addr == address:
1012                     key = tx_hash + ':%d'%i
1013                     received_coins.append(key)
1014
1015         for tx_hash, tx_height in h:
1016             tx = self.transactions.get(tx_hash)
1017             if not tx: continue
1018             v = 0
1019
1020             for item in tx.inputs:
1021                 addr = item.get('address')
1022                 if addr == address:
1023                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1024                     value = self.prevout_values.get( key )
1025                     if key in received_coins:
1026                         v -= value
1027
1028             for i, (addr, value) in enumerate(tx.outputs):
1029                 key = tx_hash + ':%d'%i
1030                 if addr == address:
1031                     v += value
1032
1033             if tx_height:
1034                 c += v
1035             else:
1036                 u += v
1037         return c, u
1038
1039
1040     def get_account_name(self, k):
1041         if k == 0:
1042             if self.seed_version == 4:
1043                 name = 'Main account'
1044             else:
1045                 name = 'Old account'
1046         else:
1047             default = "Unnamed account"
1048             m = re.match("m/0'/(\d+)", k)
1049             if m:
1050                 num = m.group(1)
1051                 if num == '0':
1052                     default = "Main account"
1053                 else:
1054                     default = "Account %s"%num
1055
1056             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1057             if m:
1058                 num = m.group(1)
1059                 default = "2of2 account %s"%num
1060             name = self.labels.get(k, default)
1061
1062         return name
1063
1064     def get_account_names(self):
1065         accounts = {}
1066         for k, account in self.accounts.items():
1067             accounts[k] = self.get_account_name(k)
1068         if self.imported_keys:
1069             accounts[-1] = 'Imported keys'
1070         return accounts
1071
1072     def get_account_addresses(self, a, include_change=True):
1073         if a is None:
1074             o = self.addresses(True)
1075         elif a == -1:
1076             o = self.imported_keys.keys()
1077         else:
1078             ac = self.accounts[a]
1079             o = ac.get_addresses(0)
1080             if include_change: o += ac.get_addresses(1)
1081         return o
1082
1083     def get_imported_balance(self):
1084         return self.get_balance(self.imported_keys.keys())
1085
1086     def get_account_balance(self, account):
1087         return self.get_balance(self.get_account_addresses(account))
1088
1089     def get_frozen_balance(self):
1090         return self.get_balance(self.frozen_addresses)
1091
1092     def get_balance(self, domain=None):
1093         if domain is None: domain = self.addresses(True)
1094         cc = uu = 0
1095         for addr in domain:
1096             c, u = self.get_addr_balance(addr)
1097             cc += c
1098             uu += u
1099         return cc, uu
1100
1101
1102     def get_unspent_coins(self, domain=None):
1103         coins = []
1104         if domain is None: domain = self.addresses(True)
1105         for addr in domain:
1106             h = self.history.get(addr, [])
1107             if h == ['*']: continue
1108             for tx_hash, tx_height in h:
1109                 tx = self.transactions.get(tx_hash)
1110                 if tx is None: raise BaseException("Wallet not synchronized")
1111                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1112                 for output in tx.d.get('outputs'):
1113                     if output.get('address') != addr: continue
1114                     key = tx_hash + ":%d" % output.get('prevout_n')
1115                     if key in self.spent_outputs: continue
1116                     output['prevout_hash'] = tx_hash
1117                     output['height'] = tx_height
1118                     output['coinbase'] = is_coinbase
1119                     coins.append((tx_height, output))
1120
1121         # sort by age
1122         if coins:
1123             coins = sorted(coins)
1124             if coins[-1][0] != 0:
1125                 while coins[0][0] == 0:
1126                     coins = coins[1:] + [ coins[0] ]
1127         return [x[1] for x in coins]
1128
1129
1130
1131     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1132         domain = self.get_account_addresses(account) if account else None
1133         return self.choose_tx_inputs( amount, fixed_fee, domain )
1134
1135
1136     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1137         """ todo: minimize tx size """
1138         total = 0
1139         fee = self.fee if fixed_fee is None else fixed_fee
1140         if domain is None:
1141             domain = self.addresses(True)
1142
1143         for i in self.frozen_addresses:
1144             if i in domain: domain.remove(i)
1145
1146         prioritized = []
1147         for i in self.prioritized_addresses:
1148             if i in domain:
1149                 domain.remove(i)
1150                 prioritized.append(i)
1151
1152         coins = self.get_unspent_coins(domain)
1153         prioritized_coins = self.get_unspent_coins(prioritized)
1154
1155         inputs = []
1156         coins = prioritized_coins + coins
1157
1158         for item in coins:
1159             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1160                 continue
1161             addr = item.get('address')
1162             v = item.get('value')
1163             total += v
1164             inputs.append(item)
1165             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1166             if total >= amount + fee: break
1167         else:
1168             inputs = []
1169
1170         return inputs, total, fee
1171
1172
1173     def set_fee(self, fee):
1174         if self.fee != fee:
1175             self.fee = fee
1176             self.storage.put('fee_per_kb', self.fee, True)
1177
1178     def estimated_fee(self, inputs):
1179         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1180         fee = self.fee * int(round(estimated_size/1024.))
1181         if fee == 0: fee = self.fee
1182         return fee
1183
1184
1185     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1186         "add change to a transaction"
1187         change_amount = total - ( amount + fee )
1188         if change_amount != 0:
1189             if not change_addr:
1190
1191                 # send change to one of the accounts involved in the tx
1192                 address = inputs[0].get('address')
1193                 account, _ = self.get_address_index(address)
1194
1195                 if not self.use_change or account == -1:
1196                     change_addr = inputs[-1]['address']
1197                 else:
1198                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1199
1200             # Insert the change output at a random position in the outputs
1201             posn = random.randint(0, len(outputs))
1202             outputs[posn:posn] = [( change_addr,  change_amount)]
1203         return outputs
1204
1205
1206     def get_history(self, address):
1207         with self.lock:
1208             return self.history.get(address)
1209
1210
1211     def get_status(self, h):
1212         if not h: return None
1213         if h == ['*']: return '*'
1214         status = ''
1215         for tx_hash, height in h:
1216             status += tx_hash + ':%d:' % height
1217         return hashlib.sha256( status ).digest().encode('hex')
1218
1219
1220     def receive_tx_callback(self, tx_hash, tx, tx_height):
1221
1222         with self.transaction_lock:
1223             self.add_extra_addresses(tx)
1224             if not self.check_new_tx(tx_hash, tx):
1225                 # may happen due to pruning
1226                 print_error("received transaction that is no longer referenced in history", tx_hash)
1227                 return
1228             self.transactions[tx_hash] = tx
1229             self.network.interface.pending_transactions_for_notifications.append(tx)
1230             self.save_transactions()
1231             if self.verifier and tx_height>0:
1232                 self.verifier.add(tx_hash, tx_height)
1233             self.update_tx_outputs(tx_hash)
1234
1235
1236     def save_transactions(self):
1237         tx = {}
1238         for k,v in self.transactions.items():
1239             tx[k] = str(v)
1240         self.storage.put('transactions', tx, True)
1241
1242     def receive_history_callback(self, addr, hist):
1243
1244         if not self.check_new_history(addr, hist):
1245             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1246
1247         with self.lock:
1248             self.history[addr] = hist
1249             self.storage.put('addr_history', self.history, True)
1250
1251         if hist != ['*']:
1252             for tx_hash, tx_height in hist:
1253                 if tx_height>0:
1254                     # add it in case it was previously unconfirmed
1255                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1256
1257
1258     def get_tx_history(self, account=None):
1259         if not self.verifier:
1260             return []
1261
1262         with self.transaction_lock:
1263             history = self.transactions.items()
1264             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1265             result = []
1266
1267             balance = 0
1268             for tx_hash, tx in history:
1269                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1270                 if v is not None: balance += v
1271
1272             c, u = self.get_account_balance(account)
1273
1274             if balance != c+u:
1275                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1276
1277             balance = c + u - balance
1278             for tx_hash, tx in history:
1279                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1280                 if not is_relevant:
1281                     continue
1282                 if value is not None:
1283                     balance += value
1284
1285                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1286                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1287
1288         return result
1289
1290
1291     def get_label(self, tx_hash):
1292         label = self.labels.get(tx_hash)
1293         is_default = (label == '') or (label is None)
1294         if is_default: label = self.get_default_label(tx_hash)
1295         return label, is_default
1296
1297
1298     def get_default_label(self, tx_hash):
1299         tx = self.transactions.get(tx_hash)
1300         default_label = ''
1301         if tx:
1302             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1303             if is_mine:
1304                 for o in tx.outputs:
1305                     o_addr, _ = o
1306                     if not self.is_mine(o_addr):
1307                         try:
1308                             default_label = self.labels[o_addr]
1309                         except KeyError:
1310                             default_label = o_addr
1311                         break
1312                 else:
1313                     default_label = '(internal)'
1314             else:
1315                 for o in tx.outputs:
1316                     o_addr, _ = o
1317                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1318                         break
1319                 else:
1320                     for o in tx.outputs:
1321                         o_addr, _ = o
1322                         if self.is_mine(o_addr):
1323                             break
1324                     else:
1325                         o_addr = None
1326
1327                 if o_addr:
1328                     dest_label = self.labels.get(o_addr)
1329                     try:
1330                         default_label = self.labels[o_addr]
1331                     except KeyError:
1332                         default_label = o_addr
1333
1334         return default_label
1335
1336
1337     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1338         for address, x in outputs:
1339             assert is_valid(address)
1340         amount = sum( map(lambda x:x[1], outputs) )
1341         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1342         if not inputs:
1343             raise ValueError("Not enough funds")
1344         self.add_input_info(inputs)
1345         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1346         return Transaction.from_io(inputs, outputs)
1347
1348
1349     def mktx_from_account(self, outputs, password, fee=None, account=None):
1350         domain = self.get_account_addresses(account) if account else None
1351         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1352
1353
1354     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1355         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1356         keypairs = {}
1357         self.add_keypairs_from_wallet(tx, keypairs, password)
1358         if keypairs:
1359             self.sign_transaction(tx, keypairs)
1360         return tx
1361
1362
1363     def add_input_info(self, inputs):
1364         for txin in inputs:
1365             address = txin['address']
1366             if address in self.imported_keys.keys():
1367                 continue
1368             account, sequence = self.get_address_index(address)
1369             txin['KeyID'] = self.get_keyID(account, sequence)
1370             redeemScript = self.accounts[account].redeem_script(sequence)
1371             if redeemScript:
1372                 txin['redeemScript'] = redeemScript
1373             else:
1374                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1375
1376
1377     def sign_transaction(self, tx, keypairs):
1378         tx.sign(keypairs)
1379         run_hook('sign_transaction', tx)
1380
1381
1382     def sendtx(self, tx):
1383         # synchronous
1384         h = self.send_tx(tx)
1385         self.tx_event.wait()
1386         return self.receive_tx(h)
1387
1388     def send_tx(self, tx):
1389         # asynchronous
1390         self.tx_event.clear()
1391         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1392         return tx.hash()
1393
1394     def on_broadcast(self, i, r):
1395         self.tx_result = r.get('result')
1396         self.tx_event.set()
1397
1398     def receive_tx(self,tx_hash):
1399         out = self.tx_result
1400         if out != tx_hash:
1401             return False, "error: " + out
1402         return True, out
1403
1404
1405
1406     def update_password(self, old_password, new_password):
1407         if new_password == '': new_password = None
1408         # this will throw an exception if unicode cannot be converted
1409         decoded = pw_decode(self.seed, old_password)
1410         self.seed = pw_encode( decoded, new_password)
1411         self.storage.put('seed', self.seed, True)
1412         self.use_encryption = (new_password != None)
1413         self.storage.put('use_encryption', self.use_encryption,True)
1414         for k in self.imported_keys.keys():
1415             a = self.imported_keys[k]
1416             b = pw_decode(a, old_password)
1417             c = pw_encode(b, new_password)
1418             self.imported_keys[k] = c
1419         self.storage.put('imported_keys', self.imported_keys, True)
1420
1421         for k, v in self.master_private_keys.items():
1422             b = pw_decode(v, old_password)
1423             c = pw_encode(b, new_password)
1424             self.master_private_keys[k] = c
1425         self.storage.put('master_private_keys', self.master_private_keys, True)
1426
1427
1428     def freeze(self,addr):
1429         if self.is_mine(addr) and addr not in self.frozen_addresses:
1430             self.unprioritize(addr)
1431             self.frozen_addresses.append(addr)
1432             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1433             return True
1434         else:
1435             return False
1436
1437     def unfreeze(self,addr):
1438         if self.is_mine(addr) and addr in self.frozen_addresses:
1439             self.frozen_addresses.remove(addr)
1440             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1441             return True
1442         else:
1443             return False
1444
1445     def prioritize(self,addr):
1446         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1447             self.unfreeze(addr)
1448             self.prioritized_addresses.append(addr)
1449             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1450             return True
1451         else:
1452             return False
1453
1454     def unprioritize(self,addr):
1455         if self.is_mine(addr) and addr in self.prioritized_addresses:
1456             self.prioritized_addresses.remove(addr)
1457             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1458             return True
1459         else:
1460             return False
1461
1462
1463     def set_verifier(self, verifier):
1464         self.verifier = verifier
1465
1466         # review transactions that are in the history
1467         for addr, hist in self.history.items():
1468             if hist == ['*']: continue
1469             for tx_hash, tx_height in hist:
1470                 if tx_height>0:
1471                     # add it in case it was previously unconfirmed
1472                     self.verifier.add(tx_hash, tx_height)
1473
1474
1475         # if we are on a pruning server, remove unverified transactions
1476         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1477         for tx_hash in self.transactions.keys():
1478             if tx_hash not in vr:
1479                 self.transactions.pop(tx_hash)
1480
1481
1482
1483     def check_new_history(self, addr, hist):
1484
1485         # check that all tx in hist are relevant
1486         if hist != ['*']:
1487             for tx_hash, height in hist:
1488                 tx = self.transactions.get(tx_hash)
1489                 if not tx: continue
1490                 if not tx.has_address(addr):
1491                     return False
1492
1493         # check that we are not "orphaning" a transaction
1494         old_hist = self.history.get(addr,[])
1495         if old_hist == ['*']: return True
1496
1497         for tx_hash, height in old_hist:
1498             if tx_hash in map(lambda x:x[0], hist): continue
1499             found = False
1500             for _addr, _hist in self.history.items():
1501                 if _addr == addr: continue
1502                 if _hist == ['*']: continue
1503                 _tx_hist = map(lambda x:x[0], _hist)
1504                 if tx_hash in _tx_hist:
1505                     found = True
1506                     break
1507
1508             if not found:
1509                 tx = self.transactions.get(tx_hash)
1510                 # tx might not be there
1511                 if not tx: continue
1512
1513                 # already verified?
1514                 if self.verifier.get_height(tx_hash):
1515                     continue
1516                 # unconfirmed tx
1517                 print_error("new history is orphaning transaction:", tx_hash)
1518                 # check that all outputs are not mine, request histories
1519                 ext_requests = []
1520                 for _addr, _v in tx.outputs:
1521                     # assert not self.is_mine(_addr)
1522                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1523
1524                 ext_h = self.network.synchronous_get(ext_requests)
1525                 print_error("sync:", ext_requests, ext_h)
1526                 height = None
1527                 for h in ext_h:
1528                     if h == ['*']: continue
1529                     for item in h:
1530                         if item.get('tx_hash') == tx_hash:
1531                             height = item.get('height')
1532                 if height:
1533                     print_error("found height for", tx_hash, height)
1534                     self.verifier.add(tx_hash, height)
1535                 else:
1536                     print_error("removing orphaned tx from history", tx_hash)
1537                     self.transactions.pop(tx_hash)
1538
1539         return True
1540
1541
1542
1543     def check_new_tx(self, tx_hash, tx):
1544         # 1 check that tx is referenced in addr_history.
1545         addresses = []
1546         for addr, hist in self.history.items():
1547             if hist == ['*']:continue
1548             for txh, height in hist:
1549                 if txh == tx_hash:
1550                     addresses.append(addr)
1551
1552         if not addresses:
1553             return False
1554
1555         # 2 check that referencing addresses are in the tx
1556         for addr in addresses:
1557             if not tx.has_address(addr):
1558                 return False
1559
1560         return True
1561
1562
1563     def start_threads(self, network):
1564         from verifier import TxVerifier
1565         self.network = network
1566         if self.network:
1567             self.verifier = TxVerifier(self.network, self.storage)
1568             self.verifier.start()
1569             self.set_verifier(self.verifier)
1570             self.synchronizer = WalletSynchronizer(self, network)
1571             self.synchronizer.start()
1572         else:
1573             self.verifier = None
1574             self.synchronizer =None
1575
1576     def stop_threads(self):
1577         if self.network:
1578             self.verifier.stop()
1579             self.synchronizer.stop()
1580
1581
1582     def restore(self, callback):
1583         from i18n import _
1584         def wait_for_wallet():
1585             self.set_up_to_date(False)
1586             while not self.is_up_to_date():
1587                 msg = "%s\n%s %d\n%s %.1f"%(
1588                     _("Please wait..."),
1589                     _("Addresses generated:"),
1590                     len(self.addresses(True)),
1591                     _("Kilobytes received:"),
1592                     self.network.interface.bytes_received/1024.)
1593
1594                 apply(callback, (msg,))
1595                 time.sleep(0.1)
1596
1597         def wait_for_network():
1598             while not self.network.is_connected():
1599                 msg = "%s \n" % (_("Connecting..."))
1600                 apply(callback, (msg,))
1601                 time.sleep(0.1)
1602
1603         # wait until we are connected, because the user might have selected another server
1604         if self.network:
1605             wait_for_network()
1606
1607         self.create_accounts()
1608
1609         if self.network:
1610             wait_for_wallet()
1611         else:
1612             self.synchronize()
1613
1614         self.fill_addressbook()
1615
1616
1617
1618
1619 class WalletSynchronizer(threading.Thread):
1620
1621
1622     def __init__(self, wallet, network):
1623         threading.Thread.__init__(self)
1624         self.daemon = True
1625         self.wallet = wallet
1626         self.network = network
1627         self.was_updated = True
1628         self.running = False
1629         self.lock = threading.Lock()
1630         self.queue = Queue.Queue()
1631
1632     def stop(self):
1633         with self.lock: self.running = False
1634
1635     def is_running(self):
1636         with self.lock: return self.running
1637
1638
1639     def subscribe_to_addresses(self, addresses):
1640         messages = []
1641         for addr in addresses:
1642             messages.append(('blockchain.address.subscribe', [addr]))
1643         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1644
1645
1646     def run(self):
1647         with self.lock:
1648             self.running = True
1649
1650         while self.is_running():
1651
1652             if not self.network.is_connected():
1653                 print_error("synchronizer: waiting for interface")
1654                 self.network.wait_until_connected()
1655
1656             self.run_interface(self.network.interface)
1657
1658
1659     def run_interface(self, interface):
1660
1661         print_error("synchronizer: connected to", interface.server)
1662
1663         requested_tx = []
1664         missing_tx = []
1665         requested_histories = {}
1666
1667         # request any missing transactions
1668         for history in self.wallet.history.values():
1669             if history == ['*']: continue
1670             for tx_hash, tx_height in history:
1671                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1672                     missing_tx.append( (tx_hash, tx_height) )
1673
1674         if missing_tx:
1675             print_error("missing tx", missing_tx)
1676
1677         # subscriptions
1678         self.subscribe_to_addresses(self.wallet.addresses(True))
1679
1680         while self.is_running():
1681             # 1. create new addresses
1682             new_addresses = self.wallet.synchronize()
1683
1684             # request missing addresses
1685             if new_addresses:
1686                 self.subscribe_to_addresses(new_addresses)
1687
1688             # request missing transactions
1689             for tx_hash, tx_height in missing_tx:
1690                 if (tx_hash, tx_height) not in requested_tx:
1691                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1692                     requested_tx.append( (tx_hash, tx_height) )
1693             missing_tx = []
1694
1695             # detect if situation has changed
1696             if interface.is_up_to_date() and self.queue.empty():
1697                 if not self.wallet.is_up_to_date():
1698                     self.wallet.set_up_to_date(True)
1699                     self.was_updated = True
1700             else:
1701                 if self.wallet.is_up_to_date():
1702                     self.wallet.set_up_to_date(False)
1703                     self.was_updated = True
1704
1705             if self.was_updated:
1706                 self.wallet.network.trigger_callback('updated')
1707                 self.was_updated = False
1708
1709             # 2. get a response
1710             try:
1711                 r = self.queue.get(block=True, timeout=1)
1712             except Queue.Empty:
1713                 continue
1714
1715             if interface != self.network.interface:
1716                 break
1717
1718             if not r:
1719                 continue
1720
1721             # 3. handle response
1722             method = r['method']
1723             params = r['params']
1724             result = r.get('result')
1725             error = r.get('error')
1726             if error:
1727                 print "error", r
1728                 continue
1729
1730             if method == 'blockchain.address.subscribe':
1731                 addr = params[0]
1732                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1733                     if requested_histories.get(addr) is None:
1734                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1735                         requested_histories[addr] = result
1736
1737             elif method == 'blockchain.address.get_history':
1738                 addr = params[0]
1739                 print_error("receiving history", addr, result)
1740                 if result == ['*']:
1741                     assert requested_histories.pop(addr) == '*'
1742                     self.wallet.receive_history_callback(addr, result)
1743                 else:
1744                     hist = []
1745                     # check that txids are unique
1746                     txids = []
1747                     for item in result:
1748                         tx_hash = item['tx_hash']
1749                         if tx_hash not in txids:
1750                             txids.append(tx_hash)
1751                             hist.append( (tx_hash, item['height']) )
1752
1753                     if len(hist) != len(result):
1754                         raise BaseException("error: server sent history with non-unique txid", result)
1755
1756                     # check that the status corresponds to what was announced
1757                     rs = requested_histories.pop(addr)
1758                     if self.wallet.get_status(hist) != rs:
1759                         raise BaseException("error: status mismatch: %s"%addr)
1760
1761                     # store received history
1762                     self.wallet.receive_history_callback(addr, hist)
1763
1764                     # request transactions that we don't have
1765                     for tx_hash, tx_height in hist:
1766                         if self.wallet.transactions.get(tx_hash) is None:
1767                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1768                                 missing_tx.append( (tx_hash, tx_height) )
1769
1770             elif method == 'blockchain.transaction.get':
1771                 tx_hash = params[0]
1772                 tx_height = params[1]
1773                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1774                 tx = Transaction(result)
1775                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1776                 self.was_updated = True
1777                 requested_tx.remove( (tx_hash, tx_height) )
1778                 print_error("received tx:", tx_hash, len(tx.raw))
1779
1780             else:
1781                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1782
1783             if self.was_updated and not requested_tx:
1784                 self.wallet.network.trigger_callback('updated')
1785                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1786
1787                 self.was_updated = False