lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         return self.data.get(key, default)
 129
 130     def put(self, key, value, save = True):
 131
 132         with self.lock:
 133             if value is not None:
 134                 self.data[key] = value
 135             else:
 136                 self.data.pop(key)
 137             if save:
 138                 self.write()
 139
 140     def write(self):
 141         s = repr(self.data)
 142         f = open(self.path,"w")
 143         f.write( s )
 144         f.close()
 145         if 'ANDROID_DATA' not in os.environ:
 146             import stat
 147             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 148
 149
 150 class Wallet:
 151
 152     def __init__(self, storage):
 153
 154         self.storage = storage
 155         self.electrum_version = ELECTRUM_VERSION
 156         self.gap_limit_for_change = 3 # constant
 157
 158         # saved fields
 159         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 160
 161         self.gap_limit             = storage.get('gap_limit', 5)
 162         self.use_change            = storage.get('use_change',True)
 163         self.use_encryption        = storage.get('use_encryption', False)
 164         self.seed                  = storage.get('seed', '')               # encrypted
 165         self.labels                = storage.get('labels', {})
 166         self.frozen_addresses      = storage.get('frozen_addresses',[])
 167         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 168         self.addressbook           = storage.get('contacts', [])
 169
 170         self.imported_keys         = storage.get('imported_keys',{})
 171         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 172
 173         self.fee                   = int(storage.get('fee_per_kb',20000))
 174
 175         self.master_public_keys = storage.get('master_public_keys',{})
 176         self.master_private_keys = storage.get('master_private_keys', {})
 177
 178         self.next_addresses = storage.get('next_addresses',{})
 179
 180         if self.seed_version not in [4, 6]:
 181             msg = "This wallet seed is not supported."
 182             if self.seed_version in [5]:
 183                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 184                 print msg
 185                 sys.exit(1)
 186
 187
 188         self.load_accounts()
 189
 190         self.transactions = {}
 191         tx_list = self.storage.get('transactions',{})
 192         for k,v in tx_list.items():
 193             try:
 194                 tx = Transaction(v)
 195             except Exception:
 196                 print_msg("Warning: Cannot deserialize transactions. skipping")
 197                 continue
 198
 199             self.add_extra_addresses(tx)
 200             self.transactions[k] = tx
 201
 202         for h,tx in self.transactions.items():
 203             if not self.check_new_tx(h, tx):
 204                 print_error("removing unreferenced tx", h)
 205                 self.transactions.pop(h)
 206
 207
 208         # not saved
 209         self.prevout_values = {}     # my own transaction outputs
 210         self.spent_outputs = []
 211
 212         # spv
 213         self.verifier = None
 214
 215         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 216         # interface.is_up_to_date() returns true when all requests have been answered and processed
 217         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 218
 219         self.up_to_date = False
 220         self.lock = threading.Lock()
 221         self.transaction_lock = threading.Lock()
 222         self.tx_event = threading.Event()
 223
 224         for tx_hash, tx in self.transactions.items():
 225             self.update_tx_outputs(tx_hash)
 226
 227
 228     def add_extra_addresses(self, tx):
 229         h = tx.hash()
 230         # find the address corresponding to pay-to-pubkey inputs
 231         tx.add_extra_addresses(self.transactions)
 232         for o in tx.d.get('outputs'):
 233             if o.get('is_pubkey'):
 234                 for tx2 in self.transactions.values():
 235                     tx2.add_extra_addresses({h:tx})
 236
 237
 238
 239
 240     def set_up_to_date(self,b):
 241         with self.lock: self.up_to_date = b
 242
 243
 244     def is_up_to_date(self):
 245         with self.lock: return self.up_to_date
 246
 247
 248     def update(self):
 249         self.up_to_date = False
 250         while not self.is_up_to_date():
 251             time.sleep(0.1)
 252
 253
 254     def import_key(self, sec, password):
 255         # check password
 256         seed = self.get_seed(password)
 257         try:
 258             address = address_from_private_key(sec)
 259         except Exception:
 260             raise Exception('Invalid private key')
 261
 262         if self.is_mine(address):
 263             raise Exception('Address already in wallet')
 264
 265         # store the originally requested keypair into the imported keys table
 266         self.imported_keys[address] = pw_encode(sec, password )
 267         self.storage.put('imported_keys', self.imported_keys, True)
 268         if self.synchronizer:
 269             self.synchronizer.subscribe_to_addresses([address])
 270         return address
 271
 272     def delete_imported_key(self, addr):
 273         if addr in self.imported_keys:
 274             self.imported_keys.pop(addr)
 275             self.storage.put('imported_keys', self.imported_keys, True)
 276
 277
 278     def make_seed(self):
 279         import mnemonic, ecdsa
 280         entropy = ecdsa.util.randrange( pow(2,160) )
 281         nonce = 0
 282         while True:
 283             ss = "%040x"%(entropy+nonce)
 284             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 285             # we keep only 13 words, that's approximately 139 bits of entropy
 286             words = mnemonic.mn_encode(s)[0:13]
 287             seed = ' '.join(words)
 288             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 289                 break  # this removes 12 bits of entropy
 290             nonce += 1
 291
 292         return seed
 293
 294
 295     def init_seed(self, seed):
 296         import mnemonic
 297
 298         if self.seed:
 299             raise Exception("a seed exists")
 300
 301         if not seed:
 302             self.seed = random_seed(128)
 303             self.seed_version = 4
 304             return
 305
 306         #if not seed:
 307         #    self.seed = self.make_seed()
 308         #    self.seed_version = SEED_VERSION
 309         #    return
 310
 311         # find out what kind of wallet we are
 312         try:
 313             seed.decode('hex')
 314             self.seed_version = 4
 315             self.seed = str(seed)
 316             return
 317         except Exception:
 318             pass
 319
 320         words = seed.split()
 321         self.seed_version = 4
 322         self.seed = mnemonic.mn_decode(words)
 323
 324         #try:
 325         #    mnemonic.mn_decode(words)
 326         #    uses_electrum_words = True
 327         #except Exception:
 328         #    uses_electrum_words = False
 329         #
 330         #if uses_electrum_words and len(words) != 13:
 331         #    self.seed_version = 4
 332         #    self.seed = mnemonic.mn_decode(words)
 333         #else:
 334         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 335         #    self.seed_version = SEED_VERSION
 336         #    self.seed = seed
 337
 338
 339     def save_seed(self):
 340         self.storage.put('seed', self.seed, True)
 341         self.storage.put('seed_version', self.seed_version, True)
 342
 343     def create_watching_only_wallet(self, params):
 344         K0, c0 = params
 345         if not K0:
 346             return
 347
 348         if not c0:
 349             self.seed_version = 4
 350             self.storage.put('seed_version', self.seed_version, True)
 351             self.create_old_account(K0)
 352             return
 353
 354         cK0 = ""
 355         self.master_public_keys = {
 356             "m/0'/": (c0, K0, cK0),
 357             }
 358         self.storage.put('master_public_keys', self.master_public_keys, True)
 359         self.storage.put('seed_version', self.seed_version, True)
 360         self.create_account('1','Main account')
 361
 362
 363     def create_accounts(self):
 364         if self.seed_version == 4:
 365             mpk = OldAccount.mpk_from_seed(self.seed)
 366             self.create_old_account(mpk)
 367         else:
 368             # create default account
 369             self.create_master_keys('1')
 370             self.create_account('1','Main account')
 371
 372
 373     def create_master_keys(self, account_type):
 374         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 375         if account_type == '1':
 376             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 377             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 378             self.master_private_keys["m/0'/"] = k0
 379         elif account_type == '2of2':
 380             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 381             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 382             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 383             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 384             self.master_private_keys["m/1'/"] = k1
 385             self.master_private_keys["m/2'/"] = k2
 386         elif account_type == '2of3':
 387             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 388             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 389             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 390             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 391             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 392             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 393             self.master_private_keys["m/3'/"] = k3
 394             self.master_private_keys["m/4'/"] = k4
 395             self.master_private_keys["m/5'/"] = k5
 396
 397         self.storage.put('master_public_keys', self.master_public_keys, True)
 398         self.storage.put('master_private_keys', self.master_private_keys, True)
 399
 400     def has_master_public_keys(self, account_type):
 401         if account_type == '1':
 402             return "m/0'/" in self.master_public_keys
 403         elif account_type == '2of2':
 404             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 405         elif account_type == '2of3':
 406             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 407
 408     def find_root_by_master_key(self, c, K):
 409         for key, v in self.master_public_keys.items():
 410             if key == "m/":continue
 411             cc, KK, _ = v
 412             if (c == cc) and (K == KK):
 413                 return key
 414
 415     def deseed_root(self, seed, password):
 416         # for safety, we ask the user to enter their seed
 417         assert seed == self.get_seed(password)
 418         self.seed = ''
 419         self.storage.put('seed', '', True)
 420
 421
 422     def deseed_branch(self, k):
 423         # check that parent has no seed
 424         assert self.seed == ''
 425         self.master_private_keys.pop(k)
 426         self.storage.put('master_private_keys', self.master_private_keys, True)
 427
 428     def is_watching_only(self):
 429         return (self.seed == '') and (self.master_private_keys == {})
 430
 431
 432
 433     def account_id(self, account_type, i):
 434         if account_type == '1':
 435             return "m/0'/%d"%i
 436         elif account_type == '2of2':
 437             return "m/1'/%d & m/2'/%d"%(i,i)
 438         elif account_type == '2of3':
 439             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 440         else:
 441             raise Exception('unknown account type')
 442
 443
 444     def num_accounts(self, account_type):
 445         keys = self.accounts.keys()
 446         i = 0
 447         while True:
 448             account_id = self.account_id(account_type, i)
 449             if account_id not in keys: break
 450             i += 1
 451         return i
 452
 453
 454     def new_account_address(self, account_type = '1'):
 455         i = self.num_accounts(account_type)
 456         k = self.account_id(account_type,i)
 457
 458         addr = self.next_addresses.get(k)
 459         if not addr:
 460             account_id, account = self.next_account(account_type)
 461             addr = account.first_address()
 462             self.next_addresses[k] = addr
 463             self.storage.put('next_addresses',self.next_addresses)
 464
 465         return k, addr
 466
 467
 468     def next_account(self, account_type = '1'):
 469
 470         i = self.num_accounts(account_type)
 471         account_id = self.account_id(account_type,i)
 472
 473         if account_type is '1':
 474             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 475             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 476             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 477
 478         elif account_type == '2of2':
 479             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 480             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 481             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 482             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 483             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 484
 485         elif account_type == '2of3':
 486             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 487             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 488             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 489             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 490             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 491             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 492             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 493
 494         return account_id, account
 495
 496
 497     def set_label(self, name, text = None):
 498         changed = False
 499         old_text = self.labels.get(name)
 500         if text:
 501             if old_text != text:
 502                 self.labels[name] = text
 503                 changed = True
 504         else:
 505             if old_text:
 506                 self.labels.pop(name)
 507                 changed = True
 508
 509         if changed:
 510             self.storage.put('labels', self.labels, True)
 511
 512         run_hook('set_label', name, text, changed)
 513         return changed
 514
 515
 516
 517     def create_account(self, account_type = '1', name = None):
 518         k, account = self.next_account(account_type)
 519         if k in self.pending_accounts:
 520             self.pending_accounts.pop(k)
 521             self.storage.put('pending_accounts', self.pending_accounts)
 522
 523         self.accounts[k] = account
 524         self.save_accounts()
 525         if name:
 526             self.set_label(k, name)
 527
 528
 529     def create_old_account(self, mpk):
 530         self.storage.put('master_public_key', mpk, True)
 531         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 532         self.save_accounts()
 533
 534
 535     def save_accounts(self):
 536         d = {}
 537         for k, v in self.accounts.items():
 538             d[k] = v.dump()
 539         self.storage.put('accounts', d, True)
 540
 541
 542
 543     def load_accounts(self):
 544         d = self.storage.get('accounts', {})
 545         self.accounts = {}
 546         for k, v in d.items():
 547             if k == 0:
 548                 v['mpk'] = self.storage.get('master_public_key')
 549                 self.accounts[k] = OldAccount(v)
 550             elif '&' in k:
 551                 self.accounts[k] = BIP32_Account_2of2(v)
 552             else:
 553                 self.accounts[k] = BIP32_Account(v)
 554
 555         self.pending_accounts = self.storage.get('pending_accounts',{})
 556
 557
 558     def delete_pending_account(self, k):
 559         self.pending_accounts.pop(k)
 560         self.storage.put('pending_accounts', self.pending_accounts)
 561
 562     def account_is_pending(self, k):
 563         return k in self.pending_accounts
 564
 565     def create_pending_account(self, acct_type, name):
 566         k, addr = self.new_account_address(acct_type)
 567         self.set_label(k, name)
 568         self.pending_accounts[k] = addr
 569         self.storage.put('pending_accounts', self.pending_accounts)
 570
 571     def get_pending_accounts(self):
 572         return self.pending_accounts.items()
 573
 574
 575     def addresses(self, include_change = True, _next=True):
 576         o = self.get_account_addresses(-1, include_change)
 577         for a in self.accounts.keys():
 578             o += self.get_account_addresses(a, include_change)
 579
 580         if _next:
 581             for addr in self.next_addresses.values():
 582                 if addr not in o:
 583                     o += [addr]
 584         return o
 585
 586
 587     def is_mine(self, address):
 588         return address in self.addresses(True)
 589
 590
 591     def is_change(self, address):
 592         if not self.is_mine(address): return False
 593         if address in self.imported_keys.keys(): return False
 594         acct, s = self.get_address_index(address)
 595         if s is None: return False
 596         return s[0] == 1
 597
 598     def get_master_public_key(self):
 599         if self.seed_version == 4:
 600             return self.storage.get("master_public_key")
 601         else:
 602             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 603             return repr((c, K))
 604
 605     def get_master_private_key(self, account, password):
 606         master_k = pw_decode( self.master_private_keys[account], password)
 607         master_c, master_K, master_Kc = self.master_public_keys[account]
 608         try:
 609             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 610             assert K.encode('hex') == master_K
 611         except Exception:
 612             raise Exception("Invalid password")
 613         return master_k
 614
 615
 616     def get_address_index(self, address):
 617         if address in self.imported_keys.keys():
 618             return -1, None
 619
 620         for account in self.accounts.keys():
 621             for for_change in [0,1]:
 622                 addresses = self.accounts[account].get_addresses(for_change)
 623                 for addr in addresses:
 624                     if address == addr:
 625                         return account, (for_change, addresses.index(addr))
 626
 627         for k,v in self.next_addresses.items():
 628             if v == address:
 629                 return k, (0,0)
 630
 631         raise Exception("Address not found", address)
 632
 633
 634     def get_roots(self, account):
 635         roots = []
 636         for a in account.split('&'):
 637             s = a.strip()
 638             m = re.match("(m/\d+'/)(\d+)", s)
 639             roots.append( m.group(1) )
 640         return roots
 641
 642     def is_seeded(self, account):
 643         if type(account) is int:
 644             return self.seed is not None
 645
 646         for root in self.get_roots(account):
 647             if root not in self.master_private_keys.keys():
 648                 return False
 649         return True
 650
 651     def rebase_sequence(self, account, sequence):
 652         c, i = sequence
 653         dd = []
 654         for a in account.split('&'):
 655             s = a.strip()
 656             m = re.match("(m/\d+'/)(\d+)", s)
 657             root = m.group(1)
 658             num = int(m.group(2))
 659             dd.append( (root, [num,c,i] ) )
 660         return dd
 661
 662
 663     def get_keyID(self, account, sequence):
 664         if account == 0:
 665             a, b = sequence
 666             mpk = self.storage.get('master_public_key')
 667             return 'old(%s,%d,%d)'%(mpk,a,b)
 668
 669         rs = self.rebase_sequence(account, sequence)
 670         dd = []
 671         for root, public_sequence in rs:
 672             c, K, _ = self.master_public_keys[root]
 673             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 674             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 675         return '&'.join(dd)
 676
 677
 678
 679     def get_seed(self, password):
 680         s = pw_decode(self.seed, password)
 681         if self.seed_version == 4:
 682             seed = s
 683             self.accounts[0].check_seed(seed)
 684         else:
 685             seed = mnemonic_hash(s)
 686         return seed
 687
 688
 689     def get_mnemonic(self, password):
 690         import mnemonic
 691         s = pw_decode(self.seed, password)
 692         if self.seed_version == 4:
 693             return ' '.join(mnemonic.mn_encode(s))
 694         else:
 695             return s
 696
 697
 698
 699     def get_private_key(self, address, password):
 700         # first check the provided password
 701         seed = self.get_seed(password)
 702
 703         out = []
 704         if address in self.imported_keys.keys():
 705             out.append( pw_decode( self.imported_keys[address], password ) )
 706         else:
 707             account, sequence = self.get_address_index(address)
 708             if account == 0:
 709                 pk = self.accounts[account].get_private_key(seed, sequence)
 710                 out.append(pk)
 711                 return out
 712
 713             # assert address == self.accounts[account].get_address(*sequence)
 714             rs = self.rebase_sequence( account, sequence)
 715             for root, public_sequence in rs:
 716
 717                 if root not in self.master_private_keys.keys(): continue
 718                 master_k = self.get_master_private_key(root, password)
 719                 master_c, _, _ = self.master_public_keys[root]
 720                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 721                 out.append(pk)
 722
 723         return out
 724
 725
 726     def add_keypairs_from_wallet(self, tx, keypairs, password):
 727         for txin in tx.inputs:
 728             address = txin['address']
 729             private_keys = self.get_private_key(address, password)
 730             for sec in private_keys:
 731                 pubkey = public_key_from_private_key(sec)
 732                 keypairs[ pubkey ] = sec
 733                 if address in self.imported_keys.keys():
 734                     txin['redeemPubkey'] = pubkey
 735
 736
 737     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 738         for txin in tx.inputs:
 739             keyid = txin.get('KeyID')
 740             if keyid:
 741
 742                 if self.seed_version==4:
 743                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 744                     if not m: continue
 745                     mpk = m.group(1)
 746                     if mpk != self.storage.get('master_public_key'): continue
 747                     index = int(m.group(2))
 748                     num = int(m.group(3))
 749                     account = self.accounts[0]
 750                     addr = account.get_address(index, num)
 751                     txin['address'] = addr # fixme: side effect
 752                     pk = self.get_private_key(addr, password)
 753                     for sec in pk:
 754                         pubkey = public_key_from_private_key(sec)
 755                         keypairs[pubkey] = sec
 756                     continue
 757
 758
 759                 roots = []
 760                 for s in keyid.split('&'):
 761                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 762                     if not m: continue
 763                     c = m.group(1)
 764                     K = m.group(2)
 765                     sequence = m.group(3)
 766                     root = self.find_root_by_master_key(c,K)
 767                     if not root: continue
 768                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 769                     root = root + '%d'%sequence[0]
 770                     sequence = sequence[1:]
 771                     roots.append((root,sequence))
 772
 773                 account_id = " & ".join( map(lambda x:x[0], roots) )
 774                 account = self.accounts.get(account_id)
 775                 if not account: continue
 776                 addr = account.get_address(*sequence)
 777                 txin['address'] = addr # fixme: side effect
 778                 pk = self.get_private_key(addr, password)
 779                 for sec in pk:
 780                     pubkey = public_key_from_private_key(sec)
 781                     keypairs[pubkey] = sec
 782
 783
 784
 785     def signrawtransaction(self, tx, input_info, private_keys, password):
 786
 787         # check that the password is correct
 788         seed = self.get_seed(password)
 789
 790         # add input info
 791         tx.add_input_info(input_info)
 792
 793         # add redeem script for coins that are in the wallet
 794         # FIXME: add redeemPubkey too!
 795         unspent_coins = self.get_unspent_coins()
 796         for txin in tx.inputs:
 797             for item in unspent_coins:
 798                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 799                     print_error( "tx input is in unspent coins" )
 800                     txin['scriptPubKey'] = item['scriptPubKey']
 801                     account, sequence = self.get_address_index(item['address'])
 802                     if account != -1:
 803                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 804                         print_error("added redeemScript", txin['redeemScript'])
 805                     break
 806
 807
 808         # build a list of public/private keys
 809         keypairs = {}
 810
 811         # add private keys from parameter
 812         for sec in private_keys:
 813             pubkey = public_key_from_private_key(sec)
 814             keypairs[ pubkey ] = sec
 815
 816         # add private_keys from KeyID
 817         self.add_keypairs_from_KeyID(tx, keypairs, password)
 818
 819         # add private keys from wallet
 820         self.add_keypairs_from_wallet(tx, keypairs, password)
 821         self.sign_transaction(tx, keypairs)
 822
 823
 824     def sign_message(self, address, message, password):
 825         keys = self.get_private_key(address, password)
 826         assert len(keys) == 1
 827         sec = keys[0]
 828         key = regenerate_key(sec)
 829         compressed = is_compressed(sec)
 830         return key.sign_message(message, compressed, address)
 831
 832
 833     def change_gap_limit(self, value):
 834         if value >= self.gap_limit:
 835             self.gap_limit = value
 836             self.storage.put('gap_limit', self.gap_limit, True)
 837             #self.interface.poke('synchronizer')
 838             return True
 839
 840         elif value >= self.min_acceptable_gap():
 841             for key, account in self.accounts.items():
 842                 addresses = account[0]
 843                 k = self.num_unused_trailing_addresses(addresses)
 844                 n = len(addresses) - k + value
 845                 addresses = addresses[0:n]
 846                 self.accounts[key][0] = addresses
 847
 848             self.gap_limit = value
 849             self.storage.put('gap_limit', self.gap_limit, True)
 850             self.save_accounts()
 851             return True
 852         else:
 853             return False
 854
 855     def num_unused_trailing_addresses(self, addresses):
 856         k = 0
 857         for a in addresses[::-1]:
 858             if self.history.get(a):break
 859             k = k + 1
 860         return k
 861
 862     def min_acceptable_gap(self):
 863         # fixme: this assumes wallet is synchronized
 864         n = 0
 865         nmax = 0
 866
 867         for account in self.accounts.values():
 868             addresses = account.get_addresses(0)
 869             k = self.num_unused_trailing_addresses(addresses)
 870             for a in addresses[0:-k]:
 871                 if self.history.get(a):
 872                     n = 0
 873                 else:
 874                     n += 1
 875                     if n > nmax: nmax = n
 876         return nmax + 1
 877
 878
 879     def address_is_old(self, address):
 880         age = -1
 881         h = self.history.get(address, [])
 882         if h == ['*']:
 883             return True
 884         for tx_hash, tx_height in h:
 885             if tx_height == 0:
 886                 tx_age = 0
 887             else:
 888                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 889             if tx_age > age:
 890                 age = tx_age
 891         return age > 2
 892
 893
 894     def synchronize_sequence(self, account, for_change):
 895         limit = self.gap_limit_for_change if for_change else self.gap_limit
 896         new_addresses = []
 897         while True:
 898             addresses = account.get_addresses(for_change)
 899             if len(addresses) < limit:
 900                 address = account.create_new_address(for_change)
 901                 self.history[address] = []
 902                 new_addresses.append( address )
 903                 continue
 904
 905             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 906                 break
 907             else:
 908                 address = account.create_new_address(for_change)
 909                 self.history[address] = []
 910                 new_addresses.append( address )
 911
 912         return new_addresses
 913
 914
 915
 916     def create_pending_accounts(self):
 917         for account_type in ['1','2of2','2of3']:
 918             if not self.has_master_public_keys(account_type):
 919                 continue
 920             k, a = self.new_account_address(account_type)
 921             if self.address_is_old(a):
 922                 print_error( "creating account", a )
 923                 self.create_account(account_type)
 924                 self.next_addresses.pop(k)
 925
 926
 927     def synchronize_account(self, account):
 928         new = []
 929         new += self.synchronize_sequence(account, 0)
 930         new += self.synchronize_sequence(account, 1)
 931         return new
 932
 933
 934     def synchronize(self):
 935         if self.master_public_keys:
 936             self.create_pending_accounts()
 937         new = []
 938         for account in self.accounts.values():
 939             new += self.synchronize_account(account)
 940         if new:
 941             self.save_accounts()
 942             self.storage.put('addr_history', self.history, True)
 943         return new
 944
 945
 946     def is_found(self):
 947         return self.history.values() != [[]] * len(self.history)
 948
 949
 950     def add_contact(self, address, label=None):
 951         self.addressbook.append(address)
 952         self.storage.put('contacts', self.addressbook, True)
 953         if label:
 954             self.set_label(address, label)
 955
 956
 957     def delete_contact(self, addr):
 958         if addr in self.addressbook:
 959             self.addressbook.remove(addr)
 960             self.storage.put('addressbook', self.addressbook, True)
 961
 962
 963     def fill_addressbook(self):
 964         for tx_hash, tx in self.transactions.items():
 965             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 966             if is_send:
 967                 for addr, v in tx.outputs:
 968                     if not self.is_mine(addr) and addr not in self.addressbook:
 969                         self.addressbook.append(addr)
 970         # redo labels
 971         # self.update_tx_labels()
 972
 973     def get_num_tx(self, address):
 974         n = 0
 975         for tx in self.transactions.values():
 976             if address in map(lambda x:x[0], tx.outputs): n += 1
 977         return n
 978
 979
 980     def get_address_flags(self, addr):
 981         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 982         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 983         return flags
 984
 985
 986     def get_tx_value(self, tx, account=None):
 987         domain = self.get_account_addresses(account)
 988         return tx.get_value(domain, self.prevout_values)
 989
 990
 991     def update_tx_outputs(self, tx_hash):
 992         tx = self.transactions.get(tx_hash)
 993
 994         for i, (addr, value) in enumerate(tx.outputs):
 995             key = tx_hash+ ':%d'%i
 996             self.prevout_values[key] = value
 997
 998         for item in tx.inputs:
 999             if self.is_mine(item.get('address')):
1000                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1001                 self.spent_outputs.append(key)
1002
1003
1004     def get_addr_balance(self, address):
1005         assert self.is_mine(address)
1006         h = self.history.get(address,[])
1007         if h == ['*']: return 0,0
1008         c = u = 0
1009         received_coins = []   # list of coins received at address
1010
1011         for tx_hash, tx_height in h:
1012             tx = self.transactions.get(tx_hash)
1013             if not tx: continue
1014
1015             for i, (addr, value) in enumerate(tx.outputs):
1016                 if addr == address:
1017                     key = tx_hash + ':%d'%i
1018                     received_coins.append(key)
1019
1020         for tx_hash, tx_height in h:
1021             tx = self.transactions.get(tx_hash)
1022             if not tx: continue
1023             v = 0
1024
1025             for item in tx.inputs:
1026                 addr = item.get('address')
1027                 if addr == address:
1028                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1029                     value = self.prevout_values.get( key )
1030                     if key in received_coins:
1031                         v -= value
1032
1033             for i, (addr, value) in enumerate(tx.outputs):
1034                 key = tx_hash + ':%d'%i
1035                 if addr == address:
1036                     v += value
1037
1038             if tx_height:
1039                 c += v
1040             else:
1041                 u += v
1042         return c, u
1043
1044
1045     def get_account_name(self, k):
1046         if k == 0:
1047             if self.seed_version == 4:
1048                 name = 'Main account'
1049             else:
1050                 name = 'Old account'
1051         else:
1052             default = "Unnamed account"
1053             m = re.match("m/0'/(\d+)", k)
1054             if m:
1055                 num = m.group(1)
1056                 if num == '0':
1057                     default = "Main account"
1058                 else:
1059                     default = "Account %s"%num
1060
1061             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1062             if m:
1063                 num = m.group(1)
1064                 default = "2of2 account %s"%num
1065             name = self.labels.get(k, default)
1066
1067         return name
1068
1069     def get_account_names(self):
1070         accounts = {}
1071         for k, account in self.accounts.items():
1072             accounts[k] = self.get_account_name(k)
1073         if self.imported_keys:
1074             accounts[-1] = 'Imported keys'
1075         return accounts
1076
1077     def get_account_addresses(self, a, include_change=True):
1078         if a is None:
1079             o = self.addresses(True)
1080         elif a == -1:
1081             o = self.imported_keys.keys()
1082         else:
1083             ac = self.accounts[a]
1084             o = ac.get_addresses(0)
1085             if include_change: o += ac.get_addresses(1)
1086         return o
1087
1088     def get_imported_balance(self):
1089         return self.get_balance(self.imported_keys.keys())
1090
1091     def get_account_balance(self, account):
1092         return self.get_balance(self.get_account_addresses(account))
1093
1094     def get_frozen_balance(self):
1095         return self.get_balance(self.frozen_addresses)
1096
1097     def get_balance(self, domain=None):
1098         if domain is None: domain = self.addresses(True)
1099         cc = uu = 0
1100         for addr in domain:
1101             c, u = self.get_addr_balance(addr)
1102             cc += c
1103             uu += u
1104         return cc, uu
1105
1106
1107     def get_unspent_coins(self, domain=None):
1108         coins = []
1109         if domain is None: domain = self.addresses(True)
1110         for addr in domain:
1111             h = self.history.get(addr, [])
1112             if h == ['*']: continue
1113             for tx_hash, tx_height in h:
1114                 tx = self.transactions.get(tx_hash)
1115                 if tx is None: raise Exception("Wallet not synchronized")
1116                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1117                 for output in tx.d.get('outputs'):
1118                     if output.get('address') != addr: continue
1119                     key = tx_hash + ":%d" % output.get('prevout_n')
1120                     if key in self.spent_outputs: continue
1121                     output['prevout_hash'] = tx_hash
1122                     output['height'] = tx_height
1123                     output['coinbase'] = is_coinbase
1124                     coins.append((tx_height, output))
1125
1126         # sort by age
1127         if coins:
1128             coins = sorted(coins)
1129             if coins[-1][0] != 0:
1130                 while coins[0][0] == 0:
1131                     coins = coins[1:] + [ coins[0] ]
1132         return [x[1] for x in coins]
1133
1134
1135
1136     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1137         domain = self.get_account_addresses(account) if account else None
1138         return self.choose_tx_inputs( amount, fixed_fee, domain )
1139
1140
1141     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1142         """ todo: minimize tx size """
1143         total = 0
1144         fee = self.fee if fixed_fee is None else fixed_fee
1145         if domain is None:
1146             domain = self.addresses(True)
1147
1148         for i in self.frozen_addresses:
1149             if i in domain: domain.remove(i)
1150
1151         prioritized = []
1152         for i in self.prioritized_addresses:
1153             if i in domain:
1154                 domain.remove(i)
1155                 prioritized.append(i)
1156
1157         coins = self.get_unspent_coins(domain)
1158         prioritized_coins = self.get_unspent_coins(prioritized)
1159
1160         inputs = []
1161         coins = prioritized_coins + coins
1162
1163         for item in coins:
1164             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1165                 continue
1166             addr = item.get('address')
1167             v = item.get('value')
1168             total += v
1169             inputs.append(item)
1170             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1171             if total >= amount + fee: break
1172         else:
1173             inputs = []
1174
1175         return inputs, total, fee
1176
1177
1178     def set_fee(self, fee):
1179         if self.fee != fee:
1180             self.fee = fee
1181             self.storage.put('fee_per_kb', self.fee, True)
1182
1183     def estimated_fee(self, inputs):
1184         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1185         fee = self.fee * int(round(estimated_size/1024.))
1186         if fee == 0: fee = self.fee
1187         return fee
1188
1189
1190     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1191         "add change to a transaction"
1192         change_amount = total - ( amount + fee )
1193         if change_amount > DUST_THRESHOLD:
1194             if not change_addr:
1195
1196                 # send change to one of the accounts involved in the tx
1197                 address = inputs[0].get('address')
1198                 account, _ = self.get_address_index(address)
1199
1200                 if not self.use_change or account == -1:
1201                     change_addr = inputs[-1]['address']
1202                 else:
1203                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1204
1205             # Insert the change output at a random position in the outputs
1206             posn = random.randint(0, len(outputs))
1207             outputs[posn:posn] = [( change_addr,  change_amount)]
1208         return outputs
1209
1210
1211     def get_history(self, address):
1212         with self.lock:
1213             return self.history.get(address)
1214
1215
1216     def get_status(self, h):
1217         if not h: return None
1218         if h == ['*']: return '*'
1219         status = ''
1220         for tx_hash, height in h:
1221             status += tx_hash + ':%d:' % height
1222         return hashlib.sha256( status ).digest().encode('hex')
1223
1224
1225     def receive_tx_callback(self, tx_hash, tx, tx_height):
1226
1227         with self.transaction_lock:
1228             self.add_extra_addresses(tx)
1229             if not self.check_new_tx(tx_hash, tx):
1230                 # may happen due to pruning
1231                 print_error("received transaction that is no longer referenced in history", tx_hash)
1232                 return
1233             self.transactions[tx_hash] = tx
1234             self.network.interface.pending_transactions_for_notifications.append(tx)
1235             self.save_transactions()
1236             if self.verifier and tx_height>0:
1237                 self.verifier.add(tx_hash, tx_height)
1238             self.update_tx_outputs(tx_hash)
1239
1240
1241     def save_transactions(self):
1242         tx = {}
1243         for k,v in self.transactions.items():
1244             tx[k] = str(v)
1245         self.storage.put('transactions', tx, True)
1246
1247     def receive_history_callback(self, addr, hist):
1248
1249         if not self.check_new_history(addr, hist):
1250             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1251
1252         with self.lock:
1253             self.history[addr] = hist
1254             self.storage.put('addr_history', self.history, True)
1255
1256         if hist != ['*']:
1257             for tx_hash, tx_height in hist:
1258                 if tx_height>0:
1259                     # add it in case it was previously unconfirmed
1260                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1261
1262
1263     def get_tx_history(self, account=None):
1264         if not self.verifier:
1265             return []
1266
1267         with self.transaction_lock:
1268             history = self.transactions.items()
1269             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1270             result = []
1271
1272             balance = 0
1273             for tx_hash, tx in history:
1274                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1275                 if v is not None: balance += v
1276
1277             c, u = self.get_account_balance(account)
1278
1279             if balance != c+u:
1280                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1281
1282             balance = c + u - balance
1283             for tx_hash, tx in history:
1284                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1285                 if not is_relevant:
1286                     continue
1287                 if value is not None:
1288                     balance += value
1289
1290                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1291                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1292
1293         return result
1294
1295
1296     def get_label(self, tx_hash):
1297         label = self.labels.get(tx_hash)
1298         is_default = (label == '') or (label is None)
1299         if is_default: label = self.get_default_label(tx_hash)
1300         return label, is_default
1301
1302
1303     def get_default_label(self, tx_hash):
1304         tx = self.transactions.get(tx_hash)
1305         default_label = ''
1306         if tx:
1307             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1308             if is_mine:
1309                 for o in tx.outputs:
1310                     o_addr, _ = o
1311                     if not self.is_mine(o_addr):
1312                         try:
1313                             default_label = self.labels[o_addr]
1314                         except KeyError:
1315                             default_label = o_addr
1316                         break
1317                 else:
1318                     default_label = '(internal)'
1319             else:
1320                 for o in tx.outputs:
1321                     o_addr, _ = o
1322                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1323                         break
1324                 else:
1325                     for o in tx.outputs:
1326                         o_addr, _ = o
1327                         if self.is_mine(o_addr):
1328                             break
1329                     else:
1330                         o_addr = None
1331
1332                 if o_addr:
1333                     dest_label = self.labels.get(o_addr)
1334                     try:
1335                         default_label = self.labels[o_addr]
1336                     except KeyError:
1337                         default_label = o_addr
1338
1339         return default_label
1340
1341
1342     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1343         for address, x in outputs:
1344             assert is_valid(address)
1345         amount = sum( map(lambda x:x[1], outputs) )
1346         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1347         if not inputs:
1348             raise ValueError("Not enough funds")
1349         self.add_input_info(inputs)
1350         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1351         return Transaction.from_io(inputs, outputs)
1352
1353
1354     def mktx_from_account(self, outputs, password, fee=None, account=None):
1355         domain = self.get_account_addresses(account) if account else None
1356         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1357
1358
1359     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1360         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1361         keypairs = {}
1362         self.add_keypairs_from_wallet(tx, keypairs, password)
1363         if keypairs:
1364             self.sign_transaction(tx, keypairs)
1365         return tx
1366
1367
1368     def add_input_info(self, inputs):
1369         for txin in inputs:
1370             address = txin['address']
1371             if address in self.imported_keys.keys():
1372                 continue
1373             account, sequence = self.get_address_index(address)
1374             txin['KeyID'] = self.get_keyID(account, sequence)
1375             redeemScript = self.accounts[account].redeem_script(sequence)
1376             if redeemScript:
1377                 txin['redeemScript'] = redeemScript
1378             else:
1379                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1380
1381
1382     def sign_transaction(self, tx, keypairs):
1383         tx.sign(keypairs)
1384         run_hook('sign_transaction', tx)
1385
1386
1387     def sendtx(self, tx):
1388         # synchronous
1389         h = self.send_tx(tx)
1390         self.tx_event.wait()
1391         return self.receive_tx(h)
1392
1393     def send_tx(self, tx):
1394         # asynchronous
1395         self.tx_event.clear()
1396         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1397         return tx.hash()
1398
1399     def on_broadcast(self, i, r):
1400         self.tx_result = r.get('result')
1401         self.tx_event.set()
1402
1403     def receive_tx(self,tx_hash):
1404         out = self.tx_result
1405         if out != tx_hash:
1406             return False, "error: " + out
1407         return True, out
1408
1409
1410
1411     def update_password(self, old_password, new_password):
1412         if new_password == '': new_password = None
1413         decoded = self.get_seed(old_password)
1414         self.seed = pw_encode( decoded, new_password)
1415         self.storage.put('seed', self.seed, True)
1416         self.use_encryption = (new_password != None)
1417         self.storage.put('use_encryption', self.use_encryption,True)
1418         for k in self.imported_keys.keys():
1419             a = self.imported_keys[k]
1420             b = pw_decode(a, old_password)
1421             c = pw_encode(b, new_password)
1422             self.imported_keys[k] = c
1423         self.storage.put('imported_keys', self.imported_keys, True)
1424
1425         for k, v in self.master_private_keys.items():
1426             b = pw_decode(v, old_password)
1427             c = pw_encode(b, new_password)
1428             self.master_private_keys[k] = c
1429         self.storage.put('master_private_keys', self.master_private_keys, True)
1430
1431
1432     def freeze(self,addr):
1433         if self.is_mine(addr) and addr not in self.frozen_addresses:
1434             self.unprioritize(addr)
1435             self.frozen_addresses.append(addr)
1436             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1437             return True
1438         else:
1439             return False
1440
1441     def unfreeze(self,addr):
1442         if self.is_mine(addr) and addr in self.frozen_addresses:
1443             self.frozen_addresses.remove(addr)
1444             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1445             return True
1446         else:
1447             return False
1448
1449     def prioritize(self,addr):
1450         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1451             self.unfreeze(addr)
1452             self.prioritized_addresses.append(addr)
1453             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1454             return True
1455         else:
1456             return False
1457
1458     def unprioritize(self,addr):
1459         if self.is_mine(addr) and addr in self.prioritized_addresses:
1460             self.prioritized_addresses.remove(addr)
1461             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1462             return True
1463         else:
1464             return False
1465
1466
1467     def set_verifier(self, verifier):
1468         self.verifier = verifier
1469
1470         # review transactions that are in the history
1471         for addr, hist in self.history.items():
1472             if hist == ['*']: continue
1473             for tx_hash, tx_height in hist:
1474                 if tx_height>0:
1475                     # add it in case it was previously unconfirmed
1476                     self.verifier.add(tx_hash, tx_height)
1477
1478
1479         # if we are on a pruning server, remove unverified transactions
1480         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1481         for tx_hash in self.transactions.keys():
1482             if tx_hash not in vr:
1483                 self.transactions.pop(tx_hash)
1484
1485
1486
1487     def check_new_history(self, addr, hist):
1488
1489         # check that all tx in hist are relevant
1490         if hist != ['*']:
1491             for tx_hash, height in hist:
1492                 tx = self.transactions.get(tx_hash)
1493                 if not tx: continue
1494                 if not tx.has_address(addr):
1495                     return False
1496
1497         # check that we are not "orphaning" a transaction
1498         old_hist = self.history.get(addr,[])
1499         if old_hist == ['*']: return True
1500
1501         for tx_hash, height in old_hist:
1502             if tx_hash in map(lambda x:x[0], hist): continue
1503             found = False
1504             for _addr, _hist in self.history.items():
1505                 if _addr == addr: continue
1506                 if _hist == ['*']: continue
1507                 _tx_hist = map(lambda x:x[0], _hist)
1508                 if tx_hash in _tx_hist:
1509                     found = True
1510                     break
1511
1512             if not found:
1513                 tx = self.transactions.get(tx_hash)
1514                 # tx might not be there
1515                 if not tx: continue
1516
1517                 # already verified?
1518                 if self.verifier.get_height(tx_hash):
1519                     continue
1520                 # unconfirmed tx
1521                 print_error("new history is orphaning transaction:", tx_hash)
1522                 # check that all outputs are not mine, request histories
1523                 ext_requests = []
1524                 for _addr, _v in tx.outputs:
1525                     # assert not self.is_mine(_addr)
1526                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1527
1528                 ext_h = self.network.synchronous_get(ext_requests)
1529                 print_error("sync:", ext_requests, ext_h)
1530                 height = None
1531                 for h in ext_h:
1532                     if h == ['*']: continue
1533                     for item in h:
1534                         if item.get('tx_hash') == tx_hash:
1535                             height = item.get('height')
1536                 if height:
1537                     print_error("found height for", tx_hash, height)
1538                     self.verifier.add(tx_hash, height)
1539                 else:
1540                     print_error("removing orphaned tx from history", tx_hash)
1541                     self.transactions.pop(tx_hash)
1542
1543         return True
1544
1545
1546
1547     def check_new_tx(self, tx_hash, tx):
1548         # 1 check that tx is referenced in addr_history.
1549         addresses = []
1550         for addr, hist in self.history.items():
1551             if hist == ['*']:continue
1552             for txh, height in hist:
1553                 if txh == tx_hash:
1554                     addresses.append(addr)
1555
1556         if not addresses:
1557             return False
1558
1559         # 2 check that referencing addresses are in the tx
1560         for addr in addresses:
1561             if not tx.has_address(addr):
1562                 return False
1563
1564         return True
1565
1566
1567     def start_threads(self, network):
1568         from verifier import TxVerifier
1569         self.network = network
1570         if self.network:
1571             self.verifier = TxVerifier(self.network, self.storage)
1572             self.verifier.start()
1573             self.set_verifier(self.verifier)
1574             self.synchronizer = WalletSynchronizer(self, network)
1575             self.synchronizer.start()
1576         else:
1577             self.verifier = None
1578             self.synchronizer =None
1579
1580     def stop_threads(self):
1581         if self.network:
1582             self.verifier.stop()
1583             self.synchronizer.stop()
1584
1585
1586     def restore(self, callback):
1587         from i18n import _
1588         def wait_for_wallet():
1589             self.set_up_to_date(False)
1590             while not self.is_up_to_date():
1591                 msg = "%s\n%s %d\n%s %.1f"%(
1592                     _("Please wait..."),
1593                     _("Addresses generated:"),
1594                     len(self.addresses(True)),
1595                     _("Kilobytes received:"),
1596                     self.network.interface.bytes_received/1024.)
1597
1598                 apply(callback, (msg,))
1599                 time.sleep(0.1)
1600
1601         def wait_for_network():
1602             while not self.network.is_connected():
1603                 msg = "%s \n" % (_("Connecting..."))
1604                 apply(callback, (msg,))
1605                 time.sleep(0.1)
1606
1607         # wait until we are connected, because the user might have selected another server
1608         if self.network:
1609             wait_for_network()
1610
1611         self.create_accounts()
1612
1613         if self.network:
1614             wait_for_wallet()
1615         else:
1616             self.synchronize()
1617
1618         self.fill_addressbook()
1619
1620
1621
1622
1623 class WalletSynchronizer(threading.Thread):
1624
1625
1626     def __init__(self, wallet, network):
1627         threading.Thread.__init__(self)
1628         self.daemon = True
1629         self.wallet = wallet
1630         self.network = network
1631         self.was_updated = True
1632         self.running = False
1633         self.lock = threading.Lock()
1634         self.queue = Queue.Queue()
1635
1636     def stop(self):
1637         with self.lock: self.running = False
1638
1639     def is_running(self):
1640         with self.lock: return self.running
1641
1642
1643     def subscribe_to_addresses(self, addresses):
1644         messages = []
1645         for addr in addresses:
1646             messages.append(('blockchain.address.subscribe', [addr]))
1647         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1648
1649
1650     def run(self):
1651         with self.lock:
1652             self.running = True
1653
1654         while self.is_running():
1655
1656             if not self.network.is_connected():
1657                 print_error("synchronizer: waiting for interface")
1658                 self.network.wait_until_connected()
1659
1660             self.run_interface(self.network.interface)
1661
1662
1663     def run_interface(self, interface):
1664
1665         print_error("synchronizer: connected to", interface.server)
1666
1667         requested_tx = []
1668         missing_tx = []
1669         requested_histories = {}
1670
1671         # request any missing transactions
1672         for history in self.wallet.history.values():
1673             if history == ['*']: continue
1674             for tx_hash, tx_height in history:
1675                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1676                     missing_tx.append( (tx_hash, tx_height) )
1677
1678         if missing_tx:
1679             print_error("missing tx", missing_tx)
1680
1681         # subscriptions
1682         self.subscribe_to_addresses(self.wallet.addresses(True))
1683
1684         while self.is_running():
1685             # 1. create new addresses
1686             new_addresses = self.wallet.synchronize()
1687
1688             # request missing addresses
1689             if new_addresses:
1690                 self.subscribe_to_addresses(new_addresses)
1691
1692             # request missing transactions
1693             for tx_hash, tx_height in missing_tx:
1694                 if (tx_hash, tx_height) not in requested_tx:
1695                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1696                     requested_tx.append( (tx_hash, tx_height) )
1697             missing_tx = []
1698
1699             # detect if situation has changed
1700             if interface.is_up_to_date() and self.queue.empty():
1701                 if not self.wallet.is_up_to_date():
1702                     self.wallet.set_up_to_date(True)
1703                     self.was_updated = True
1704             else:
1705                 if self.wallet.is_up_to_date():
1706                     self.wallet.set_up_to_date(False)
1707                     self.was_updated = True
1708
1709             if self.was_updated:
1710                 self.wallet.network.trigger_callback('updated')
1711                 self.was_updated = False
1712
1713             # 2. get a response
1714             try:
1715                 r = self.queue.get(block=True, timeout=1)
1716             except Queue.Empty:
1717                 continue
1718
1719             if interface != self.network.interface:
1720                 break
1721
1722             if not r:
1723                 continue
1724
1725             # 3. handle response
1726             method = r['method']
1727             params = r['params']
1728             result = r.get('result')
1729             error = r.get('error')
1730             if error:
1731                 print "error", r
1732                 continue
1733
1734             if method == 'blockchain.address.subscribe':
1735                 addr = params[0]
1736                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1737                     if requested_histories.get(addr) is None:
1738                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1739                         requested_histories[addr] = result
1740
1741             elif method == 'blockchain.address.get_history':
1742                 addr = params[0]
1743                 print_error("receiving history", addr, result)
1744                 if result == ['*']:
1745                     assert requested_histories.pop(addr) == '*'
1746                     self.wallet.receive_history_callback(addr, result)
1747                 else:
1748                     hist = []
1749                     # check that txids are unique
1750                     txids = []
1751                     for item in result:
1752                         tx_hash = item['tx_hash']
1753                         if tx_hash not in txids:
1754                             txids.append(tx_hash)
1755                             hist.append( (tx_hash, item['height']) )
1756
1757                     if len(hist) != len(result):
1758                         raise Exception("error: server sent history with non-unique txid", result)
1759
1760                     # check that the status corresponds to what was announced
1761                     rs = requested_histories.pop(addr)
1762                     if self.wallet.get_status(hist) != rs:
1763                         raise Exception("error: status mismatch: %s"%addr)
1764
1765                     # store received history
1766                     self.wallet.receive_history_callback(addr, hist)
1767
1768                     # request transactions that we don't have
1769                     for tx_hash, tx_height in hist:
1770                         if self.wallet.transactions.get(tx_hash) is None:
1771                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1772                                 missing_tx.append( (tx_hash, tx_height) )
1773
1774             elif method == 'blockchain.transaction.get':
1775                 tx_hash = params[0]
1776                 tx_height = params[1]
1777                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1778                 tx = Transaction(result)
1779                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1780                 self.was_updated = True
1781                 requested_tx.remove( (tx_hash, tx_height) )
1782                 print_error("received tx:", tx_hash, len(tx.raw))
1783
1784             else:
1785                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1786
1787             if self.was_updated and not requested_tx:
1788                 self.wallet.network.trigger_callback('updated')
1789                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1790
1791                 self.was_updated = False