lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except:
  59             raise BaseException('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         return self.data.get(key, default)
 129
 130     def put(self, key, value, save = True):
 131
 132         with self.lock:
 133             if value is not None:
 134                 self.data[key] = value
 135             else:
 136                 self.data.pop(key)
 137             if save:
 138                 self.write()
 139
 140     def write(self):
 141         s = repr(self.data)
 142         f = open(self.path,"w")
 143         f.write( s )
 144         f.close()
 145         if 'ANDROID_DATA' not in os.environ:
 146             import stat
 147             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 148
 149
 150 class Wallet:
 151
 152     def __init__(self, storage):
 153
 154         self.storage = storage
 155         self.electrum_version = ELECTRUM_VERSION
 156         self.gap_limit_for_change = 3 # constant
 157
 158         # saved fields
 159         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 160
 161         self.gap_limit             = storage.get('gap_limit', 5)
 162         self.use_change            = storage.get('use_change',True)
 163         self.use_encryption        = storage.get('use_encryption', False)
 164         self.seed                  = storage.get('seed', '')               # encrypted
 165         self.labels                = storage.get('labels', {})
 166         self.frozen_addresses      = storage.get('frozen_addresses',[])
 167         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 168         self.addressbook           = storage.get('contacts', [])
 169
 170         self.imported_keys         = storage.get('imported_keys',{})
 171         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 172
 173         self.fee                   = int(storage.get('fee_per_kb',20000))
 174
 175         self.master_public_keys = storage.get('master_public_keys',{})
 176         self.master_private_keys = storage.get('master_private_keys', {})
 177
 178         self.next_addresses = storage.get('next_addresses',{})
 179
 180         if self.seed_version not in [4, 6]:
 181             msg = "This wallet seed is not supported."
 182             if self.seed_version in [5]:
 183                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 184                 print msg
 185                 sys.exit(1)
 186
 187
 188         self.load_accounts()
 189
 190         self.transactions = {}
 191         tx_list = self.storage.get('transactions',{})
 192         for k,v in tx_list.items():
 193             try:
 194                 tx = Transaction(v)
 195             except:
 196                 print_msg("Warning: Cannot deserialize transactions. skipping")
 197                 continue
 198
 199             self.add_extra_addresses(tx)
 200             self.transactions[k] = tx
 201
 202         for h,tx in self.transactions.items():
 203             if not self.check_new_tx(h, tx):
 204                 print_error("removing unreferenced tx", h)
 205                 self.transactions.pop(h)
 206
 207
 208         # not saved
 209         self.prevout_values = {}     # my own transaction outputs
 210         self.spent_outputs = []
 211
 212         # spv
 213         self.verifier = None
 214
 215         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 216         # interface.is_up_to_date() returns true when all requests have been answered and processed
 217         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 218
 219         self.up_to_date = False
 220         self.lock = threading.Lock()
 221         self.transaction_lock = threading.Lock()
 222         self.tx_event = threading.Event()
 223
 224         for tx_hash, tx in self.transactions.items():
 225             self.update_tx_outputs(tx_hash)
 226
 227
 228     def add_extra_addresses(self, tx):
 229         h = tx.hash()
 230         # find the address corresponding to pay-to-pubkey inputs
 231         tx.add_extra_addresses(self.transactions)
 232         for o in tx.d.get('outputs'):
 233             if o.get('is_pubkey'):
 234                 for tx2 in self.transactions.values():
 235                     tx2.add_extra_addresses({h:tx})
 236
 237
 238
 239
 240     def set_up_to_date(self,b):
 241         with self.lock: self.up_to_date = b
 242
 243
 244     def is_up_to_date(self):
 245         with self.lock: return self.up_to_date
 246
 247
 248     def update(self):
 249         self.up_to_date = False
 250         while not self.is_up_to_date():
 251             time.sleep(0.1)
 252
 253
 254     def import_key(self, sec, password):
 255         # check password
 256         seed = self.get_seed(password)
 257         try:
 258             address = address_from_private_key(sec)
 259         except:
 260             raise BaseException('Invalid private key')
 261
 262         if self.is_mine(address):
 263             raise BaseException('Address already in wallet')
 264
 265         # store the originally requested keypair into the imported keys table
 266         self.imported_keys[address] = pw_encode(sec, password )
 267         self.storage.put('imported_keys', self.imported_keys, True)
 268         if self.synchronizer:
 269             self.synchronizer.subscribe_to_addresses([address])
 270         return address
 271
 272     def delete_imported_key(self, addr):
 273         if addr in self.imported_keys:
 274             self.imported_keys.pop(addr)
 275             self.storage.put('imported_keys', self.imported_keys, True)
 276
 277
 278     def make_seed(self):
 279         import mnemonic, ecdsa
 280         entropy = ecdsa.util.randrange( pow(2,160) )
 281         nonce = 0
 282         while True:
 283             ss = "%040x"%(entropy+nonce)
 284             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 285             # we keep only 13 words, that's approximately 139 bits of entropy
 286             words = mnemonic.mn_encode(s)[0:13]
 287             seed = ' '.join(words)
 288             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 289                 break  # this removes 12 bits of entropy
 290             nonce += 1
 291
 292         return seed
 293
 294
 295     def init_seed(self, seed):
 296         import mnemonic
 297
 298         if self.seed:
 299             raise BaseException("a seed exists")
 300
 301         if not seed:
 302             self.seed = random_seed(128)
 303             self.seed_version = 4
 304             return
 305
 306         #if not seed:
 307         #    self.seed = self.make_seed()
 308         #    self.seed_version = SEED_VERSION
 309         #    return
 310
 311         # find out what kind of wallet we are
 312         try:
 313             seed.decode('hex')
 314             self.seed_version = 4
 315             self.seed = str(seed)
 316             return
 317         except:
 318             pass
 319
 320         words = seed.split()
 321         self.seed_version = 4
 322         self.seed = mnemonic.mn_decode(words)
 323
 324         #try:
 325         #    mnemonic.mn_decode(words)
 326         #    uses_electrum_words = True
 327         #except:
 328         #    uses_electrum_words = False
 329         #
 330         #if uses_electrum_words and len(words) != 13:
 331         #    self.seed_version = 4
 332         #    self.seed = mnemonic.mn_decode(words)
 333         #else:
 334         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 335         #    self.seed_version = SEED_VERSION
 336         #    self.seed = seed
 337
 338
 339     def save_seed(self):
 340         self.storage.put('seed', self.seed, True)
 341         self.storage.put('seed_version', self.seed_version, True)
 342
 343     def create_watching_only_wallet(self, params):
 344         K0, c0 = params
 345         if not K0:
 346             return
 347
 348         if not c0:
 349             self.seed_version = 4
 350             self.create_old_account(K0)
 351             return
 352
 353         cK0 = ""
 354         self.master_public_keys = {
 355             "m/0'/": (c0, K0, cK0),
 356             }
 357         self.storage.put('master_public_keys', self.master_public_keys, True)
 358         self.create_account('1','Main account')
 359
 360
 361     def create_accounts(self):
 362         if self.seed_version == 4:
 363             mpk = OldAccount.mpk_from_seed(self.seed)
 364             self.create_old_account(mpk)
 365         else:
 366             # create default account
 367             self.create_master_keys('1')
 368             self.create_account('1','Main account')
 369
 370
 371     def create_master_keys(self, account_type):
 372         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 373         if account_type == '1':
 374             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 375             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 376             self.master_private_keys["m/0'/"] = k0
 377         elif account_type == '2of2':
 378             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 379             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 380             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 381             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 382             self.master_private_keys["m/1'/"] = k1
 383             self.master_private_keys["m/2'/"] = k2
 384         elif account_type == '2of3':
 385             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 386             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 387             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 388             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 389             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 390             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 391             self.master_private_keys["m/3'/"] = k3
 392             self.master_private_keys["m/4'/"] = k4
 393             self.master_private_keys["m/5'/"] = k5
 394
 395         self.storage.put('master_public_keys', self.master_public_keys, True)
 396         self.storage.put('master_private_keys', self.master_private_keys, True)
 397
 398     def has_master_public_keys(self, account_type):
 399         if account_type == '1':
 400             return "m/0'/" in self.master_public_keys
 401         elif account_type == '2of2':
 402             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 403         elif account_type == '2of3':
 404             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 405
 406     def find_root_by_master_key(self, c, K):
 407         for key, v in self.master_public_keys.items():
 408             if key == "m/":continue
 409             cc, KK, _ = v
 410             if (c == cc) and (K == KK):
 411                 return key
 412
 413     def deseed_root(self, seed, password):
 414         # for safety, we ask the user to enter their seed
 415         assert seed == self.get_seed(password)
 416         self.seed = ''
 417         self.storage.put('seed', '', True)
 418
 419
 420     def deseed_branch(self, k):
 421         # check that parent has no seed
 422         assert self.seed == ''
 423         self.master_private_keys.pop(k)
 424         self.storage.put('master_private_keys', self.master_private_keys, True)
 425
 426     def is_watching_only(self):
 427         return (self.seed == '') and (self.master_private_keys == {})
 428
 429
 430
 431     def account_id(self, account_type, i):
 432         if account_type == '1':
 433             return "m/0'/%d"%i
 434         elif account_type == '2of2':
 435             return "m/1'/%d & m/2'/%d"%(i,i)
 436         elif account_type == '2of3':
 437             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 438         else:
 439             raise BaseException('unknown account type')
 440
 441
 442     def num_accounts(self, account_type):
 443         keys = self.accounts.keys()
 444         i = 0
 445         while True:
 446             account_id = self.account_id(account_type, i)
 447             if account_id not in keys: break
 448             i += 1
 449         return i
 450
 451
 452     def new_account_address(self, account_type = '1'):
 453         i = self.num_accounts(account_type)
 454         k = self.account_id(account_type,i)
 455
 456         addr = self.next_addresses.get(k)
 457         if not addr:
 458             account_id, account = self.next_account(account_type)
 459             addr = account.first_address()
 460             self.next_addresses[k] = addr
 461             self.storage.put('next_addresses',self.next_addresses)
 462
 463         return k, addr
 464
 465
 466     def next_account(self, account_type = '1'):
 467
 468         i = self.num_accounts(account_type)
 469         account_id = self.account_id(account_type,i)
 470
 471         if account_type is '1':
 472             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 473             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 474             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 475
 476         elif account_type == '2of2':
 477             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 478             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 479             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 480             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 481             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 482
 483         elif account_type == '2of3':
 484             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 485             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 486             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 487             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 488             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 489             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 490             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 491
 492         return account_id, account
 493
 494
 495     def set_label(self, name, text = None):
 496         changed = False
 497         old_text = self.labels.get(name)
 498         if text:
 499             if old_text != text:
 500                 self.labels[name] = text
 501                 changed = True
 502         else:
 503             if old_text:
 504                 self.labels.pop(name)
 505                 changed = True
 506
 507         if changed:
 508             self.storage.put('labels', self.labels, True)
 509
 510         run_hook('set_label', name, text, changed)
 511         return changed
 512
 513
 514
 515     def create_account(self, account_type = '1', name = None):
 516         k, account = self.next_account(account_type)
 517         if k in self.pending_accounts:
 518             self.pending_accounts.pop(k)
 519             self.storage.put('pending_accounts', self.pending_accounts)
 520
 521         self.accounts[k] = account
 522         self.save_accounts()
 523         if name:
 524             self.set_label(k, name)
 525
 526
 527     def create_old_account(self, mpk):
 528         self.storage.put('master_public_key', mpk, True)
 529         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 530         self.save_accounts()
 531
 532
 533     def save_accounts(self):
 534         d = {}
 535         for k, v in self.accounts.items():
 536             d[k] = v.dump()
 537         self.storage.put('accounts', d, True)
 538
 539
 540
 541     def load_accounts(self):
 542         d = self.storage.get('accounts', {})
 543         self.accounts = {}
 544         for k, v in d.items():
 545             if k == 0:
 546                 v['mpk'] = self.storage.get('master_public_key')
 547                 self.accounts[k] = OldAccount(v)
 548             elif '&' in k:
 549                 self.accounts[k] = BIP32_Account_2of2(v)
 550             else:
 551                 self.accounts[k] = BIP32_Account(v)
 552
 553         self.pending_accounts = self.storage.get('pending_accounts',{})
 554
 555
 556     def delete_pending_account(self, k):
 557         self.pending_accounts.pop(k)
 558         self.storage.put('pending_accounts', self.pending_accounts)
 559
 560     def account_is_pending(self, k):
 561         return k in self.pending_accounts
 562
 563     def create_pending_account(self, acct_type, name):
 564         k, addr = self.new_account_address(acct_type)
 565         self.set_label(k, name)
 566         self.pending_accounts[k] = addr
 567         self.storage.put('pending_accounts', self.pending_accounts)
 568
 569     def get_pending_accounts(self):
 570         return self.pending_accounts.items()
 571
 572
 573     def addresses(self, include_change = True, _next=True):
 574         o = self.get_account_addresses(-1, include_change)
 575         for a in self.accounts.keys():
 576             o += self.get_account_addresses(a, include_change)
 577
 578         if _next:
 579             for addr in self.next_addresses.values():
 580                 if addr not in o:
 581                     o += [addr]
 582         return o
 583
 584
 585     def is_mine(self, address):
 586         return address in self.addresses(True)
 587
 588
 589     def is_change(self, address):
 590         if not self.is_mine(address): return False
 591         if address in self.imported_keys.keys(): return False
 592         acct, s = self.get_address_index(address)
 593         if s is None: return False
 594         return s[0] == 1
 595
 596     def get_master_public_key(self):
 597         if self.seed_version == 4:
 598             return self.storage.get("master_public_key")
 599         else:
 600             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 601             return repr((c, K))
 602
 603     def get_master_private_key(self, account, password):
 604         master_k = pw_decode( self.master_private_keys[account], password)
 605         master_c, master_K, master_Kc = self.master_public_keys[account]
 606         try:
 607             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 608             assert K.encode('hex') == master_K
 609         except:
 610             raise BaseException("Invalid password")
 611         return master_k
 612
 613
 614     def get_address_index(self, address):
 615         if address in self.imported_keys.keys():
 616             return -1, None
 617
 618         for account in self.accounts.keys():
 619             for for_change in [0,1]:
 620                 addresses = self.accounts[account].get_addresses(for_change)
 621                 for addr in addresses:
 622                     if address == addr:
 623                         return account, (for_change, addresses.index(addr))
 624
 625         for k,v in self.next_addresses.items():
 626             if v == address:
 627                 return k, (0,0)
 628
 629         raise BaseException("Address not found", address)
 630
 631
 632     def get_roots(self, account):
 633         roots = []
 634         for a in account.split('&'):
 635             s = a.strip()
 636             m = re.match("(m/\d+'/)(\d+)", s)
 637             roots.append( m.group(1) )
 638         return roots
 639
 640     def is_seeded(self, account):
 641         if type(account) is int:
 642             return self.seed is not None
 643
 644         for root in self.get_roots(account):
 645             if root not in self.master_private_keys.keys():
 646                 return False
 647         return True
 648
 649     def rebase_sequence(self, account, sequence):
 650         c, i = sequence
 651         dd = []
 652         for a in account.split('&'):
 653             s = a.strip()
 654             m = re.match("(m/\d+'/)(\d+)", s)
 655             root = m.group(1)
 656             num = int(m.group(2))
 657             dd.append( (root, [num,c,i] ) )
 658         return dd
 659
 660
 661     def get_keyID(self, account, sequence):
 662         if account == 0:
 663             a, b = sequence
 664             mpk = self.storage.get('master_public_key')
 665             return 'old(%s,%d,%d)'%(mpk,a,b)
 666
 667         rs = self.rebase_sequence(account, sequence)
 668         dd = []
 669         for root, public_sequence in rs:
 670             c, K, _ = self.master_public_keys[root]
 671             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 672             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 673         return '&'.join(dd)
 674
 675
 676
 677     def get_seed(self, password):
 678         s = pw_decode(self.seed, password)
 679         if self.seed_version == 4:
 680             seed = s
 681             self.accounts[0].check_seed(seed)
 682         else:
 683             seed = mnemonic_hash(s)
 684         return seed
 685
 686
 687     def get_mnemonic(self, password):
 688         import mnemonic
 689         s = pw_decode(self.seed, password)
 690         if self.seed_version == 4:
 691             return ' '.join(mnemonic.mn_encode(s))
 692         else:
 693             return s
 694
 695
 696
 697     def get_private_key(self, address, password):
 698         out = []
 699         if address in self.imported_keys.keys():
 700             out.append( pw_decode( self.imported_keys[address], password ) )
 701         else:
 702             account, sequence = self.get_address_index(address)
 703             if account == 0:
 704                 seed = self.get_seed(password)
 705                 pk = self.accounts[account].get_private_key(seed, sequence)
 706                 out.append(pk)
 707                 return out
 708
 709             # assert address == self.accounts[account].get_address(*sequence)
 710             rs = self.rebase_sequence( account, sequence)
 711             for root, public_sequence in rs:
 712
 713                 if root not in self.master_private_keys.keys(): continue
 714                 master_k = self.get_master_private_key(root, password)
 715                 master_c, _, _ = self.master_public_keys[root]
 716                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 717                 out.append(pk)
 718
 719         return out
 720
 721
 722     def add_keypairs_from_wallet(self, tx, keypairs, password):
 723         for txin in tx.inputs:
 724             address = txin['address']
 725             private_keys = self.get_private_key(address, password)
 726             for sec in private_keys:
 727                 pubkey = public_key_from_private_key(sec)
 728                 keypairs[ pubkey ] = sec
 729                 if address in self.imported_keys.keys():
 730                     txin['redeemPubkey'] = pubkey
 731
 732
 733     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 734         for txin in tx.inputs:
 735             keyid = txin.get('KeyID')
 736             if keyid:
 737
 738                 if self.seed_version==4:
 739                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 740                     if not m: continue
 741                     mpk = m.group(1)
 742                     if mpk != self.storage.get('master_public_key'): continue
 743                     index = int(m.group(2))
 744                     num = int(m.group(3))
 745                     account = self.accounts[0]
 746                     addr = account.get_address(index, num)
 747                     txin['address'] = addr # fixme: side effect
 748                     pk = self.get_private_key(addr, password)
 749                     for sec in pk:
 750                         pubkey = public_key_from_private_key(sec)
 751                         keypairs[pubkey] = sec
 752                     continue
 753
 754
 755                 roots = []
 756                 for s in keyid.split('&'):
 757                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 758                     if not m: continue
 759                     c = m.group(1)
 760                     K = m.group(2)
 761                     sequence = m.group(3)
 762                     root = self.find_root_by_master_key(c,K)
 763                     if not root: continue
 764                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 765                     root = root + '%d'%sequence[0]
 766                     sequence = sequence[1:]
 767                     roots.append((root,sequence))
 768
 769                 account_id = " & ".join( map(lambda x:x[0], roots) )
 770                 account = self.accounts.get(account_id)
 771                 if not account: continue
 772                 addr = account.get_address(*sequence)
 773                 txin['address'] = addr # fixme: side effect
 774                 pk = self.get_private_key(addr, password)
 775                 for sec in pk:
 776                     pubkey = public_key_from_private_key(sec)
 777                     keypairs[pubkey] = sec
 778
 779
 780
 781     def signrawtransaction(self, tx, input_info, private_keys, password):
 782
 783         # check that the password is correct
 784         seed = self.get_seed(password)
 785
 786         # add input info
 787         tx.add_input_info(input_info)
 788
 789         # add redeem script for coins that are in the wallet
 790         # FIXME: add redeemPubkey too!
 791         unspent_coins = self.get_unspent_coins()
 792         for txin in tx.inputs:
 793             for item in unspent_coins:
 794                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 795                     print_error( "tx input is in unspent coins" )
 796                     txin['scriptPubKey'] = item['scriptPubKey']
 797                     account, sequence = self.get_address_index(item['address'])
 798                     if account != -1:
 799                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 800                         print_error("added redeemScript", txin['redeemScript'])
 801                     break
 802
 803
 804         # build a list of public/private keys
 805         keypairs = {}
 806
 807         # add private keys from parameter
 808         for sec in private_keys:
 809             pubkey = public_key_from_private_key(sec)
 810             keypairs[ pubkey ] = sec
 811
 812         # add private_keys from KeyID
 813         self.add_keypairs_from_KeyID(tx, keypairs, password)
 814
 815         # add private keys from wallet
 816         self.add_keypairs_from_wallet(tx, keypairs, password)
 817         self.sign_transaction(tx, keypairs)
 818
 819
 820     def sign_message(self, address, message, password):
 821         keys = self.get_private_key(address, password)
 822         assert len(keys) == 1
 823         sec = keys[0]
 824         key = regenerate_key(sec)
 825         compressed = is_compressed(sec)
 826         return key.sign_message(message, compressed, address)
 827
 828
 829     def change_gap_limit(self, value):
 830         if value >= self.gap_limit:
 831             self.gap_limit = value
 832             self.storage.put('gap_limit', self.gap_limit, True)
 833             #self.interface.poke('synchronizer')
 834             return True
 835
 836         elif value >= self.min_acceptable_gap():
 837             for key, account in self.accounts.items():
 838                 addresses = account[0]
 839                 k = self.num_unused_trailing_addresses(addresses)
 840                 n = len(addresses) - k + value
 841                 addresses = addresses[0:n]
 842                 self.accounts[key][0] = addresses
 843
 844             self.gap_limit = value
 845             self.storage.put('gap_limit', self.gap_limit, True)
 846             self.save_accounts()
 847             return True
 848         else:
 849             return False
 850
 851     def num_unused_trailing_addresses(self, addresses):
 852         k = 0
 853         for a in addresses[::-1]:
 854             if self.history.get(a):break
 855             k = k + 1
 856         return k
 857
 858     def min_acceptable_gap(self):
 859         # fixme: this assumes wallet is synchronized
 860         n = 0
 861         nmax = 0
 862
 863         for account in self.accounts.values():
 864             addresses = account.get_addresses(0)
 865             k = self.num_unused_trailing_addresses(addresses)
 866             for a in addresses[0:-k]:
 867                 if self.history.get(a):
 868                     n = 0
 869                 else:
 870                     n += 1
 871                     if n > nmax: nmax = n
 872         return nmax + 1
 873
 874
 875     def address_is_old(self, address):
 876         age = -1
 877         h = self.history.get(address, [])
 878         if h == ['*']:
 879             return True
 880         for tx_hash, tx_height in h:
 881             if tx_height == 0:
 882                 tx_age = 0
 883             else:
 884                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 885             if tx_age > age:
 886                 age = tx_age
 887         return age > 2
 888
 889
 890     def synchronize_sequence(self, account, for_change):
 891         limit = self.gap_limit_for_change if for_change else self.gap_limit
 892         new_addresses = []
 893         while True:
 894             addresses = account.get_addresses(for_change)
 895             if len(addresses) < limit:
 896                 address = account.create_new_address(for_change)
 897                 self.history[address] = []
 898                 new_addresses.append( address )
 899                 continue
 900
 901             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 902                 break
 903             else:
 904                 address = account.create_new_address(for_change)
 905                 self.history[address] = []
 906                 new_addresses.append( address )
 907
 908         return new_addresses
 909
 910
 911
 912     def create_pending_accounts(self):
 913         for account_type in ['1','2of2','2of3']:
 914             if not self.has_master_public_keys(account_type):
 915                 continue
 916             k, a = self.new_account_address(account_type)
 917             if self.address_is_old(a):
 918                 print_error( "creating account", a )
 919                 self.create_account(account_type)
 920                 self.next_addresses.pop(k)
 921
 922
 923     def synchronize_account(self, account):
 924         new = []
 925         new += self.synchronize_sequence(account, 0)
 926         new += self.synchronize_sequence(account, 1)
 927         return new
 928
 929
 930     def synchronize(self):
 931         if self.master_public_keys:
 932             self.create_pending_accounts()
 933         new = []
 934         for account in self.accounts.values():
 935             new += self.synchronize_account(account)
 936         if new:
 937             self.save_accounts()
 938             self.storage.put('addr_history', self.history, True)
 939         return new
 940
 941
 942     def is_found(self):
 943         return self.history.values() != [[]] * len(self.history)
 944
 945
 946     def add_contact(self, address, label=None):
 947         self.addressbook.append(address)
 948         self.storage.put('contacts', self.addressbook, True)
 949         if label:
 950             self.set_label(address, label)
 951
 952
 953     def delete_contact(self, addr):
 954         if addr in self.addressbook:
 955             self.addressbook.remove(addr)
 956             self.storage.put('addressbook', self.addressbook, True)
 957
 958
 959     def fill_addressbook(self):
 960         for tx_hash, tx in self.transactions.items():
 961             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 962             if is_send:
 963                 for addr, v in tx.outputs:
 964                     if not self.is_mine(addr) and addr not in self.addressbook:
 965                         self.addressbook.append(addr)
 966         # redo labels
 967         # self.update_tx_labels()
 968
 969     def get_num_tx(self, address):
 970         n = 0
 971         for tx in self.transactions.values():
 972             if address in map(lambda x:x[0], tx.outputs): n += 1
 973         return n
 974
 975
 976     def get_address_flags(self, addr):
 977         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 978         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 979         return flags
 980
 981
 982     def get_tx_value(self, tx, account=None):
 983         domain = self.get_account_addresses(account)
 984         return tx.get_value(domain, self.prevout_values)
 985
 986
 987     def update_tx_outputs(self, tx_hash):
 988         tx = self.transactions.get(tx_hash)
 989
 990         for i, (addr, value) in enumerate(tx.outputs):
 991             key = tx_hash+ ':%d'%i
 992             self.prevout_values[key] = value
 993
 994         for item in tx.inputs:
 995             if self.is_mine(item.get('address')):
 996                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 997                 self.spent_outputs.append(key)
 998
 999
1000     def get_addr_balance(self, address):
1001         assert self.is_mine(address)
1002         h = self.history.get(address,[])
1003         if h == ['*']: return 0,0
1004         c = u = 0
1005         received_coins = []   # list of coins received at address
1006
1007         for tx_hash, tx_height in h:
1008             tx = self.transactions.get(tx_hash)
1009             if not tx: continue
1010
1011             for i, (addr, value) in enumerate(tx.outputs):
1012                 if addr == address:
1013                     key = tx_hash + ':%d'%i
1014                     received_coins.append(key)
1015
1016         for tx_hash, tx_height in h:
1017             tx = self.transactions.get(tx_hash)
1018             if not tx: continue
1019             v = 0
1020
1021             for item in tx.inputs:
1022                 addr = item.get('address')
1023                 if addr == address:
1024                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1025                     value = self.prevout_values.get( key )
1026                     if key in received_coins:
1027                         v -= value
1028
1029             for i, (addr, value) in enumerate(tx.outputs):
1030                 key = tx_hash + ':%d'%i
1031                 if addr == address:
1032                     v += value
1033
1034             if tx_height:
1035                 c += v
1036             else:
1037                 u += v
1038         return c, u
1039
1040
1041     def get_account_name(self, k):
1042         if k == 0:
1043             if self.seed_version == 4:
1044                 name = 'Main account'
1045             else:
1046                 name = 'Old account'
1047         else:
1048             default = "Unnamed account"
1049             m = re.match("m/0'/(\d+)", k)
1050             if m:
1051                 num = m.group(1)
1052                 if num == '0':
1053                     default = "Main account"
1054                 else:
1055                     default = "Account %s"%num
1056
1057             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1058             if m:
1059                 num = m.group(1)
1060                 default = "2of2 account %s"%num
1061             name = self.labels.get(k, default)
1062
1063         return name
1064
1065     def get_account_names(self):
1066         accounts = {}
1067         for k, account in self.accounts.items():
1068             accounts[k] = self.get_account_name(k)
1069         if self.imported_keys:
1070             accounts[-1] = 'Imported keys'
1071         return accounts
1072
1073     def get_account_addresses(self, a, include_change=True):
1074         if a is None:
1075             o = self.addresses(True)
1076         elif a == -1:
1077             o = self.imported_keys.keys()
1078         else:
1079             ac = self.accounts[a]
1080             o = ac.get_addresses(0)
1081             if include_change: o += ac.get_addresses(1)
1082         return o
1083
1084     def get_imported_balance(self):
1085         return self.get_balance(self.imported_keys.keys())
1086
1087     def get_account_balance(self, account):
1088         return self.get_balance(self.get_account_addresses(account))
1089
1090     def get_frozen_balance(self):
1091         return self.get_balance(self.frozen_addresses)
1092
1093     def get_balance(self, domain=None):
1094         if domain is None: domain = self.addresses(True)
1095         cc = uu = 0
1096         for addr in domain:
1097             c, u = self.get_addr_balance(addr)
1098             cc += c
1099             uu += u
1100         return cc, uu
1101
1102
1103     def get_unspent_coins(self, domain=None):
1104         coins = []
1105         if domain is None: domain = self.addresses(True)
1106         for addr in domain:
1107             h = self.history.get(addr, [])
1108             if h == ['*']: continue
1109             for tx_hash, tx_height in h:
1110                 tx = self.transactions.get(tx_hash)
1111                 if tx is None: raise BaseException("Wallet not synchronized")
1112                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1113                 for output in tx.d.get('outputs'):
1114                     if output.get('address') != addr: continue
1115                     key = tx_hash + ":%d" % output.get('prevout_n')
1116                     if key in self.spent_outputs: continue
1117                     output['prevout_hash'] = tx_hash
1118                     output['height'] = tx_height
1119                     output['coinbase'] = is_coinbase
1120                     coins.append((tx_height, output))
1121
1122         # sort by age
1123         if coins:
1124             coins = sorted(coins)
1125             if coins[-1][0] != 0:
1126                 while coins[0][0] == 0:
1127                     coins = coins[1:] + [ coins[0] ]
1128         return [x[1] for x in coins]
1129
1130
1131
1132     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1133         domain = self.get_account_addresses(account) if account else None
1134         return self.choose_tx_inputs( amount, fixed_fee, domain )
1135
1136
1137     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1138         """ todo: minimize tx size """
1139         total = 0
1140         fee = self.fee if fixed_fee is None else fixed_fee
1141         if domain is None:
1142             domain = self.addresses(True)
1143
1144         for i in self.frozen_addresses:
1145             if i in domain: domain.remove(i)
1146
1147         prioritized = []
1148         for i in self.prioritized_addresses:
1149             if i in domain:
1150                 domain.remove(i)
1151                 prioritized.append(i)
1152
1153         coins = self.get_unspent_coins(domain)
1154         prioritized_coins = self.get_unspent_coins(prioritized)
1155
1156         inputs = []
1157         coins = prioritized_coins + coins
1158
1159         for item in coins:
1160             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1161                 continue
1162             addr = item.get('address')
1163             v = item.get('value')
1164             total += v
1165             inputs.append(item)
1166             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1167             if total >= amount + fee: break
1168         else:
1169             inputs = []
1170
1171         return inputs, total, fee
1172
1173
1174     def set_fee(self, fee):
1175         if self.fee != fee:
1176             self.fee = fee
1177             self.storage.put('fee_per_kb', self.fee, True)
1178
1179     def estimated_fee(self, inputs):
1180         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1181         fee = self.fee * int(round(estimated_size/1024.))
1182         if fee == 0: fee = self.fee
1183         return fee
1184
1185
1186     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1187         "add change to a transaction"
1188         change_amount = total - ( amount + fee )
1189         if change_amount > DUST_THRESHOLD:
1190             if not change_addr:
1191
1192                 # send change to one of the accounts involved in the tx
1193                 address = inputs[0].get('address')
1194                 account, _ = self.get_address_index(address)
1195
1196                 if not self.use_change or account == -1:
1197                     change_addr = inputs[-1]['address']
1198                 else:
1199                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1200
1201             # Insert the change output at a random position in the outputs
1202             posn = random.randint(0, len(outputs))
1203             outputs[posn:posn] = [( change_addr,  change_amount)]
1204         return outputs
1205
1206
1207     def get_history(self, address):
1208         with self.lock:
1209             return self.history.get(address)
1210
1211
1212     def get_status(self, h):
1213         if not h: return None
1214         if h == ['*']: return '*'
1215         status = ''
1216         for tx_hash, height in h:
1217             status += tx_hash + ':%d:' % height
1218         return hashlib.sha256( status ).digest().encode('hex')
1219
1220
1221     def receive_tx_callback(self, tx_hash, tx, tx_height):
1222
1223         with self.transaction_lock:
1224             self.add_extra_addresses(tx)
1225             if not self.check_new_tx(tx_hash, tx):
1226                 # may happen due to pruning
1227                 print_error("received transaction that is no longer referenced in history", tx_hash)
1228                 return
1229             self.transactions[tx_hash] = tx
1230             self.network.interface.pending_transactions_for_notifications.append(tx)
1231             self.save_transactions()
1232             if self.verifier and tx_height>0:
1233                 self.verifier.add(tx_hash, tx_height)
1234             self.update_tx_outputs(tx_hash)
1235
1236
1237     def save_transactions(self):
1238         tx = {}
1239         for k,v in self.transactions.items():
1240             tx[k] = str(v)
1241         self.storage.put('transactions', tx, True)
1242
1243     def receive_history_callback(self, addr, hist):
1244
1245         if not self.check_new_history(addr, hist):
1246             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1247
1248         with self.lock:
1249             self.history[addr] = hist
1250             self.storage.put('addr_history', self.history, True)
1251
1252         if hist != ['*']:
1253             for tx_hash, tx_height in hist:
1254                 if tx_height>0:
1255                     # add it in case it was previously unconfirmed
1256                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1257
1258
1259     def get_tx_history(self, account=None):
1260         if not self.verifier:
1261             return []
1262
1263         with self.transaction_lock:
1264             history = self.transactions.items()
1265             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1266             result = []
1267
1268             balance = 0
1269             for tx_hash, tx in history:
1270                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1271                 if v is not None: balance += v
1272
1273             c, u = self.get_account_balance(account)
1274
1275             if balance != c+u:
1276                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1277
1278             balance = c + u - balance
1279             for tx_hash, tx in history:
1280                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1281                 if not is_relevant:
1282                     continue
1283                 if value is not None:
1284                     balance += value
1285
1286                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1287                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1288
1289         return result
1290
1291
1292     def get_label(self, tx_hash):
1293         label = self.labels.get(tx_hash)
1294         is_default = (label == '') or (label is None)
1295         if is_default: label = self.get_default_label(tx_hash)
1296         return label, is_default
1297
1298
1299     def get_default_label(self, tx_hash):
1300         tx = self.transactions.get(tx_hash)
1301         default_label = ''
1302         if tx:
1303             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1304             if is_mine:
1305                 for o in tx.outputs:
1306                     o_addr, _ = o
1307                     if not self.is_mine(o_addr):
1308                         try:
1309                             default_label = self.labels[o_addr]
1310                         except KeyError:
1311                             default_label = o_addr
1312                         break
1313                 else:
1314                     default_label = '(internal)'
1315             else:
1316                 for o in tx.outputs:
1317                     o_addr, _ = o
1318                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1319                         break
1320                 else:
1321                     for o in tx.outputs:
1322                         o_addr, _ = o
1323                         if self.is_mine(o_addr):
1324                             break
1325                     else:
1326                         o_addr = None
1327
1328                 if o_addr:
1329                     dest_label = self.labels.get(o_addr)
1330                     try:
1331                         default_label = self.labels[o_addr]
1332                     except KeyError:
1333                         default_label = o_addr
1334
1335         return default_label
1336
1337
1338     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1339         for address, x in outputs:
1340             assert is_valid(address)
1341         amount = sum( map(lambda x:x[1], outputs) )
1342         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1343         if not inputs:
1344             raise ValueError("Not enough funds")
1345         self.add_input_info(inputs)
1346         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1347         return Transaction.from_io(inputs, outputs)
1348
1349
1350     def mktx_from_account(self, outputs, password, fee=None, account=None):
1351         domain = self.get_account_addresses(account) if account else None
1352         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1353
1354
1355     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1356         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1357         keypairs = {}
1358         self.add_keypairs_from_wallet(tx, keypairs, password)
1359         if keypairs:
1360             self.sign_transaction(tx, keypairs)
1361         return tx
1362
1363
1364     def add_input_info(self, inputs):
1365         for txin in inputs:
1366             address = txin['address']
1367             if address in self.imported_keys.keys():
1368                 continue
1369             account, sequence = self.get_address_index(address)
1370             txin['KeyID'] = self.get_keyID(account, sequence)
1371             redeemScript = self.accounts[account].redeem_script(sequence)
1372             if redeemScript:
1373                 txin['redeemScript'] = redeemScript
1374             else:
1375                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1376
1377
1378     def sign_transaction(self, tx, keypairs):
1379         tx.sign(keypairs)
1380         run_hook('sign_transaction', tx)
1381
1382
1383     def sendtx(self, tx):
1384         # synchronous
1385         h = self.send_tx(tx)
1386         self.tx_event.wait()
1387         return self.receive_tx(h)
1388
1389     def send_tx(self, tx):
1390         # asynchronous
1391         self.tx_event.clear()
1392         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1393         return tx.hash()
1394
1395     def on_broadcast(self, i, r):
1396         self.tx_result = r.get('result')
1397         self.tx_event.set()
1398
1399     def receive_tx(self,tx_hash):
1400         out = self.tx_result
1401         if out != tx_hash:
1402             return False, "error: " + out
1403         return True, out
1404
1405
1406
1407     def update_password(self, old_password, new_password):
1408         if new_password == '': new_password = None
1409         # this will throw an exception if unicode cannot be converted
1410         decoded = pw_decode(self.seed, old_password)
1411         self.seed = pw_encode( decoded, new_password)
1412         self.storage.put('seed', self.seed, True)
1413         self.use_encryption = (new_password != None)
1414         self.storage.put('use_encryption', self.use_encryption,True)
1415         for k in self.imported_keys.keys():
1416             a = self.imported_keys[k]
1417             b = pw_decode(a, old_password)
1418             c = pw_encode(b, new_password)
1419             self.imported_keys[k] = c
1420         self.storage.put('imported_keys', self.imported_keys, True)
1421
1422         for k, v in self.master_private_keys.items():
1423             b = pw_decode(v, old_password)
1424             c = pw_encode(b, new_password)
1425             self.master_private_keys[k] = c
1426         self.storage.put('master_private_keys', self.master_private_keys, True)
1427
1428
1429     def freeze(self,addr):
1430         if self.is_mine(addr) and addr not in self.frozen_addresses:
1431             self.unprioritize(addr)
1432             self.frozen_addresses.append(addr)
1433             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1434             return True
1435         else:
1436             return False
1437
1438     def unfreeze(self,addr):
1439         if self.is_mine(addr) and addr in self.frozen_addresses:
1440             self.frozen_addresses.remove(addr)
1441             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1442             return True
1443         else:
1444             return False
1445
1446     def prioritize(self,addr):
1447         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1448             self.unfreeze(addr)
1449             self.prioritized_addresses.append(addr)
1450             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1451             return True
1452         else:
1453             return False
1454
1455     def unprioritize(self,addr):
1456         if self.is_mine(addr) and addr in self.prioritized_addresses:
1457             self.prioritized_addresses.remove(addr)
1458             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1459             return True
1460         else:
1461             return False
1462
1463
1464     def set_verifier(self, verifier):
1465         self.verifier = verifier
1466
1467         # review transactions that are in the history
1468         for addr, hist in self.history.items():
1469             if hist == ['*']: continue
1470             for tx_hash, tx_height in hist:
1471                 if tx_height>0:
1472                     # add it in case it was previously unconfirmed
1473                     self.verifier.add(tx_hash, tx_height)
1474
1475
1476         # if we are on a pruning server, remove unverified transactions
1477         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1478         for tx_hash in self.transactions.keys():
1479             if tx_hash not in vr:
1480                 self.transactions.pop(tx_hash)
1481
1482
1483
1484     def check_new_history(self, addr, hist):
1485
1486         # check that all tx in hist are relevant
1487         if hist != ['*']:
1488             for tx_hash, height in hist:
1489                 tx = self.transactions.get(tx_hash)
1490                 if not tx: continue
1491                 if not tx.has_address(addr):
1492                     return False
1493
1494         # check that we are not "orphaning" a transaction
1495         old_hist = self.history.get(addr,[])
1496         if old_hist == ['*']: return True
1497
1498         for tx_hash, height in old_hist:
1499             if tx_hash in map(lambda x:x[0], hist): continue
1500             found = False
1501             for _addr, _hist in self.history.items():
1502                 if _addr == addr: continue
1503                 if _hist == ['*']: continue
1504                 _tx_hist = map(lambda x:x[0], _hist)
1505                 if tx_hash in _tx_hist:
1506                     found = True
1507                     break
1508
1509             if not found:
1510                 tx = self.transactions.get(tx_hash)
1511                 # tx might not be there
1512                 if not tx: continue
1513
1514                 # already verified?
1515                 if self.verifier.get_height(tx_hash):
1516                     continue
1517                 # unconfirmed tx
1518                 print_error("new history is orphaning transaction:", tx_hash)
1519                 # check that all outputs are not mine, request histories
1520                 ext_requests = []
1521                 for _addr, _v in tx.outputs:
1522                     # assert not self.is_mine(_addr)
1523                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1524
1525                 ext_h = self.network.synchronous_get(ext_requests)
1526                 print_error("sync:", ext_requests, ext_h)
1527                 height = None
1528                 for h in ext_h:
1529                     if h == ['*']: continue
1530                     for item in h:
1531                         if item.get('tx_hash') == tx_hash:
1532                             height = item.get('height')
1533                 if height:
1534                     print_error("found height for", tx_hash, height)
1535                     self.verifier.add(tx_hash, height)
1536                 else:
1537                     print_error("removing orphaned tx from history", tx_hash)
1538                     self.transactions.pop(tx_hash)
1539
1540         return True
1541
1542
1543
1544     def check_new_tx(self, tx_hash, tx):
1545         # 1 check that tx is referenced in addr_history.
1546         addresses = []
1547         for addr, hist in self.history.items():
1548             if hist == ['*']:continue
1549             for txh, height in hist:
1550                 if txh == tx_hash:
1551                     addresses.append(addr)
1552
1553         if not addresses:
1554             return False
1555
1556         # 2 check that referencing addresses are in the tx
1557         for addr in addresses:
1558             if not tx.has_address(addr):
1559                 return False
1560
1561         return True
1562
1563
1564     def start_threads(self, network):
1565         from verifier import TxVerifier
1566         self.network = network
1567         if self.network:
1568             self.verifier = TxVerifier(self.network, self.storage)
1569             self.verifier.start()
1570             self.set_verifier(self.verifier)
1571             self.synchronizer = WalletSynchronizer(self, network)
1572             self.synchronizer.start()
1573         else:
1574             self.verifier = None
1575             self.synchronizer =None
1576
1577     def stop_threads(self):
1578         if self.network:
1579             self.verifier.stop()
1580             self.synchronizer.stop()
1581
1582
1583     def restore(self, callback):
1584         from i18n import _
1585         def wait_for_wallet():
1586             self.set_up_to_date(False)
1587             while not self.is_up_to_date():
1588                 msg = "%s\n%s %d\n%s %.1f"%(
1589                     _("Please wait..."),
1590                     _("Addresses generated:"),
1591                     len(self.addresses(True)),
1592                     _("Kilobytes received:"),
1593                     self.network.interface.bytes_received/1024.)
1594
1595                 apply(callback, (msg,))
1596                 time.sleep(0.1)
1597
1598         def wait_for_network():
1599             while not self.network.is_connected():
1600                 msg = "%s \n" % (_("Connecting..."))
1601                 apply(callback, (msg,))
1602                 time.sleep(0.1)
1603
1604         # wait until we are connected, because the user might have selected another server
1605         if self.network:
1606             wait_for_network()
1607
1608         self.create_accounts()
1609
1610         if self.network:
1611             wait_for_wallet()
1612         else:
1613             self.synchronize()
1614
1615         self.fill_addressbook()
1616
1617
1618
1619
1620 class WalletSynchronizer(threading.Thread):
1621
1622
1623     def __init__(self, wallet, network):
1624         threading.Thread.__init__(self)
1625         self.daemon = True
1626         self.wallet = wallet
1627         self.network = network
1628         self.was_updated = True
1629         self.running = False
1630         self.lock = threading.Lock()
1631         self.queue = Queue.Queue()
1632
1633     def stop(self):
1634         with self.lock: self.running = False
1635
1636     def is_running(self):
1637         with self.lock: return self.running
1638
1639
1640     def subscribe_to_addresses(self, addresses):
1641         messages = []
1642         for addr in addresses:
1643             messages.append(('blockchain.address.subscribe', [addr]))
1644         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1645
1646
1647     def run(self):
1648         with self.lock:
1649             self.running = True
1650
1651         while self.is_running():
1652
1653             if not self.network.is_connected():
1654                 print_error("synchronizer: waiting for interface")
1655                 self.network.wait_until_connected()
1656
1657             self.run_interface(self.network.interface)
1658
1659
1660     def run_interface(self, interface):
1661
1662         print_error("synchronizer: connected to", interface.server)
1663
1664         requested_tx = []
1665         missing_tx = []
1666         requested_histories = {}
1667
1668         # request any missing transactions
1669         for history in self.wallet.history.values():
1670             if history == ['*']: continue
1671             for tx_hash, tx_height in history:
1672                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1673                     missing_tx.append( (tx_hash, tx_height) )
1674
1675         if missing_tx:
1676             print_error("missing tx", missing_tx)
1677
1678         # subscriptions
1679         self.subscribe_to_addresses(self.wallet.addresses(True))
1680
1681         while self.is_running():
1682             # 1. create new addresses
1683             new_addresses = self.wallet.synchronize()
1684
1685             # request missing addresses
1686             if new_addresses:
1687                 self.subscribe_to_addresses(new_addresses)
1688
1689             # request missing transactions
1690             for tx_hash, tx_height in missing_tx:
1691                 if (tx_hash, tx_height) not in requested_tx:
1692                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1693                     requested_tx.append( (tx_hash, tx_height) )
1694             missing_tx = []
1695
1696             # detect if situation has changed
1697             if interface.is_up_to_date() and self.queue.empty():
1698                 if not self.wallet.is_up_to_date():
1699                     self.wallet.set_up_to_date(True)
1700                     self.was_updated = True
1701             else:
1702                 if self.wallet.is_up_to_date():
1703                     self.wallet.set_up_to_date(False)
1704                     self.was_updated = True
1705
1706             if self.was_updated:
1707                 self.wallet.network.trigger_callback('updated')
1708                 self.was_updated = False
1709
1710             # 2. get a response
1711             try:
1712                 r = self.queue.get(block=True, timeout=1)
1713             except Queue.Empty:
1714                 continue
1715
1716             if interface != self.network.interface:
1717                 break
1718
1719             if not r:
1720                 continue
1721
1722             # 3. handle response
1723             method = r['method']
1724             params = r['params']
1725             result = r.get('result')
1726             error = r.get('error')
1727             if error:
1728                 print "error", r
1729                 continue
1730
1731             if method == 'blockchain.address.subscribe':
1732                 addr = params[0]
1733                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1734                     if requested_histories.get(addr) is None:
1735                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1736                         requested_histories[addr] = result
1737
1738             elif method == 'blockchain.address.get_history':
1739                 addr = params[0]
1740                 print_error("receiving history", addr, result)
1741                 if result == ['*']:
1742                     assert requested_histories.pop(addr) == '*'
1743                     self.wallet.receive_history_callback(addr, result)
1744                 else:
1745                     hist = []
1746                     # check that txids are unique
1747                     txids = []
1748                     for item in result:
1749                         tx_hash = item['tx_hash']
1750                         if tx_hash not in txids:
1751                             txids.append(tx_hash)
1752                             hist.append( (tx_hash, item['height']) )
1753
1754                     if len(hist) != len(result):
1755                         raise BaseException("error: server sent history with non-unique txid", result)
1756
1757                     # check that the status corresponds to what was announced
1758                     rs = requested_histories.pop(addr)
1759                     if self.wallet.get_status(hist) != rs:
1760                         raise BaseException("error: status mismatch: %s"%addr)
1761
1762                     # store received history
1763                     self.wallet.receive_history_callback(addr, hist)
1764
1765                     # request transactions that we don't have
1766                     for tx_hash, tx_height in hist:
1767                         if self.wallet.transactions.get(tx_hash) is None:
1768                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1769                                 missing_tx.append( (tx_hash, tx_height) )
1770
1771             elif method == 'blockchain.transaction.get':
1772                 tx_hash = params[0]
1773                 tx_height = params[1]
1774                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1775                 tx = Transaction(result)
1776                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1777                 self.was_updated = True
1778                 requested_tx.remove( (tx_hash, tx_height) )
1779                 print_error("received tx:", tx_hash, len(tx.raw))
1780
1781             else:
1782                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1783
1784             if self.was_updated and not requested_tx:
1785                 self.wallet.network.trigger_callback('updated')
1786                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1787
1788                 self.was_updated = False