lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         v = self.data.get(key)
 129         if v is None:
 130             v = default
 131         return v
 132
 133     def put(self, key, value, save = True):
 134
 135         with self.lock:
 136             if value is not None:
 137                 self.data[key] = value
 138             else:
 139                 self.data.pop(key)
 140             if save:
 141                 self.write()
 142
 143     def write(self):
 144         s = repr(self.data)
 145         f = open(self.path,"w")
 146         f.write( s )
 147         f.close()
 148         if 'ANDROID_DATA' not in os.environ:
 149             import stat
 150             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 151
 152
 153 class Wallet:
 154
 155     def __init__(self, storage):
 156
 157         self.storage = storage
 158         self.electrum_version = ELECTRUM_VERSION
 159         self.gap_limit_for_change = 3 # constant
 160
 161         # saved fields
 162         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 163
 164         self.gap_limit             = storage.get('gap_limit', 5)
 165         self.use_change            = storage.get('use_change',True)
 166         self.use_encryption        = storage.get('use_encryption', False)
 167         self.seed                  = storage.get('seed', '')               # encrypted
 168         self.labels                = storage.get('labels', {})
 169         self.frozen_addresses      = storage.get('frozen_addresses',[])
 170         self.addressbook           = storage.get('contacts', [])
 171
 172         self.imported_keys         = storage.get('imported_keys',{})
 173         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 174
 175         self.fee                   = int(storage.get('fee_per_kb',20000))
 176
 177         self.master_public_keys = storage.get('master_public_keys',{})
 178         self.master_private_keys = storage.get('master_private_keys', {})
 179
 180         self.next_addresses = storage.get('next_addresses',{})
 181
 182         if self.seed_version not in [4, 6]:
 183             msg = "This wallet seed is not supported."
 184             if self.seed_version in [5]:
 185                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 186                 print msg
 187                 sys.exit(1)
 188
 189         # This attribute is set when wallet.start_threads is called.
 190         self.synchronizer = None
 191
 192         self.load_accounts()
 193
 194         self.transactions = {}
 195         tx_list = self.storage.get('transactions',{})
 196         for k,v in tx_list.items():
 197             try:
 198                 tx = Transaction(v)
 199             except Exception:
 200                 print_msg("Warning: Cannot deserialize transactions. skipping")
 201                 continue
 202
 203             self.add_extra_addresses(tx)
 204             self.transactions[k] = tx
 205
 206         for h,tx in self.transactions.items():
 207             if not self.check_new_tx(h, tx):
 208                 print_error("removing unreferenced tx", h)
 209                 self.transactions.pop(h)
 210
 211
 212         # not saved
 213         self.prevout_values = {}     # my own transaction outputs
 214         self.spent_outputs = []
 215
 216         # spv
 217         self.verifier = None
 218
 219         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 220         # interface.is_up_to_date() returns true when all requests have been answered and processed
 221         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 222
 223         self.up_to_date = False
 224         self.lock = threading.Lock()
 225         self.transaction_lock = threading.Lock()
 226         self.tx_event = threading.Event()
 227
 228         for tx_hash, tx in self.transactions.items():
 229             self.update_tx_outputs(tx_hash)
 230
 231
 232     def add_extra_addresses(self, tx):
 233         h = tx.hash()
 234         # find the address corresponding to pay-to-pubkey inputs
 235         tx.add_extra_addresses(self.transactions)
 236         for o in tx.d.get('outputs'):
 237             if o.get('is_pubkey'):
 238                 for tx2 in self.transactions.values():
 239                     tx2.add_extra_addresses({h:tx})
 240
 241
 242
 243
 244     def set_up_to_date(self,b):
 245         with self.lock: self.up_to_date = b
 246
 247
 248     def is_up_to_date(self):
 249         with self.lock: return self.up_to_date
 250
 251
 252     def update(self):
 253         self.up_to_date = False
 254         while not self.is_up_to_date():
 255             time.sleep(0.1)
 256
 257
 258     def import_key(self, sec, password):
 259         # check password
 260         seed = self.get_seed(password)
 261         try:
 262             address = address_from_private_key(sec)
 263         except Exception:
 264             raise Exception('Invalid private key')
 265
 266         if self.is_mine(address):
 267             raise Exception('Address already in wallet')
 268
 269         # store the originally requested keypair into the imported keys table
 270         self.imported_keys[address] = pw_encode(sec, password )
 271         self.storage.put('imported_keys', self.imported_keys, True)
 272         if self.synchronizer:
 273             self.synchronizer.subscribe_to_addresses([address])
 274         return address
 275
 276     def delete_imported_key(self, addr):
 277         if addr in self.imported_keys:
 278             self.imported_keys.pop(addr)
 279             self.storage.put('imported_keys', self.imported_keys, True)
 280
 281
 282     def make_seed(self):
 283         import mnemonic, ecdsa
 284         entropy = ecdsa.util.randrange( pow(2,160) )
 285         nonce = 0
 286         while True:
 287             ss = "%040x"%(entropy+nonce)
 288             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 289             # we keep only 13 words, that's approximately 139 bits of entropy
 290             words = mnemonic.mn_encode(s)[0:13]
 291             seed = ' '.join(words)
 292             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 293                 break  # this removes 12 bits of entropy
 294             nonce += 1
 295
 296         return seed
 297
 298
 299     def init_seed(self, seed):
 300         import mnemonic
 301
 302         if self.seed:
 303             raise Exception("a seed exists")
 304
 305         if not seed:
 306             self.seed = random_seed(128)
 307             self.seed_version = 4
 308             return
 309
 310         #if not seed:
 311         #    self.seed = self.make_seed()
 312         #    self.seed_version = SEED_VERSION
 313         #    return
 314
 315         # find out what kind of wallet we are
 316         try:
 317             seed.strip().decode('hex')
 318             self.seed_version = 4
 319             self.seed = str(seed)
 320             return
 321         except Exception:
 322             pass
 323
 324         words = seed.split()
 325         self.seed_version = 4
 326         self.seed = mnemonic.mn_decode(words)
 327
 328         #try:
 329         #    mnemonic.mn_decode(words)
 330         #    uses_electrum_words = True
 331         #except Exception:
 332         #    uses_electrum_words = False
 333         #
 334         #if uses_electrum_words and len(words) != 13:
 335         #    self.seed_version = 4
 336         #    self.seed = mnemonic.mn_decode(words)
 337         #else:
 338         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 339         #    self.seed_version = SEED_VERSION
 340         #    self.seed = seed
 341
 342
 343     def save_seed(self, password):
 344         if password:
 345             self.seed = pw_encode( self.seed, password)
 346             self.use_encryption = True
 347         self.storage.put('seed', self.seed, True)
 348         self.storage.put('seed_version', self.seed_version, True)
 349         self.storage.put('use_encryption', self.use_encryption,True)
 350         self.create_accounts(password)
 351
 352
 353     def create_watching_only_wallet(self, params):
 354         K0, c0 = params
 355         if not K0:
 356             return
 357
 358         if not c0:
 359             self.seed_version = 4
 360             self.storage.put('seed_version', self.seed_version, True)
 361             self.create_old_account(K0)
 362             return
 363
 364         cK0 = ""
 365         self.master_public_keys = {
 366             "m/0'/": (c0, K0, cK0),
 367             }
 368         self.storage.put('master_public_keys', self.master_public_keys, True)
 369         self.storage.put('seed_version', self.seed_version, True)
 370         self.create_account('1','Main account')
 371
 372
 373     def create_accounts(self, password):
 374         seed = pw_decode(self.seed, password)
 375
 376         if self.seed_version == 4:
 377             mpk = OldAccount.mpk_from_seed(seed)
 378             self.create_old_account(mpk)
 379         else:
 380             # create default account
 381             self.create_master_keys('1', password)
 382             self.create_account('1','Main account')
 383
 384
 385     def create_master_keys(self, account_type, password):
 386         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 387         if account_type == '1':
 388             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 389             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 390             self.master_private_keys["m/0'/"] = pw_encode(k0, password)
 391         elif account_type == '2of2':
 392             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 393             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 394             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 395             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 396             self.master_private_keys["m/1'/"] = pw_encode(k1, password)
 397             self.master_private_keys["m/2'/"] = pw_encode(k2, password)
 398         elif account_type == '2of3':
 399             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 400             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 401             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 402             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 403             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 404             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 405             self.master_private_keys["m/3'/"] = pw_encode(k3, password)
 406             self.master_private_keys["m/4'/"] = pw_encode(k4, password)
 407             self.master_private_keys["m/5'/"] = pw_encode(k5, password)
 408
 409         self.storage.put('master_public_keys', self.master_public_keys, True)
 410         self.storage.put('master_private_keys', self.master_private_keys, True)
 411
 412     def has_master_public_keys(self, account_type):
 413         if account_type == '1':
 414             return "m/0'/" in self.master_public_keys
 415         elif account_type == '2of2':
 416             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 417         elif account_type == '2of3':
 418             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 419
 420     def find_root_by_master_key(self, c, K):
 421         for key, v in self.master_public_keys.items():
 422             if key == "m/":continue
 423             cc, KK, _ = v
 424             if (c == cc) and (K == KK):
 425                 return key
 426
 427     def deseed_root(self, seed, password):
 428         # for safety, we ask the user to enter their seed
 429         assert seed == self.get_seed(password)
 430         self.seed = ''
 431         self.storage.put('seed', '', True)
 432
 433
 434     def deseed_branch(self, k):
 435         # check that parent has no seed
 436         assert self.seed == ''
 437         self.master_private_keys.pop(k)
 438         self.storage.put('master_private_keys', self.master_private_keys, True)
 439
 440     def is_watching_only(self):
 441         return (self.seed == '') and (self.master_private_keys == {})
 442
 443
 444
 445     def account_id(self, account_type, i):
 446         if account_type == '1':
 447             return "m/0'/%d"%i
 448         elif account_type == '2of2':
 449             return "m/1'/%d & m/2'/%d"%(i,i)
 450         elif account_type == '2of3':
 451             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 452         else:
 453             raise Exception('unknown account type')
 454
 455
 456     def num_accounts(self, account_type):
 457         keys = self.accounts.keys()
 458         i = 0
 459         while True:
 460             account_id = self.account_id(account_type, i)
 461             if account_id not in keys: break
 462             i += 1
 463         return i
 464
 465
 466     def new_account_address(self, account_type = '1'):
 467         i = self.num_accounts(account_type)
 468         k = self.account_id(account_type,i)
 469
 470         addr = self.next_addresses.get(k)
 471         if not addr:
 472             account_id, account = self.next_account(account_type)
 473             addr = account.first_address()
 474             self.next_addresses[k] = addr
 475             self.storage.put('next_addresses',self.next_addresses)
 476
 477         return k, addr
 478
 479
 480     def next_account(self, account_type = '1'):
 481
 482         i = self.num_accounts(account_type)
 483         account_id = self.account_id(account_type,i)
 484
 485         if account_type is '1':
 486             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 487             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 488             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 489
 490         elif account_type == '2of2':
 491             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 492             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 493             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 494             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 495             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 496
 497         elif account_type == '2of3':
 498             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 499             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 500             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 501             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 502             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 503             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 504             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 505
 506         return account_id, account
 507
 508
 509     def set_label(self, name, text = None):
 510         changed = False
 511         old_text = self.labels.get(name)
 512         if text:
 513             if old_text != text:
 514                 self.labels[name] = text
 515                 changed = True
 516         else:
 517             if old_text:
 518                 self.labels.pop(name)
 519                 changed = True
 520
 521         if changed:
 522             self.storage.put('labels', self.labels, True)
 523
 524         run_hook('set_label', name, text, changed)
 525         return changed
 526
 527
 528
 529     def create_account(self, account_type = '1', name = None):
 530         k, account = self.next_account(account_type)
 531         if k in self.pending_accounts:
 532             self.pending_accounts.pop(k)
 533             self.storage.put('pending_accounts', self.pending_accounts)
 534
 535         self.accounts[k] = account
 536         self.save_accounts()
 537         if name:
 538             self.set_label(k, name)
 539
 540
 541     def create_old_account(self, mpk):
 542         self.storage.put('master_public_key', mpk, True)
 543         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 544         self.save_accounts()
 545
 546
 547     def save_accounts(self):
 548         d = {}
 549         for k, v in self.accounts.items():
 550             d[k] = v.dump()
 551         self.storage.put('accounts', d, True)
 552
 553
 554
 555     def load_accounts(self):
 556         d = self.storage.get('accounts', {})
 557         self.accounts = {}
 558         for k, v in d.items():
 559             if k == 0:
 560                 v['mpk'] = self.storage.get('master_public_key')
 561                 self.accounts[k] = OldAccount(v)
 562             elif '&' in k:
 563                 self.accounts[k] = BIP32_Account_2of2(v)
 564             else:
 565                 self.accounts[k] = BIP32_Account(v)
 566
 567         self.pending_accounts = self.storage.get('pending_accounts',{})
 568
 569
 570     def delete_pending_account(self, k):
 571         self.pending_accounts.pop(k)
 572         self.storage.put('pending_accounts', self.pending_accounts)
 573
 574     def account_is_pending(self, k):
 575         return k in self.pending_accounts
 576
 577     def create_pending_account(self, acct_type, name):
 578         k, addr = self.new_account_address(acct_type)
 579         self.set_label(k, name)
 580         self.pending_accounts[k] = addr
 581         self.storage.put('pending_accounts', self.pending_accounts)
 582
 583     def get_pending_accounts(self):
 584         return self.pending_accounts.items()
 585
 586
 587     def addresses(self, include_change = True, _next=True):
 588         o = self.get_account_addresses(-1, include_change)
 589         for a in self.accounts.keys():
 590             o += self.get_account_addresses(a, include_change)
 591
 592         if _next:
 593             for addr in self.next_addresses.values():
 594                 if addr not in o:
 595                     o += [addr]
 596         return o
 597
 598
 599     def is_mine(self, address):
 600         return address in self.addresses(True)
 601
 602
 603     def is_change(self, address):
 604         if not self.is_mine(address): return False
 605         if address in self.imported_keys.keys(): return False
 606         acct, s = self.get_address_index(address)
 607         if s is None: return False
 608         return s[0] == 1
 609
 610     def get_master_public_key(self):
 611         if self.seed_version == 4:
 612             return self.storage.get("master_public_key")
 613         else:
 614             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 615             return repr((c, K))
 616
 617     def get_master_private_key(self, account, password):
 618         master_k = pw_decode( self.master_private_keys[account], password)
 619         master_c, master_K, master_Kc = self.master_public_keys[account]
 620         try:
 621             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 622             assert K.encode('hex') == master_K
 623         except Exception:
 624             raise Exception("Invalid password")
 625         return master_k
 626
 627
 628     def get_address_index(self, address):
 629         if address in self.imported_keys.keys():
 630             return -1, None
 631
 632         for account in self.accounts.keys():
 633             for for_change in [0,1]:
 634                 addresses = self.accounts[account].get_addresses(for_change)
 635                 for addr in addresses:
 636                     if address == addr:
 637                         return account, (for_change, addresses.index(addr))
 638
 639         for k,v in self.next_addresses.items():
 640             if v == address:
 641                 return k, (0,0)
 642
 643         raise Exception("Address not found", address)
 644
 645
 646     def get_roots(self, account):
 647         roots = []
 648         for a in account.split('&'):
 649             s = a.strip()
 650             m = re.match("(m/\d+'/)(\d+)", s)
 651             roots.append( m.group(1) )
 652         return roots
 653
 654     def is_seeded(self, account):
 655         if type(account) is int:
 656             return self.seed is not None
 657
 658         for root in self.get_roots(account):
 659             if root not in self.master_private_keys.keys():
 660                 return False
 661         return True
 662
 663     def rebase_sequence(self, account, sequence):
 664         c, i = sequence
 665         dd = []
 666         for a in account.split('&'):
 667             s = a.strip()
 668             m = re.match("(m/\d+'/)(\d+)", s)
 669             root = m.group(1)
 670             num = int(m.group(2))
 671             dd.append( (root, [num,c,i] ) )
 672         return dd
 673
 674
 675     def get_keyID(self, account, sequence):
 676         if account == 0:
 677             a, b = sequence
 678             mpk = self.storage.get('master_public_key')
 679             return 'old(%s,%d,%d)'%(mpk,a,b)
 680
 681         rs = self.rebase_sequence(account, sequence)
 682         dd = []
 683         for root, public_sequence in rs:
 684             c, K, _ = self.master_public_keys[root]
 685             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 686             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 687         return '&'.join(dd)
 688
 689
 690
 691     def get_seed(self, password):
 692         s = pw_decode(self.seed, password)
 693         if self.seed_version == 4:
 694             seed = s
 695             self.accounts[0].check_seed(seed)
 696         else:
 697             seed = mnemonic_hash(s)
 698         return seed
 699
 700
 701     def get_mnemonic(self, password):
 702         import mnemonic
 703         s = pw_decode(self.seed, password)
 704         if self.seed_version == 4:
 705             return ' '.join(mnemonic.mn_encode(s))
 706         else:
 707             return s
 708
 709
 710
 711     def get_private_key(self, address, password):
 712         if self.is_watching_only():
 713             return []
 714
 715         # first check the provided password
 716         seed = self.get_seed(password)
 717
 718         out = []
 719         if address in self.imported_keys.keys():
 720             out.append( pw_decode( self.imported_keys[address], password ) )
 721         else:
 722             account, sequence = self.get_address_index(address)
 723             if account == 0:
 724                 pk = self.accounts[account].get_private_key(seed, sequence)
 725                 out.append(pk)
 726                 return out
 727
 728             # assert address == self.accounts[account].get_address(*sequence)
 729             rs = self.rebase_sequence( account, sequence)
 730             for root, public_sequence in rs:
 731
 732                 if root not in self.master_private_keys.keys(): continue
 733                 master_k = self.get_master_private_key(root, password)
 734                 master_c, _, _ = self.master_public_keys[root]
 735                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 736                 out.append(pk)
 737
 738         return out
 739
 740
 741     def add_keypairs_from_wallet(self, tx, keypairs, password):
 742         for txin in tx.inputs:
 743             address = txin['address']
 744             if not self.is_mine(address):
 745                 continue
 746             private_keys = self.get_private_key(address, password)
 747             for sec in private_keys:
 748                 pubkey = public_key_from_private_key(sec)
 749                 keypairs[ pubkey ] = sec
 750                 if address in self.imported_keys.keys():
 751                     txin['redeemPubkey'] = pubkey
 752
 753
 754     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 755         # first check the provided password
 756         seed = self.get_seed(password)
 757
 758         for txin in tx.inputs:
 759             keyid = txin.get('KeyID')
 760             if keyid:
 761
 762                 if self.seed_version == 4:
 763                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 764                     if not m: continue
 765                     mpk = m.group(1)
 766                     if mpk != self.storage.get('master_public_key'): continue
 767                     for_change = int(m.group(2))
 768                     num = int(m.group(3))
 769                     account = self.accounts[0]
 770                     addr = account.get_address(for_change, num)
 771                     txin['address'] = addr # fixme: side effect
 772                     pk = account.get_private_key(seed, (for_change, num))
 773                     pubkey = public_key_from_private_key(pk)
 774                     keypairs[pubkey] = pk
 775                     continue
 776
 777
 778                 roots = []
 779                 for s in keyid.split('&'):
 780                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 781                     if not m: continue
 782                     c = m.group(1)
 783                     K = m.group(2)
 784                     sequence = m.group(3)
 785                     root = self.find_root_by_master_key(c,K)
 786                     if not root: continue
 787                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 788                     root = root + '%d'%sequence[0]
 789                     sequence = sequence[1:]
 790                     roots.append((root,sequence))
 791
 792                 account_id = " & ".join( map(lambda x:x[0], roots) )
 793                 account = self.accounts.get(account_id)
 794                 if not account: continue
 795                 addr = account.get_address(*sequence)
 796                 txin['address'] = addr # fixme: side effect
 797                 pk = self.get_private_key(addr, password)
 798                 for sec in pk:
 799                     pubkey = public_key_from_private_key(sec)
 800                     keypairs[pubkey] = sec
 801
 802
 803
 804     def signrawtransaction(self, tx, input_info, private_keys, password):
 805
 806         # check that the password is correct
 807         seed = self.get_seed(password)
 808
 809         # add input info
 810         tx.add_input_info(input_info)
 811
 812         # add redeem script for coins that are in the wallet
 813         # FIXME: add redeemPubkey too!
 814         unspent_coins = self.get_unspent_coins()
 815         for txin in tx.inputs:
 816             for item in unspent_coins:
 817                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 818                     print_error( "tx input is in unspent coins" )
 819                     txin['scriptPubKey'] = item['scriptPubKey']
 820                     account, sequence = self.get_address_index(item['address'])
 821                     if account != -1:
 822                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 823                         print_error("added redeemScript", txin['redeemScript'])
 824                     break
 825
 826
 827         # build a list of public/private keys
 828         keypairs = {}
 829
 830         # add private keys from parameter
 831         for sec in private_keys:
 832             pubkey = public_key_from_private_key(sec)
 833             keypairs[ pubkey ] = sec
 834
 835         # add private_keys from KeyID
 836         self.add_keypairs_from_KeyID(tx, keypairs, password)
 837
 838         # add private keys from wallet
 839         self.add_keypairs_from_wallet(tx, keypairs, password)
 840         self.sign_transaction(tx, keypairs)
 841
 842
 843     def sign_message(self, address, message, password):
 844         keys = self.get_private_key(address, password)
 845         assert len(keys) == 1
 846         sec = keys[0]
 847         key = regenerate_key(sec)
 848         compressed = is_compressed(sec)
 849         return key.sign_message(message, compressed, address)
 850
 851
 852     def change_gap_limit(self, value):
 853         if value >= self.gap_limit:
 854             self.gap_limit = value
 855             self.storage.put('gap_limit', self.gap_limit, True)
 856             #self.interface.poke('synchronizer')
 857             return True
 858
 859         elif value >= self.min_acceptable_gap():
 860             for key, account in self.accounts.items():
 861                 addresses = account[0]
 862                 k = self.num_unused_trailing_addresses(addresses)
 863                 n = len(addresses) - k + value
 864                 addresses = addresses[0:n]
 865                 self.accounts[key][0] = addresses
 866
 867             self.gap_limit = value
 868             self.storage.put('gap_limit', self.gap_limit, True)
 869             self.save_accounts()
 870             return True
 871         else:
 872             return False
 873
 874     def num_unused_trailing_addresses(self, addresses):
 875         k = 0
 876         for a in addresses[::-1]:
 877             if self.history.get(a):break
 878             k = k + 1
 879         return k
 880
 881     def min_acceptable_gap(self):
 882         # fixme: this assumes wallet is synchronized
 883         n = 0
 884         nmax = 0
 885
 886         for account in self.accounts.values():
 887             addresses = account.get_addresses(0)
 888             k = self.num_unused_trailing_addresses(addresses)
 889             for a in addresses[0:-k]:
 890                 if self.history.get(a):
 891                     n = 0
 892                 else:
 893                     n += 1
 894                     if n > nmax: nmax = n
 895         return nmax + 1
 896
 897
 898     def address_is_old(self, address):
 899         age = -1
 900         h = self.history.get(address, [])
 901         if h == ['*']:
 902             return True
 903         for tx_hash, tx_height in h:
 904             if tx_height == 0:
 905                 tx_age = 0
 906             else:
 907                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 908             if tx_age > age:
 909                 age = tx_age
 910         return age > 2
 911
 912
 913     def synchronize_sequence(self, account, for_change):
 914         limit = self.gap_limit_for_change if for_change else self.gap_limit
 915         new_addresses = []
 916         while True:
 917             addresses = account.get_addresses(for_change)
 918             if len(addresses) < limit:
 919                 address = account.create_new_address(for_change)
 920                 self.history[address] = []
 921                 new_addresses.append( address )
 922                 continue
 923
 924             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 925                 break
 926             else:
 927                 address = account.create_new_address(for_change)
 928                 self.history[address] = []
 929                 new_addresses.append( address )
 930
 931         return new_addresses
 932
 933
 934
 935     def create_pending_accounts(self):
 936         for account_type in ['1','2of2','2of3']:
 937             if not self.has_master_public_keys(account_type):
 938                 continue
 939             k, a = self.new_account_address(account_type)
 940             if self.address_is_old(a):
 941                 print_error( "creating account", a )
 942                 self.create_account(account_type)
 943                 self.next_addresses.pop(k)
 944
 945
 946     def synchronize_account(self, account):
 947         new = []
 948         new += self.synchronize_sequence(account, 0)
 949         new += self.synchronize_sequence(account, 1)
 950         return new
 951
 952
 953     def synchronize(self):
 954         if self.master_public_keys:
 955             self.create_pending_accounts()
 956         new = []
 957         for account in self.accounts.values():
 958             new += self.synchronize_account(account)
 959         if new:
 960             self.save_accounts()
 961             self.storage.put('addr_history', self.history, True)
 962         return new
 963
 964
 965     def is_found(self):
 966         return self.history.values() != [[]] * len(self.history)
 967
 968
 969     def add_contact(self, address, label=None):
 970         self.addressbook.append(address)
 971         self.storage.put('contacts', self.addressbook, True)
 972         if label:
 973             self.set_label(address, label)
 974
 975
 976     def delete_contact(self, addr):
 977         if addr in self.addressbook:
 978             self.addressbook.remove(addr)
 979             self.storage.put('addressbook', self.addressbook, True)
 980
 981
 982     def fill_addressbook(self):
 983         for tx_hash, tx in self.transactions.items():
 984             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 985             if is_send:
 986                 for addr, v in tx.outputs:
 987                     if not self.is_mine(addr) and addr not in self.addressbook:
 988                         self.addressbook.append(addr)
 989         # redo labels
 990         # self.update_tx_labels()
 991
 992     def get_num_tx(self, address):
 993         n = 0
 994         for tx in self.transactions.values():
 995             if address in map(lambda x:x[0], tx.outputs): n += 1
 996         return n
 997
 998
 999     def get_address_flags(self, addr):
1000         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
1001         flags += "F" if addr in self.frozen_addresses else "-"
1002         return flags
1003
1004
1005     def get_tx_value(self, tx, account=None):
1006         domain = self.get_account_addresses(account)
1007         return tx.get_value(domain, self.prevout_values)
1008
1009
1010     def update_tx_outputs(self, tx_hash):
1011         tx = self.transactions.get(tx_hash)
1012
1013         for i, (addr, value) in enumerate(tx.outputs):
1014             key = tx_hash+ ':%d'%i
1015             self.prevout_values[key] = value
1016
1017         for item in tx.inputs:
1018             if self.is_mine(item.get('address')):
1019                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1020                 self.spent_outputs.append(key)
1021
1022
1023     def get_addr_balance(self, address):
1024         assert self.is_mine(address)
1025         h = self.history.get(address,[])
1026         if h == ['*']: return 0,0
1027         c = u = 0
1028         received_coins = []   # list of coins received at address
1029
1030         for tx_hash, tx_height in h:
1031             tx = self.transactions.get(tx_hash)
1032             if not tx: continue
1033
1034             for i, (addr, value) in enumerate(tx.outputs):
1035                 if addr == address:
1036                     key = tx_hash + ':%d'%i
1037                     received_coins.append(key)
1038
1039         for tx_hash, tx_height in h:
1040             tx = self.transactions.get(tx_hash)
1041             if not tx: continue
1042             v = 0
1043
1044             for item in tx.inputs:
1045                 addr = item.get('address')
1046                 if addr == address:
1047                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1048                     value = self.prevout_values.get( key )
1049                     if key in received_coins:
1050                         v -= value
1051
1052             for i, (addr, value) in enumerate(tx.outputs):
1053                 key = tx_hash + ':%d'%i
1054                 if addr == address:
1055                     v += value
1056
1057             if tx_height:
1058                 c += v
1059             else:
1060                 u += v
1061         return c, u
1062
1063
1064     def get_account_name(self, k):
1065         if k == 0:
1066             if self.seed_version == 4:
1067                 name = 'Main account'
1068             else:
1069                 name = 'Old account'
1070         else:
1071             default = "Unnamed account"
1072             m = re.match("m/0'/(\d+)", k)
1073             if m:
1074                 num = m.group(1)
1075                 if num == '0':
1076                     default = "Main account"
1077                 else:
1078                     default = "Account %s"%num
1079
1080             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1081             if m:
1082                 num = m.group(1)
1083                 default = "2of2 account %s"%num
1084             name = self.labels.get(k, default)
1085
1086         return name
1087
1088     def get_account_names(self):
1089         accounts = {}
1090         for k, account in self.accounts.items():
1091             accounts[k] = self.get_account_name(k)
1092         if self.imported_keys:
1093             accounts[-1] = 'Imported keys'
1094         return accounts
1095
1096     def get_account_addresses(self, a, include_change=True):
1097         if a is None:
1098             o = self.addresses(True)
1099         elif a == -1:
1100             o = self.imported_keys.keys()
1101         else:
1102             ac = self.accounts[a]
1103             o = ac.get_addresses(0)
1104             if include_change: o += ac.get_addresses(1)
1105         return o
1106
1107     def get_imported_balance(self):
1108         return self.get_balance(self.imported_keys.keys())
1109
1110     def get_account_balance(self, account):
1111         return self.get_balance(self.get_account_addresses(account))
1112
1113     def get_frozen_balance(self):
1114         return self.get_balance(self.frozen_addresses)
1115
1116     def get_balance(self, domain=None):
1117         if domain is None: domain = self.addresses(True)
1118         cc = uu = 0
1119         for addr in domain:
1120             c, u = self.get_addr_balance(addr)
1121             cc += c
1122             uu += u
1123         return cc, uu
1124
1125
1126     def get_unspent_coins(self, domain=None):
1127         coins = []
1128         if domain is None: domain = self.addresses(True)
1129         for addr in domain:
1130             h = self.history.get(addr, [])
1131             if h == ['*']: continue
1132             for tx_hash, tx_height in h:
1133                 tx = self.transactions.get(tx_hash)
1134                 if tx is None: raise Exception("Wallet not synchronized")
1135                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1136                 for o in tx.d.get('outputs'):
1137                     output = o.copy()
1138                     if output.get('address') != addr: continue
1139                     key = tx_hash + ":%d" % output.get('prevout_n')
1140                     if key in self.spent_outputs: continue
1141                     output['prevout_hash'] = tx_hash
1142                     output['height'] = tx_height
1143                     output['coinbase'] = is_coinbase
1144                     coins.append((tx_height, output))
1145
1146         # sort by age
1147         if coins:
1148             coins = sorted(coins)
1149             if coins[-1][0] != 0:
1150                 while coins[0][0] == 0:
1151                     coins = coins[1:] + [ coins[0] ]
1152         return [x[1] for x in coins]
1153
1154
1155     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1156         """ todo: minimize tx size """
1157         total = 0
1158         fee = self.fee if fixed_fee is None else fixed_fee
1159         if domain is None:
1160             domain = self.addresses(True)
1161
1162         for i in self.frozen_addresses:
1163             if i in domain: domain.remove(i)
1164
1165         coins = self.get_unspent_coins(domain)
1166         inputs = []
1167
1168         for item in coins:
1169             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1170                 continue
1171             addr = item.get('address')
1172             v = item.get('value')
1173             total += v
1174             inputs.append(item)
1175             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1176             if total >= amount + fee: break
1177         else:
1178             inputs = []
1179
1180         return inputs, total, fee
1181
1182
1183     def set_fee(self, fee):
1184         if self.fee != fee:
1185             self.fee = fee
1186             self.storage.put('fee_per_kb', self.fee, True)
1187
1188     def estimated_fee(self, inputs):
1189         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1190         fee = self.fee * int(round(estimated_size/1024.))
1191         if fee == 0: fee = self.fee
1192         return fee
1193
1194
1195     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1196         "add change to a transaction"
1197         change_amount = total - ( amount + fee )
1198         if change_amount > DUST_THRESHOLD:
1199             if not change_addr:
1200
1201                 # send change to one of the accounts involved in the tx
1202                 address = inputs[0].get('address')
1203                 account, _ = self.get_address_index(address)
1204
1205                 if not self.use_change or account == -1:
1206                     change_addr = inputs[-1]['address']
1207                 else:
1208                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1209
1210             # Insert the change output at a random position in the outputs
1211             posn = random.randint(0, len(outputs))
1212             outputs[posn:posn] = [( change_addr,  change_amount)]
1213         return outputs
1214
1215
1216     def get_history(self, address):
1217         with self.lock:
1218             return self.history.get(address)
1219
1220
1221     def get_status(self, h):
1222         if not h: return None
1223         if h == ['*']: return '*'
1224         status = ''
1225         for tx_hash, height in h:
1226             status += tx_hash + ':%d:' % height
1227         return hashlib.sha256( status ).digest().encode('hex')
1228
1229
1230     def receive_tx_callback(self, tx_hash, tx, tx_height):
1231
1232         with self.transaction_lock:
1233             self.add_extra_addresses(tx)
1234             if not self.check_new_tx(tx_hash, tx):
1235                 # may happen due to pruning
1236                 print_error("received transaction that is no longer referenced in history", tx_hash)
1237                 return
1238             self.transactions[tx_hash] = tx
1239             self.network.interface.pending_transactions_for_notifications.append(tx)
1240             self.save_transactions()
1241             if self.verifier and tx_height>0:
1242                 self.verifier.add(tx_hash, tx_height)
1243             self.update_tx_outputs(tx_hash)
1244
1245
1246     def save_transactions(self):
1247         tx = {}
1248         for k,v in self.transactions.items():
1249             tx[k] = str(v)
1250         self.storage.put('transactions', tx, True)
1251
1252     def receive_history_callback(self, addr, hist):
1253
1254         if not self.check_new_history(addr, hist):
1255             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1256
1257         with self.lock:
1258             self.history[addr] = hist
1259             self.storage.put('addr_history', self.history, True)
1260
1261         if hist != ['*']:
1262             for tx_hash, tx_height in hist:
1263                 if tx_height>0:
1264                     # add it in case it was previously unconfirmed
1265                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1266
1267
1268     def get_tx_history(self, account=None):
1269         if not self.verifier:
1270             return []
1271
1272         with self.transaction_lock:
1273             history = self.transactions.items()
1274             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1275             result = []
1276
1277             balance = 0
1278             for tx_hash, tx in history:
1279                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1280                 if v is not None: balance += v
1281
1282             c, u = self.get_account_balance(account)
1283
1284             if balance != c+u:
1285                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1286
1287             balance = c + u - balance
1288             for tx_hash, tx in history:
1289                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1290                 if not is_relevant:
1291                     continue
1292                 if value is not None:
1293                     balance += value
1294
1295                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1296                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1297
1298         return result
1299
1300
1301     def get_label(self, tx_hash):
1302         label = self.labels.get(tx_hash)
1303         is_default = (label == '') or (label is None)
1304         if is_default: label = self.get_default_label(tx_hash)
1305         return label, is_default
1306
1307
1308     def get_default_label(self, tx_hash):
1309         tx = self.transactions.get(tx_hash)
1310         default_label = ''
1311         if tx:
1312             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1313             if is_mine:
1314                 for o in tx.outputs:
1315                     o_addr, _ = o
1316                     if not self.is_mine(o_addr):
1317                         try:
1318                             default_label = self.labels[o_addr]
1319                         except KeyError:
1320                             default_label = o_addr
1321                         break
1322                 else:
1323                     default_label = '(internal)'
1324             else:
1325                 for o in tx.outputs:
1326                     o_addr, _ = o
1327                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1328                         break
1329                 else:
1330                     for o in tx.outputs:
1331                         o_addr, _ = o
1332                         if self.is_mine(o_addr):
1333                             break
1334                     else:
1335                         o_addr = None
1336
1337                 if o_addr:
1338                     dest_label = self.labels.get(o_addr)
1339                     try:
1340                         default_label = self.labels[o_addr]
1341                     except KeyError:
1342                         default_label = o_addr
1343
1344         return default_label
1345
1346
1347     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1348         for address, x in outputs:
1349             assert is_valid(address)
1350         amount = sum( map(lambda x:x[1], outputs) )
1351         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1352         if not inputs:
1353             raise ValueError("Not enough funds")
1354         self.add_input_info(inputs)
1355         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1356         return Transaction.from_io(inputs, outputs)
1357
1358
1359     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1360         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1361         keypairs = {}
1362         self.add_keypairs_from_wallet(tx, keypairs, password)
1363         if keypairs:
1364             self.sign_transaction(tx, keypairs)
1365         return tx
1366
1367
1368     def add_input_info(self, inputs):
1369         for txin in inputs:
1370             address = txin['address']
1371             if address in self.imported_keys.keys():
1372                 continue
1373             account, sequence = self.get_address_index(address)
1374             txin['KeyID'] = self.get_keyID(account, sequence)
1375             redeemScript = self.accounts[account].redeem_script(sequence)
1376             if redeemScript:
1377                 txin['redeemScript'] = redeemScript
1378             else:
1379                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1380
1381
1382     def sign_transaction(self, tx, keypairs):
1383         tx.sign(keypairs)
1384         run_hook('sign_transaction', tx)
1385
1386
1387     def sendtx(self, tx):
1388         # synchronous
1389         h = self.send_tx(tx)
1390         self.tx_event.wait()
1391         return self.receive_tx(h, tx)
1392
1393     def send_tx(self, tx):
1394         # asynchronous
1395         self.tx_event.clear()
1396         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1397         return tx.hash()
1398
1399     def on_broadcast(self, i, r):
1400         self.tx_result = r.get('result')
1401         self.tx_event.set()
1402
1403     def receive_tx(self, tx_hash, tx):
1404         out = self.tx_result
1405         if out != tx_hash:
1406             return False, "error: " + out
1407         run_hook('receive_tx', tx, self)
1408         return True, out
1409
1410
1411
1412     def update_password(self, old_password, new_password):
1413         if new_password == '': new_password = None
1414         decoded = self.get_seed(old_password)
1415         self.seed = pw_encode( decoded, new_password)
1416         self.storage.put('seed', self.seed, True)
1417         self.use_encryption = (new_password != None)
1418         self.storage.put('use_encryption', self.use_encryption,True)
1419         for k in self.imported_keys.keys():
1420             a = self.imported_keys[k]
1421             b = pw_decode(a, old_password)
1422             c = pw_encode(b, new_password)
1423             self.imported_keys[k] = c
1424         self.storage.put('imported_keys', self.imported_keys, True)
1425
1426         for k, v in self.master_private_keys.items():
1427             b = pw_decode(v, old_password)
1428             c = pw_encode(b, new_password)
1429             self.master_private_keys[k] = c
1430         self.storage.put('master_private_keys', self.master_private_keys, True)
1431
1432
1433     def freeze(self,addr):
1434         if self.is_mine(addr) and addr not in self.frozen_addresses:
1435             self.frozen_addresses.append(addr)
1436             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1437             return True
1438         else:
1439             return False
1440
1441     def unfreeze(self,addr):
1442         if self.is_mine(addr) and addr in self.frozen_addresses:
1443             self.frozen_addresses.remove(addr)
1444             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1445             return True
1446         else:
1447             return False
1448
1449
1450     def set_verifier(self, verifier):
1451         self.verifier = verifier
1452
1453         # review transactions that are in the history
1454         for addr, hist in self.history.items():
1455             if hist == ['*']: continue
1456             for tx_hash, tx_height in hist:
1457                 if tx_height>0:
1458                     # add it in case it was previously unconfirmed
1459                     self.verifier.add(tx_hash, tx_height)
1460
1461
1462         # if we are on a pruning server, remove unverified transactions
1463         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1464         for tx_hash in self.transactions.keys():
1465             if tx_hash not in vr:
1466                 self.transactions.pop(tx_hash)
1467
1468
1469
1470     def check_new_history(self, addr, hist):
1471
1472         # check that all tx in hist are relevant
1473         if hist != ['*']:
1474             for tx_hash, height in hist:
1475                 tx = self.transactions.get(tx_hash)
1476                 if not tx: continue
1477                 if not tx.has_address(addr):
1478                     return False
1479
1480         # check that we are not "orphaning" a transaction
1481         old_hist = self.history.get(addr,[])
1482         if old_hist == ['*']: return True
1483
1484         for tx_hash, height in old_hist:
1485             if tx_hash in map(lambda x:x[0], hist): continue
1486             found = False
1487             for _addr, _hist in self.history.items():
1488                 if _addr == addr: continue
1489                 if _hist == ['*']: continue
1490                 _tx_hist = map(lambda x:x[0], _hist)
1491                 if tx_hash in _tx_hist:
1492                     found = True
1493                     break
1494
1495             if not found:
1496                 tx = self.transactions.get(tx_hash)
1497                 # tx might not be there
1498                 if not tx: continue
1499
1500                 # already verified?
1501                 if self.verifier.get_height(tx_hash):
1502                     continue
1503                 # unconfirmed tx
1504                 print_error("new history is orphaning transaction:", tx_hash)
1505                 # check that all outputs are not mine, request histories
1506                 ext_requests = []
1507                 for _addr, _v in tx.outputs:
1508                     # assert not self.is_mine(_addr)
1509                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1510
1511                 ext_h = self.network.synchronous_get(ext_requests)
1512                 print_error("sync:", ext_requests, ext_h)
1513                 height = None
1514                 for h in ext_h:
1515                     if h == ['*']: continue
1516                     for item in h:
1517                         if item.get('tx_hash') == tx_hash:
1518                             height = item.get('height')
1519                 if height:
1520                     print_error("found height for", tx_hash, height)
1521                     self.verifier.add(tx_hash, height)
1522                 else:
1523                     print_error("removing orphaned tx from history", tx_hash)
1524                     self.transactions.pop(tx_hash)
1525
1526         return True
1527
1528
1529
1530     def check_new_tx(self, tx_hash, tx):
1531         # 1 check that tx is referenced in addr_history.
1532         addresses = []
1533         for addr, hist in self.history.items():
1534             if hist == ['*']:continue
1535             for txh, height in hist:
1536                 if txh == tx_hash:
1537                     addresses.append(addr)
1538
1539         if not addresses:
1540             return False
1541
1542         # 2 check that referencing addresses are in the tx
1543         for addr in addresses:
1544             if not tx.has_address(addr):
1545                 return False
1546
1547         return True
1548
1549
1550     def start_threads(self, network):
1551         from verifier import TxVerifier
1552         self.network = network
1553         if self.network is not None:
1554             self.verifier = TxVerifier(self.network, self.storage)
1555             self.verifier.start()
1556             self.set_verifier(self.verifier)
1557             self.synchronizer = WalletSynchronizer(self, network)
1558             self.synchronizer.start()
1559         else:
1560             self.verifier = None
1561             self.synchronizer =None
1562
1563     def stop_threads(self):
1564         if self.network:
1565             self.verifier.stop()
1566             self.synchronizer.stop()
1567
1568
1569     def restore(self, callback):
1570         from i18n import _
1571         def wait_for_wallet():
1572             self.set_up_to_date(False)
1573             while not self.is_up_to_date():
1574                 msg = "%s\n%s %d\n%s %.1f"%(
1575                     _("Please wait..."),
1576                     _("Addresses generated:"),
1577                     len(self.addresses(True)),
1578                     _("Kilobytes received:"),
1579                     self.network.interface.bytes_received/1024.)
1580
1581                 apply(callback, (msg,))
1582                 time.sleep(0.1)
1583
1584         def wait_for_network():
1585             while not self.network.is_connected():
1586                 msg = "%s \n" % (_("Connecting..."))
1587                 apply(callback, (msg,))
1588                 time.sleep(0.1)
1589
1590         # wait until we are connected, because the user might have selected another server
1591         if self.network:
1592             wait_for_network()
1593             wait_for_wallet()
1594         else:
1595             self.synchronize()
1596
1597         self.fill_addressbook()
1598
1599
1600
1601
1602 class WalletSynchronizer(threading.Thread):
1603
1604
1605     def __init__(self, wallet, network):
1606         threading.Thread.__init__(self)
1607         self.daemon = True
1608         self.wallet = wallet
1609         self.network = network
1610         self.was_updated = True
1611         self.running = False
1612         self.lock = threading.Lock()
1613         self.queue = Queue.Queue()
1614
1615     def stop(self):
1616         with self.lock: self.running = False
1617
1618     def is_running(self):
1619         with self.lock: return self.running
1620
1621
1622     def subscribe_to_addresses(self, addresses):
1623         messages = []
1624         for addr in addresses:
1625             messages.append(('blockchain.address.subscribe', [addr]))
1626         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1627
1628
1629     def run(self):
1630         with self.lock:
1631             self.running = True
1632
1633         while self.is_running():
1634
1635             if not self.network.is_connected():
1636                 self.network.wait_until_connected()
1637
1638             self.run_interface()
1639
1640
1641     def run_interface(self):
1642
1643         print_error("synchronizer: connected to", self.network.main_server())
1644
1645         requested_tx = []
1646         missing_tx = []
1647         requested_histories = {}
1648
1649         # request any missing transactions
1650         for history in self.wallet.history.values():
1651             if history == ['*']: continue
1652             for tx_hash, tx_height in history:
1653                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1654                     missing_tx.append( (tx_hash, tx_height) )
1655
1656         if missing_tx:
1657             print_error("missing tx", missing_tx)
1658
1659         # subscriptions
1660         self.subscribe_to_addresses(self.wallet.addresses(True))
1661
1662         while self.is_running():
1663             # 1. create new addresses
1664             new_addresses = self.wallet.synchronize()
1665
1666             # request missing addresses
1667             if new_addresses:
1668                 self.subscribe_to_addresses(new_addresses)
1669
1670             # request missing transactions
1671             for tx_hash, tx_height in missing_tx:
1672                 if (tx_hash, tx_height) not in requested_tx:
1673                     self.network.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1674                     requested_tx.append( (tx_hash, tx_height) )
1675             missing_tx = []
1676
1677             # detect if situation has changed
1678             if self.network.is_up_to_date() and self.queue.empty():
1679                 if not self.wallet.is_up_to_date():
1680                     self.wallet.set_up_to_date(True)
1681                     self.was_updated = True
1682             else:
1683                 if self.wallet.is_up_to_date():
1684                     self.wallet.set_up_to_date(False)
1685                     self.was_updated = True
1686
1687             if self.was_updated:
1688                 self.network.trigger_callback('updated')
1689                 self.was_updated = False
1690
1691             # 2. get a response
1692             try:
1693                 r = self.queue.get(block=True, timeout=1)
1694             except Queue.Empty:
1695                 continue
1696
1697             # see if it changed
1698             #if interface != self.network.interface:
1699             #    break
1700
1701             if not r:
1702                 continue
1703
1704             # 3. handle response
1705             method = r['method']
1706             params = r['params']
1707             result = r.get('result')
1708             error = r.get('error')
1709             if error:
1710                 print "error", r
1711                 continue
1712
1713             if method == 'blockchain.address.subscribe':
1714                 addr = params[0]
1715                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1716                     if requested_histories.get(addr) is None:
1717                         self.network.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1718                         requested_histories[addr] = result
1719
1720             elif method == 'blockchain.address.get_history':
1721                 addr = params[0]
1722                 print_error("receiving history", addr, result)
1723                 if result == ['*']:
1724                     assert requested_histories.pop(addr) == '*'
1725                     self.wallet.receive_history_callback(addr, result)
1726                 else:
1727                     hist = []
1728                     # check that txids are unique
1729                     txids = []
1730                     for item in result:
1731                         tx_hash = item['tx_hash']
1732                         if tx_hash not in txids:
1733                             txids.append(tx_hash)
1734                             hist.append( (tx_hash, item['height']) )
1735
1736                     if len(hist) != len(result):
1737                         raise Exception("error: server sent history with non-unique txid", result)
1738
1739                     # check that the status corresponds to what was announced
1740                     rs = requested_histories.pop(addr)
1741                     if self.wallet.get_status(hist) != rs:
1742                         raise Exception("error: status mismatch: %s"%addr)
1743
1744                     # store received history
1745                     self.wallet.receive_history_callback(addr, hist)
1746
1747                     # request transactions that we don't have
1748                     for tx_hash, tx_height in hist:
1749                         if self.wallet.transactions.get(tx_hash) is None:
1750                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1751                                 missing_tx.append( (tx_hash, tx_height) )
1752
1753             elif method == 'blockchain.transaction.get':
1754                 tx_hash = params[0]
1755                 tx_height = params[1]
1756                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1757                 tx = Transaction(result)
1758                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1759                 self.was_updated = True
1760                 requested_tx.remove( (tx_hash, tx_height) )
1761                 print_error("received tx:", tx_hash, len(tx.raw))
1762
1763             else:
1764                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1765
1766             if self.was_updated and not requested_tx:
1767                 self.network.trigger_callback('updated')
1768                 # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1769                 self.network.trigger_callback("new_transaction")
1770                 self.was_updated = False