lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40
  41 # AES encryption
  42 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  43 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  44
  45 def pw_encode(s, password):
  46     if password:
  47         secret = Hash(password)
  48         return EncodeAES(secret, s)
  49     else:
  50         return s
  51
  52 def pw_decode(s, password):
  53     if password is not None:
  54         secret = Hash(password)
  55         try:
  56             d = DecodeAES(secret, s)
  57         except:
  58             raise BaseException('Invalid password')
  59         return d
  60     else:
  61         return s
  62
  63
  64
  65
  66
  67 from version import *
  68
  69
  70 class WalletStorage:
  71
  72     def __init__(self, config):
  73         self.lock = threading.Lock()
  74         self.data = {}
  75         self.file_exists = False
  76         self.path = self.init_path(config)
  77         print_error( "wallet path", self.path )
  78         if self.path:
  79             self.read(self.path)
  80
  81
  82     def init_path(self, config):
  83         """Set the path of the wallet."""
  84
  85         # command line -w option
  86         path = config.get('wallet_path')
  87         if path:
  88             return path
  89
  90         # path in config file
  91         path = config.get('default_wallet_path')
  92         if path:
  93             return path
  94
  95         # default path
  96         dirpath = os.path.join(config.path, "wallets")
  97         if not os.path.exists(dirpath):
  98             os.mkdir(dirpath)
  99
 100         new_path = os.path.join(config.path, "wallets", "default_wallet")
 101
 102         # default path in pre 1.9 versions
 103         old_path = os.path.join(config.path, "electrum.dat")
 104         if os.path.exists(old_path) and not os.path.exists(new_path):
 105             os.rename(old_path, new_path)
 106
 107         return new_path
 108
 109
 110     def read(self, path):
 111         """Read the contents of the wallet file."""
 112         try:
 113             with open(self.path, "r") as f:
 114                 data = f.read()
 115         except IOError:
 116             return
 117         try:
 118             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 119         except:
 120             raise IOError("Cannot read wallet file.")
 121
 122         self.data = d
 123         self.file_exists = True
 124
 125
 126     def get(self, key, default=None):
 127         return self.data.get(key, default)
 128
 129     def put(self, key, value, save = True):
 130
 131         with self.lock:
 132             if value is not None:
 133                 self.data[key] = value
 134             else:
 135                 self.data.pop[key]
 136             if save:
 137                 self.write()
 138
 139     def write(self):
 140         s = repr(self.data)
 141         f = open(self.path,"w")
 142         f.write( s )
 143         f.close()
 144         if 'ANDROID_DATA' not in os.environ:
 145             import stat
 146             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 147
 148
 149 class Wallet:
 150
 151     def __init__(self, storage):
 152
 153         self.storage = storage
 154         self.electrum_version = ELECTRUM_VERSION
 155         self.gap_limit_for_change = 3 # constant
 156
 157         # saved fields
 158         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 159
 160         self.gap_limit             = storage.get('gap_limit', 5)
 161         self.use_change            = storage.get('use_change',True)
 162         self.use_encryption        = storage.get('use_encryption', False)
 163         self.seed                  = storage.get('seed', '')               # encrypted
 164         self.labels                = storage.get('labels', {})
 165         self.frozen_addresses      = storage.get('frozen_addresses',[])
 166         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 167         self.addressbook           = storage.get('contacts', [])
 168
 169         self.imported_keys         = storage.get('imported_keys',{})
 170         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 171
 172         self.fee                   = int(storage.get('fee_per_kb',20000))
 173
 174         self.master_public_keys = storage.get('master_public_keys',{})
 175         self.master_private_keys = storage.get('master_private_keys', {})
 176
 177         self.next_addresses = storage.get('next_addresses',{})
 178
 179         if self.seed_version not in [4, 6]:
 180             msg = "This wallet seed is not supported."
 181             if self.seed_version in [5]:
 182                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 183                 print msg
 184                 sys.exit(1)
 185
 186
 187         self.load_accounts()
 188
 189         self.transactions = {}
 190         tx_list = self.storage.get('transactions',{})
 191         for k,v in tx_list.items():
 192             try:
 193                 tx = Transaction(v)
 194             except:
 195                 print_msg("Warning: Cannot deserialize transactions. skipping")
 196                 continue
 197
 198             self.add_extra_addresses(tx)
 199             self.transactions[k] = tx
 200
 201         for h,tx in self.transactions.items():
 202             if not self.check_new_tx(h, tx):
 203                 print_error("removing unreferenced tx", h)
 204                 self.transactions.pop(h)
 205
 206
 207         # not saved
 208         self.prevout_values = {}     # my own transaction outputs
 209         self.spent_outputs = []
 210
 211         # spv
 212         self.verifier = None
 213
 214         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 215         # interface.is_up_to_date() returns true when all requests have been answered and processed
 216         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 217
 218         self.up_to_date = False
 219         self.lock = threading.Lock()
 220         self.transaction_lock = threading.Lock()
 221         self.tx_event = threading.Event()
 222
 223         for tx_hash, tx in self.transactions.items():
 224             self.update_tx_outputs(tx_hash)
 225
 226
 227     def add_extra_addresses(self, tx):
 228         h = tx.hash()
 229         # find the address corresponding to pay-to-pubkey inputs
 230         tx.add_extra_addresses(self.transactions)
 231         for o in tx.d.get('outputs'):
 232             if o.get('is_pubkey'):
 233                 for tx2 in self.transactions.values():
 234                     tx2.add_extra_addresses({h:tx})
 235
 236
 237
 238
 239     def set_up_to_date(self,b):
 240         with self.lock: self.up_to_date = b
 241
 242
 243     def is_up_to_date(self):
 244         with self.lock: return self.up_to_date
 245
 246
 247     def update(self):
 248         self.up_to_date = False
 249         while not self.is_up_to_date():
 250             time.sleep(0.1)
 251
 252
 253     def import_key(self, sec, password):
 254         # check password
 255         seed = self.get_seed(password)
 256         try:
 257             address = address_from_private_key(sec)
 258         except:
 259             raise BaseException('Invalid private key')
 260
 261         if self.is_mine(address):
 262             raise BaseException('Address already in wallet')
 263
 264         # store the originally requested keypair into the imported keys table
 265         self.imported_keys[address] = pw_encode(sec, password )
 266         self.storage.put('imported_keys', self.imported_keys, True)
 267         if self.synchronizer:
 268             self.synchronizer.subscribe_to_addresses([address])
 269         return address
 270
 271     def delete_imported_key(self, addr):
 272         if addr in self.imported_keys:
 273             self.imported_keys.pop(addr)
 274             self.storage.put('imported_keys', self.imported_keys, True)
 275
 276
 277     def make_seed(self):
 278         import mnemonic, ecdsa
 279         entropy = ecdsa.util.randrange( pow(2,160) )
 280         nonce = 0
 281         while True:
 282             ss = "%040x"%(entropy+nonce)
 283             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 284             # we keep only 13 words, that's approximately 139 bits of entropy
 285             words = mnemonic.mn_encode(s)[0:13]
 286             seed = ' '.join(words)
 287             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 288                 break  # this removes 12 bits of entropy
 289             nonce += 1
 290
 291         return seed
 292
 293
 294     def init_seed(self, seed):
 295         import mnemonic
 296
 297         if self.seed:
 298             raise BaseException("a seed exists")
 299
 300         if not seed:
 301             self.seed = random_seed(128)
 302             self.seed_version = 4
 303             return
 304
 305         #if not seed:
 306         #    self.seed = self.make_seed()
 307         #    self.seed_version = SEED_VERSION
 308         #    return
 309
 310         # find out what kind of wallet we are
 311         try:
 312             seed.decode('hex')
 313             self.seed_version = 4
 314             return
 315         except:
 316             pass
 317
 318         words = seed.split()
 319         self.seed_version = 4
 320         self.seed = mnemonic.mn_decode(words)
 321
 322         #try:
 323         #    mnemonic.mn_decode(words)
 324         #    uses_electrum_words = True
 325         #except:
 326         #    uses_electrum_words = False
 327         #
 328         #if uses_electrum_words and len(words) != 13:
 329         #    self.seed_version = 4
 330         #    self.seed = mnemonic.mn_decode(words)
 331         #else:
 332         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 333         #    self.seed_version = SEED_VERSION
 334         #    self.seed = seed
 335
 336
 337     def save_seed(self):
 338         self.storage.put('seed', self.seed, True)
 339         self.storage.put('seed_version', self.seed_version, True)
 340
 341     def create_watching_only_wallet(self, c0, K0):
 342         cK0 = ""
 343         self.master_public_keys = {
 344             "m/0'/": (c0, K0, cK0),
 345             }
 346         self.storage.put('master_public_keys', self.master_public_keys, True)
 347         self.create_account('1','Main account')
 348
 349
 350     def create_accounts(self):
 351         if self.seed_version == 4:
 352             self.create_old_account()
 353         else:
 354             # create default account
 355             self.create_master_keys('1')
 356             self.create_account('1','Main account')
 357
 358
 359     def create_master_keys(self, account_type):
 360         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 361         if account_type == '1':
 362             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 363             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 364             self.master_private_keys["m/0'/"] = k0
 365         elif account_type == '2of2':
 366             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 367             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 368             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 369             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 370             self.master_private_keys["m/1'/"] = k1
 371             self.master_private_keys["m/2'/"] = k2
 372         elif account_type == '2of3':
 373             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 374             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 375             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 376             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 377             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 378             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 379             self.master_private_keys["m/3'/"] = k3
 380             self.master_private_keys["m/4'/"] = k4
 381             self.master_private_keys["m/5'/"] = k5
 382
 383         self.storage.put('master_public_keys', self.master_public_keys, True)
 384         self.storage.put('master_private_keys', self.master_private_keys, True)
 385
 386     def has_master_public_keys(self, account_type):
 387         if account_type == '1':
 388             return "m/0'/" in self.master_public_keys
 389         elif account_type == '2of2':
 390             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 391         elif account_type == '2of3':
 392             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 393
 394     def find_root_by_master_key(self, c, K):
 395         for key, v in self.master_public_keys.items():
 396             if key == "m/":continue
 397             cc, KK, _ = v
 398             if (c == cc) and (K == KK):
 399                 return key
 400
 401     def deseed_root(self, seed, password):
 402         # for safety, we ask the user to enter their seed
 403         assert seed == self.get_seed(password)
 404         self.seed = ''
 405         self.storage.put('seed', '', True)
 406
 407
 408     def deseed_branch(self, k):
 409         # check that parent has no seed
 410         assert self.seed == ''
 411         self.master_private_keys.pop(k)
 412         self.storage.put('master_private_keys', self.master_private_keys, True)
 413
 414     def is_watching_only(self):
 415         return (self.seed == '') and (self.master_private_keys == {})
 416
 417
 418
 419     def account_id(self, account_type, i):
 420         if account_type == '1':
 421             return "m/0'/%d"%i
 422         elif account_type == '2of2':
 423             return "m/1'/%d & m/2'/%d"%(i,i)
 424         elif account_type == '2of3':
 425             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 426         else:
 427             raise BaseException('unknown account type')
 428
 429
 430     def num_accounts(self, account_type):
 431         keys = self.accounts.keys()
 432         i = 0
 433         while True:
 434             account_id = self.account_id(account_type, i)
 435             if account_id not in keys: break
 436             i += 1
 437         return i
 438
 439
 440     def new_account_address(self, account_type = '1'):
 441         i = self.num_accounts(account_type)
 442         k = self.account_id(account_type,i)
 443
 444         addr = self.next_addresses.get(k)
 445         if not addr:
 446             account_id, account = self.next_account(account_type)
 447             addr = account.first_address()
 448             self.next_addresses[k] = addr
 449             self.storage.put('next_addresses',self.next_addresses)
 450
 451         return k, addr
 452
 453
 454     def next_account(self, account_type = '1'):
 455
 456         i = self.num_accounts(account_type)
 457         account_id = self.account_id(account_type,i)
 458
 459         if account_type is '1':
 460             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 461             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 462             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 463
 464         elif account_type == '2of2':
 465             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 466             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 467             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 468             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 469             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 470
 471         elif account_type == '2of3':
 472             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 473             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 474             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 475             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 476             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 477             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 478             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 479
 480         return account_id, account
 481
 482
 483     def set_label(self, name, text = None):
 484         changed = False
 485         old_text = self.labels.get(name)
 486         if text:
 487             if old_text != text:
 488                 self.labels[name] = text
 489                 changed = True
 490         else:
 491             if old_text:
 492                 self.labels.pop(name)
 493                 changed = True
 494
 495         if changed:
 496             self.storage.put('labels', self.labels, True)
 497
 498         run_hook('set_label', name, text, changed)
 499         return changed
 500
 501
 502
 503     def create_account(self, account_type = '1', name = None):
 504         k, account = self.next_account(account_type)
 505         if k in self.pending_accounts:
 506             self.pending_accounts.pop(k)
 507             self.storage.put('pending_accounts', self.pending_accounts)
 508
 509         self.accounts[k] = account
 510         self.save_accounts()
 511         if name:
 512             self.set_label(k, name)
 513
 514
 515     def create_old_account(self):
 516         mpk = OldAccount.mpk_from_seed(self.seed)
 517         self.storage.put('master_public_key', mpk, True)
 518         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 519         self.save_accounts()
 520
 521
 522     def save_accounts(self):
 523         d = {}
 524         for k, v in self.accounts.items():
 525             d[k] = v.dump()
 526         self.storage.put('accounts', d, True)
 527
 528
 529
 530     def load_accounts(self):
 531         d = self.storage.get('accounts', {})
 532         self.accounts = {}
 533         for k, v in d.items():
 534             if k == 0:
 535                 v['mpk'] = self.storage.get('master_public_key')
 536                 self.accounts[k] = OldAccount(v)
 537             elif '&' in k:
 538                 self.accounts[k] = BIP32_Account_2of2(v)
 539             else:
 540                 self.accounts[k] = BIP32_Account(v)
 541
 542         self.pending_accounts = self.storage.get('pending_accounts',{})
 543
 544
 545     def delete_pending_account(self, k):
 546         self.pending_accounts.pop(k)
 547         self.storage.put('pending_accounts', self.pending_accounts)
 548
 549     def account_is_pending(self, k):
 550         return k in self.pending_accounts
 551
 552     def create_pending_account(self, acct_type, name):
 553         k, addr = self.new_account_address(acct_type)
 554         self.set_label(k, name)
 555         self.pending_accounts[k] = addr
 556         self.storage.put('pending_accounts', self.pending_accounts)
 557
 558     def get_pending_accounts(self):
 559         return self.pending_accounts.items()
 560
 561
 562     def addresses(self, include_change = True, _next=True):
 563         o = self.get_account_addresses(-1, include_change)
 564         for a in self.accounts.keys():
 565             o += self.get_account_addresses(a, include_change)
 566
 567         if _next:
 568             for addr in self.next_addresses.values():
 569                 if addr not in o:
 570                     o += [addr]
 571         return o
 572
 573
 574     def is_mine(self, address):
 575         return address in self.addresses(True)
 576
 577
 578     def is_change(self, address):
 579         if not self.is_mine(address): return False
 580         if address in self.imported_keys.keys(): return False
 581         acct, s = self.get_address_index(address)
 582         if s is None: return False
 583         return s[0] == 1
 584
 585     def get_master_public_key(self):
 586         if self.seed_version == 4:
 587             return self.storage.get("master_public_key")
 588         else:
 589             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 590             return repr((c, K))
 591
 592     def get_master_private_key(self, account, password):
 593         master_k = pw_decode( self.master_private_keys[account], password)
 594         master_c, master_K, master_Kc = self.master_public_keys[account]
 595         try:
 596             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 597             assert K.encode('hex') == master_K
 598         except:
 599             raise BaseException("Invalid password")
 600         return master_k
 601
 602
 603     def get_address_index(self, address):
 604         if address in self.imported_keys.keys():
 605             return -1, None
 606
 607         for account in self.accounts.keys():
 608             for for_change in [0,1]:
 609                 addresses = self.accounts[account].get_addresses(for_change)
 610                 for addr in addresses:
 611                     if address == addr:
 612                         return account, (for_change, addresses.index(addr))
 613
 614         for k,v in self.next_addresses.items():
 615             if v == address:
 616                 return k, (0,0)
 617
 618         raise BaseException("Address not found", address)
 619
 620
 621     def get_roots(self, account):
 622         roots = []
 623         for a in account.split('&'):
 624             s = a.strip()
 625             m = re.match("(m/\d+'/)(\d+)", s)
 626             roots.append( m.group(1) )
 627         return roots
 628
 629     def is_seeded(self, account):
 630         if type(account) is int:
 631             return self.seed is not None
 632
 633         for root in self.get_roots(account):
 634             if root not in self.master_private_keys.keys():
 635                 return False
 636         return True
 637
 638     def rebase_sequence(self, account, sequence):
 639         c, i = sequence
 640         dd = []
 641         for a in account.split('&'):
 642             s = a.strip()
 643             m = re.match("(m/\d+'/)(\d+)", s)
 644             root = m.group(1)
 645             num = int(m.group(2))
 646             dd.append( (root, [num,c,i] ) )
 647         return dd
 648
 649
 650     def get_keyID(self, account, sequence):
 651         if account == 0:
 652             return 'old'
 653
 654         rs = self.rebase_sequence(account, sequence)
 655         dd = []
 656         for root, public_sequence in rs:
 657             c, K, _ = self.master_public_keys[root]
 658             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 659             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 660         return '&'.join(dd)
 661
 662
 663
 664     def get_seed(self, password):
 665         s = pw_decode(self.seed, password)
 666         if self.seed_version == 4:
 667             seed = s
 668         else:
 669             seed = mnemonic_hash(s)
 670         #todo:  #self.sequences[0].check_seed(seed)
 671         return seed
 672
 673
 674     def get_mnemonic(self, password):
 675         import mnemonic
 676         s = pw_decode(self.seed, password)
 677         if self.seed_version == 4:
 678             return ' '.join(mnemonic.mn_encode(s))
 679         else:
 680             return s
 681
 682
 683
 684     def get_private_key(self, address, password):
 685         out = []
 686         if address in self.imported_keys.keys():
 687             out.append( pw_decode( self.imported_keys[address], password ) )
 688         else:
 689             account, sequence = self.get_address_index(address)
 690             if account == 0:
 691                 seed = self.get_seed(password)
 692                 pk = self.accounts[account].get_private_key(seed, sequence)
 693                 out.append(pk)
 694                 return out
 695
 696             # assert address == self.accounts[account].get_address(*sequence)
 697             rs = self.rebase_sequence( account, sequence)
 698             for root, public_sequence in rs:
 699
 700                 if root not in self.master_private_keys.keys(): continue
 701                 master_k = self.get_master_private_key(root, password)
 702                 master_c, _, _ = self.master_public_keys[root]
 703                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 704                 out.append(pk)
 705
 706         return out
 707
 708
 709     def add_keypairs_from_wallet(self, tx, keypairs, password):
 710         for txin in tx.inputs:
 711             address = txin['address']
 712             private_keys = self.get_private_key(address, password)
 713             for sec in private_keys:
 714                 pubkey = public_key_from_private_key(sec)
 715                 keypairs[ pubkey ] = sec
 716
 717
 718     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 719         for txin in tx.inputs:
 720             keyid = txin.get('KeyID')
 721             if keyid:
 722                 roots = []
 723                 for s in keyid.split('&'):
 724                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 725                     if not m: continue
 726                     c = m.group(1)
 727                     K = m.group(2)
 728                     sequence = m.group(3)
 729                     root = self.find_root_by_master_key(c,K)
 730                     if not root: continue
 731                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 732                     root = root + '%d'%sequence[0]
 733                     sequence = sequence[1:]
 734                     roots.append((root,sequence))
 735
 736                 account_id = " & ".join( map(lambda x:x[0], roots) )
 737                 account = self.accounts.get(account_id)
 738                 if not account: continue
 739                 addr = account.get_address(*sequence)
 740                 txin['address'] = addr # fixme: side effect
 741                 pk = self.get_private_key(addr, password)
 742                 for sec in pk:
 743                     pubkey = public_key_from_private_key(sec)
 744                     keypairs[pubkey] = sec
 745
 746
 747
 748     def signrawtransaction(self, tx, input_info, private_keys, password):
 749
 750         # check that the password is correct
 751         seed = self.get_seed(password)
 752
 753         # add input info
 754         tx.add_input_info(input_info)
 755
 756         # add redeem script for coins that are in the wallet
 757         # FIXME: add redeemPubkey too!
 758         unspent_coins = self.get_unspent_coins()
 759         for txin in tx.inputs:
 760             for item in unspent_coins:
 761                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 762                     print_error( "tx input is in unspent coins" )
 763                     txin['scriptPubKey'] = item['scriptPubKey']
 764                     account, sequence = self.get_address_index(item['address'])
 765                     if account != -1:
 766                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 767                         print_error("added redeemScript", txin['redeemScript'])
 768                     break
 769
 770
 771         # build a list of public/private keys
 772         keypairs = {}
 773
 774         # add private keys from parameter
 775         for sec in private_keys:
 776             pubkey = public_key_from_private_key(sec)
 777             keypairs[ pubkey ] = sec
 778
 779         # add private_keys from KeyID
 780         self.add_keypairs_from_KeyID(tx, keypairs, password)
 781
 782         # add private keys from wallet
 783         self.add_keypairs_from_wallet(tx, keypairs, password)
 784         self.sign_transaction(tx, keypairs)
 785
 786
 787     def sign_message(self, address, message, password):
 788         keys = self.get_private_key(address, password)
 789         assert len(keys) == 1
 790         sec = keys[0]
 791         key = regenerate_key(sec)
 792         compressed = is_compressed(sec)
 793         return key.sign_message(message, compressed, address)
 794
 795
 796     def change_gap_limit(self, value):
 797         if value >= self.gap_limit:
 798             self.gap_limit = value
 799             self.storage.put('gap_limit', self.gap_limit, True)
 800             #self.interface.poke('synchronizer')
 801             return True
 802
 803         elif value >= self.min_acceptable_gap():
 804             for key, account in self.accounts.items():
 805                 addresses = account[0]
 806                 k = self.num_unused_trailing_addresses(addresses)
 807                 n = len(addresses) - k + value
 808                 addresses = addresses[0:n]
 809                 self.accounts[key][0] = addresses
 810
 811             self.gap_limit = value
 812             self.storage.put('gap_limit', self.gap_limit, True)
 813             self.save_accounts()
 814             return True
 815         else:
 816             return False
 817
 818     def num_unused_trailing_addresses(self, addresses):
 819         k = 0
 820         for a in addresses[::-1]:
 821             if self.history.get(a):break
 822             k = k + 1
 823         return k
 824
 825     def min_acceptable_gap(self):
 826         # fixme: this assumes wallet is synchronized
 827         n = 0
 828         nmax = 0
 829
 830         for account in self.accounts.values():
 831             addresses = account.get_addresses(0)
 832             k = self.num_unused_trailing_addresses(addresses)
 833             for a in addresses[0:-k]:
 834                 if self.history.get(a):
 835                     n = 0
 836                 else:
 837                     n += 1
 838                     if n > nmax: nmax = n
 839         return nmax + 1
 840
 841
 842     def address_is_old(self, address):
 843         age = -1
 844         h = self.history.get(address, [])
 845         if h == ['*']:
 846             return True
 847         for tx_hash, tx_height in h:
 848             if tx_height == 0:
 849                 tx_age = 0
 850             else:
 851                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 852             if tx_age > age:
 853                 age = tx_age
 854         return age > 2
 855
 856
 857     def synchronize_sequence(self, account, for_change):
 858         limit = self.gap_limit_for_change if for_change else self.gap_limit
 859         new_addresses = []
 860         while True:
 861             addresses = account.get_addresses(for_change)
 862             if len(addresses) < limit:
 863                 address = account.create_new_address(for_change)
 864                 self.history[address] = []
 865                 new_addresses.append( address )
 866                 continue
 867
 868             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 869                 break
 870             else:
 871                 address = account.create_new_address(for_change)
 872                 self.history[address] = []
 873                 new_addresses.append( address )
 874
 875         return new_addresses
 876
 877
 878
 879     def create_pending_accounts(self):
 880         for account_type in ['1','2of2','2of3']:
 881             if not self.has_master_public_keys(account_type):
 882                 continue
 883             k, a = self.new_account_address(account_type)
 884             if self.address_is_old(a):
 885                 print_error( "creating account", a )
 886                 self.create_account(account_type)
 887                 self.next_addresses.pop(k)
 888
 889
 890     def synchronize_account(self, account):
 891         new = []
 892         new += self.synchronize_sequence(account, 0)
 893         new += self.synchronize_sequence(account, 1)
 894         return new
 895
 896
 897     def synchronize(self):
 898         if self.master_public_keys:
 899             self.create_pending_accounts()
 900         new = []
 901         for account in self.accounts.values():
 902             new += self.synchronize_account(account)
 903         if new:
 904             self.save_accounts()
 905             self.storage.put('addr_history', self.history, True)
 906         return new
 907
 908
 909     def is_found(self):
 910         return self.history.values() != [[]] * len(self.history)
 911
 912
 913     def add_contact(self, address, label=None):
 914         self.addressbook.append(address)
 915         self.storage.put('contacts', self.addressbook, True)
 916         if label:
 917             self.set_label(address, label)
 918
 919
 920     def delete_contact(self, addr):
 921         if addr in self.addressbook:
 922             self.addressbook.remove(addr)
 923             self.storage.put('addressbook', self.addressbook, True)
 924
 925
 926     def fill_addressbook(self):
 927         for tx_hash, tx in self.transactions.items():
 928             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 929             if is_send:
 930                 for addr, v in tx.outputs:
 931                     if not self.is_mine(addr) and addr not in self.addressbook:
 932                         self.addressbook.append(addr)
 933         # redo labels
 934         # self.update_tx_labels()
 935
 936     def get_num_tx(self, address):
 937         n = 0
 938         for tx in self.transactions.values():
 939             if address in map(lambda x:x[0], tx.outputs): n += 1
 940         return n
 941
 942
 943     def get_address_flags(self, addr):
 944         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 945         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 946         return flags
 947
 948
 949     def get_tx_value(self, tx, account=None):
 950         domain = self.get_account_addresses(account)
 951         return tx.get_value(domain, self.prevout_values)
 952
 953
 954     def update_tx_outputs(self, tx_hash):
 955         tx = self.transactions.get(tx_hash)
 956
 957         for i, (addr, value) in enumerate(tx.outputs):
 958             key = tx_hash+ ':%d'%i
 959             self.prevout_values[key] = value
 960
 961         for item in tx.inputs:
 962             if self.is_mine(item.get('address')):
 963                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 964                 self.spent_outputs.append(key)
 965
 966
 967     def get_addr_balance(self, address):
 968         assert self.is_mine(address)
 969         h = self.history.get(address,[])
 970         if h == ['*']: return 0,0
 971         c = u = 0
 972         received_coins = []   # list of coins received at address
 973
 974         for tx_hash, tx_height in h:
 975             tx = self.transactions.get(tx_hash)
 976             if not tx: continue
 977
 978             for i, (addr, value) in enumerate(tx.outputs):
 979                 if addr == address:
 980                     key = tx_hash + ':%d'%i
 981                     received_coins.append(key)
 982
 983         for tx_hash, tx_height in h:
 984             tx = self.transactions.get(tx_hash)
 985             if not tx: continue
 986             v = 0
 987
 988             for item in tx.inputs:
 989                 addr = item.get('address')
 990                 if addr == address:
 991                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 992                     value = self.prevout_values.get( key )
 993                     if key in received_coins:
 994                         v -= value
 995
 996             for i, (addr, value) in enumerate(tx.outputs):
 997                 key = tx_hash + ':%d'%i
 998                 if addr == address:
 999                     v += value
1000
1001             if tx_height:
1002                 c += v
1003             else:
1004                 u += v
1005         return c, u
1006
1007
1008     def get_account_name(self, k):
1009         if k == 0:
1010             if self.seed_version == 4:
1011                 name = 'Main account'
1012             else:
1013                 name = 'Old account'
1014         else:
1015             default = "Unnamed account"
1016             m = re.match("m/0'/(\d+)", k)
1017             if m:
1018                 num = m.group(1)
1019                 if num == '0':
1020                     default = "Main account"
1021                 else:
1022                     default = "Account %s"%num
1023
1024             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1025             if m:
1026                 num = m.group(1)
1027                 default = "2of2 account %s"%num
1028             name = self.labels.get(k, default)
1029
1030         return name
1031
1032     def get_account_names(self):
1033         accounts = {}
1034         for k, account in self.accounts.items():
1035             accounts[k] = self.get_account_name(k)
1036         if self.imported_keys:
1037             accounts[-1] = 'Imported keys'
1038         return accounts
1039
1040     def get_account_addresses(self, a, include_change=True):
1041         if a is None:
1042             o = self.addresses(True)
1043         elif a == -1:
1044             o = self.imported_keys.keys()
1045         else:
1046             ac = self.accounts[a]
1047             o = ac.get_addresses(0)
1048             if include_change: o += ac.get_addresses(1)
1049         return o
1050
1051     def get_imported_balance(self):
1052         return self.get_balance(self.imported_keys.keys())
1053
1054     def get_account_balance(self, account):
1055         return self.get_balance(self.get_account_addresses(account))
1056
1057     def get_frozen_balance(self):
1058         return self.get_balance(self.frozen_addresses)
1059
1060     def get_balance(self, domain=None):
1061         if domain is None: domain = self.addresses(True)
1062         cc = uu = 0
1063         for addr in domain:
1064             c, u = self.get_addr_balance(addr)
1065             cc += c
1066             uu += u
1067         return cc, uu
1068
1069
1070     def get_unspent_coins(self, domain=None):
1071         coins = []
1072         if domain is None: domain = self.addresses(True)
1073         for addr in domain:
1074             h = self.history.get(addr, [])
1075             if h == ['*']: continue
1076             for tx_hash, tx_height in h:
1077                 tx = self.transactions.get(tx_hash)
1078                 if tx is None: raise BaseException("Wallet not synchronized")
1079                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1080                 for output in tx.d.get('outputs'):
1081                     if output.get('address') != addr: continue
1082                     key = tx_hash + ":%d" % output.get('prevout_n')
1083                     if key in self.spent_outputs: continue
1084                     output['prevout_hash'] = tx_hash
1085                     output['height'] = tx_height
1086                     output['coinbase'] = is_coinbase
1087                     coins.append((tx_height, output))
1088
1089         # sort by age
1090         if coins:
1091             coins = sorted(coins)
1092             if coins[-1][0] != 0:
1093                 while coins[0][0] == 0:
1094                     coins = coins[1:] + [ coins[0] ]
1095         return [x[1] for x in coins]
1096
1097
1098
1099     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1100         domain = self.get_account_addresses(account) if account else None
1101         return self.choose_tx_inputs( amount, fixed_fee, domain )
1102
1103
1104     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1105         """ todo: minimize tx size """
1106         total = 0
1107         fee = self.fee if fixed_fee is None else fixed_fee
1108         if domain is None:
1109             domain = self.addresses(True)
1110
1111         for i in self.frozen_addresses:
1112             if i in domain: domain.remove(i)
1113
1114         prioritized = []
1115         for i in self.prioritized_addresses:
1116             if i in domain:
1117                 domain.remove(i)
1118                 prioritized.append(i)
1119
1120         coins = self.get_unspent_coins(domain)
1121         prioritized_coins = self.get_unspent_coins(prioritized)
1122
1123         inputs = []
1124         coins = prioritized_coins + coins
1125
1126         for item in coins:
1127             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1128                 continue
1129             addr = item.get('address')
1130             v = item.get('value')
1131             total += v
1132             inputs.append(item)
1133             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1134             if total >= amount + fee: break
1135         else:
1136             inputs = []
1137
1138         return inputs, total, fee
1139
1140
1141     def set_fee(self, fee):
1142         if self.fee != fee:
1143             self.fee = fee
1144             self.storage.put('fee_per_kb', self.fee, True)
1145
1146     def estimated_fee(self, inputs):
1147         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1148         fee = self.fee * int(round(estimated_size/1024.))
1149         if fee == 0: fee = self.fee
1150         return fee
1151
1152
1153     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1154         "add change to a transaction"
1155         change_amount = total - ( amount + fee )
1156         if change_amount != 0:
1157             if not change_addr:
1158
1159                 # send change to one of the accounts involved in the tx
1160                 address = inputs[0].get('address')
1161                 account, _ = self.get_address_index(address)
1162
1163                 if not self.use_change or account == -1:
1164                     change_addr = inputs[-1]['address']
1165                 else:
1166                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1167
1168             # Insert the change output at a random position in the outputs
1169             posn = random.randint(0, len(outputs))
1170             outputs[posn:posn] = [( change_addr,  change_amount)]
1171         return outputs
1172
1173
1174     def get_history(self, address):
1175         with self.lock:
1176             return self.history.get(address)
1177
1178
1179     def get_status(self, h):
1180         if not h: return None
1181         if h == ['*']: return '*'
1182         status = ''
1183         for tx_hash, height in h:
1184             status += tx_hash + ':%d:' % height
1185         return hashlib.sha256( status ).digest().encode('hex')
1186
1187
1188     def receive_tx_callback(self, tx_hash, tx, tx_height):
1189
1190         with self.transaction_lock:
1191             self.add_extra_addresses(tx)
1192             if not self.check_new_tx(tx_hash, tx):
1193                 # may happen due to pruning
1194                 print_error("received transaction that is no longer referenced in history", tx_hash)
1195                 return
1196             self.transactions[tx_hash] = tx
1197             self.network.interface.pending_transactions_for_notifications.append(tx)
1198             self.save_transactions()
1199             if self.verifier and tx_height>0:
1200                 self.verifier.add(tx_hash, tx_height)
1201             self.update_tx_outputs(tx_hash)
1202
1203
1204     def save_transactions(self):
1205         tx = {}
1206         for k,v in self.transactions.items():
1207             tx[k] = str(v)
1208         self.storage.put('transactions', tx, True)
1209
1210     def receive_history_callback(self, addr, hist):
1211
1212         if not self.check_new_history(addr, hist):
1213             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1214
1215         with self.lock:
1216             self.history[addr] = hist
1217             self.storage.put('addr_history', self.history, True)
1218
1219         if hist != ['*']:
1220             for tx_hash, tx_height in hist:
1221                 if tx_height>0:
1222                     # add it in case it was previously unconfirmed
1223                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1224
1225
1226     def get_tx_history(self, account=None):
1227         with self.transaction_lock:
1228             history = self.transactions.items()
1229             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1230             result = []
1231
1232             balance = 0
1233             for tx_hash, tx in history:
1234                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1235                 if v is not None: balance += v
1236
1237             c, u = self.get_account_balance(account)
1238
1239             if balance != c+u:
1240                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1241
1242             balance = c + u - balance
1243             for tx_hash, tx in history:
1244                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1245                 if not is_relevant:
1246                     continue
1247                 if value is not None:
1248                     balance += value
1249
1250                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1251                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1252
1253         return result
1254
1255
1256     def get_label(self, tx_hash):
1257         label = self.labels.get(tx_hash)
1258         is_default = (label == '') or (label is None)
1259         if is_default: label = self.get_default_label(tx_hash)
1260         return label, is_default
1261
1262
1263     def get_default_label(self, tx_hash):
1264         tx = self.transactions.get(tx_hash)
1265         default_label = ''
1266         if tx:
1267             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1268             if is_mine:
1269                 for o in tx.outputs:
1270                     o_addr, _ = o
1271                     if not self.is_mine(o_addr):
1272                         try:
1273                             default_label = self.labels[o_addr]
1274                         except KeyError:
1275                             default_label = o_addr
1276                         break
1277                 else:
1278                     default_label = '(internal)'
1279             else:
1280                 for o in tx.outputs:
1281                     o_addr, _ = o
1282                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1283                         break
1284                 else:
1285                     for o in tx.outputs:
1286                         o_addr, _ = o
1287                         if self.is_mine(o_addr):
1288                             break
1289                     else:
1290                         o_addr = None
1291
1292                 if o_addr:
1293                     dest_label = self.labels.get(o_addr)
1294                     try:
1295                         default_label = self.labels[o_addr]
1296                     except KeyError:
1297                         default_label = o_addr
1298
1299         return default_label
1300
1301
1302     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1303         for address, x in outputs:
1304             assert is_valid(address)
1305         amount = sum( map(lambda x:x[1], outputs) )
1306         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1307         if not inputs:
1308             raise ValueError("Not enough funds")
1309         self.add_input_info(inputs)
1310         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1311         return Transaction.from_io(inputs, outputs)
1312
1313
1314     def mktx_from_account(self, outputs, password, fee=None, account=None):
1315         domain = self.get_account_addresses(account) if account else None
1316         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1317
1318
1319     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1320         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1321         keypairs = {}
1322         self.add_keypairs_from_wallet(tx, keypairs, password)
1323         if keypairs:
1324             self.sign_transaction(tx, keypairs)
1325         return tx
1326
1327
1328     def add_input_info(self, inputs):
1329         for txin in inputs:
1330             address = txin['address']
1331             if address in self.imported_keys.keys():
1332                 continue
1333             account, sequence = self.get_address_index(address)
1334             txin['KeyID'] = self.get_keyID(account, sequence)
1335             redeemScript = self.accounts[account].redeem_script(sequence)
1336             if redeemScript:
1337                 txin['redeemScript'] = redeemScript
1338             else:
1339                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1340
1341
1342     def sign_transaction(self, tx, keypairs):
1343         tx.sign(keypairs)
1344         run_hook('sign_transaction', tx)
1345
1346
1347     def sendtx(self, tx):
1348         # synchronous
1349         h = self.send_tx(tx)
1350         self.tx_event.wait()
1351         return self.receive_tx(h)
1352
1353     def send_tx(self, tx):
1354         # asynchronous
1355         self.tx_event.clear()
1356         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1357         return tx.hash()
1358
1359     def on_broadcast(self, i, r):
1360         self.tx_result = r.get('result')
1361         self.tx_event.set()
1362
1363     def receive_tx(self,tx_hash):
1364         out = self.tx_result
1365         if out != tx_hash:
1366             return False, "error: " + out
1367         return True, out
1368
1369
1370
1371     def update_password(self, old_password, new_password):
1372         if new_password == '': new_password = None
1373         # this will throw an exception if unicode cannot be converted
1374         decoded = pw_decode(self.seed, old_password)
1375         self.seed = pw_encode( decoded, new_password)
1376         self.storage.put('seed', self.seed, True)
1377         self.use_encryption = (new_password != None)
1378         self.storage.put('use_encryption', self.use_encryption,True)
1379         for k in self.imported_keys.keys():
1380             a = self.imported_keys[k]
1381             b = pw_decode(a, old_password)
1382             c = pw_encode(b, new_password)
1383             self.imported_keys[k] = c
1384         self.storage.put('imported_keys', self.imported_keys, True)
1385
1386         for k, v in self.master_private_keys.items():
1387             b = pw_decode(v, old_password)
1388             c = pw_encode(b, new_password)
1389             self.master_private_keys[k] = c
1390         self.storage.put('master_private_keys', self.master_private_keys, True)
1391
1392
1393     def freeze(self,addr):
1394         if self.is_mine(addr) and addr not in self.frozen_addresses:
1395             self.unprioritize(addr)
1396             self.frozen_addresses.append(addr)
1397             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1398             return True
1399         else:
1400             return False
1401
1402     def unfreeze(self,addr):
1403         if self.is_mine(addr) and addr in self.frozen_addresses:
1404             self.frozen_addresses.remove(addr)
1405             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1406             return True
1407         else:
1408             return False
1409
1410     def prioritize(self,addr):
1411         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1412             self.unfreeze(addr)
1413             self.prioritized_addresses.append(addr)
1414             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1415             return True
1416         else:
1417             return False
1418
1419     def unprioritize(self,addr):
1420         if self.is_mine(addr) and addr in self.prioritized_addresses:
1421             self.prioritized_addresses.remove(addr)
1422             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1423             return True
1424         else:
1425             return False
1426
1427
1428     def set_verifier(self, verifier):
1429         self.verifier = verifier
1430
1431         # review transactions that are in the history
1432         for addr, hist in self.history.items():
1433             if hist == ['*']: continue
1434             for tx_hash, tx_height in hist:
1435                 if tx_height>0:
1436                     # add it in case it was previously unconfirmed
1437                     self.verifier.add(tx_hash, tx_height)
1438
1439
1440         # if we are on a pruning server, remove unverified transactions
1441         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1442         for tx_hash in self.transactions.keys():
1443             if tx_hash not in vr:
1444                 self.transactions.pop(tx_hash)
1445
1446
1447
1448     def check_new_history(self, addr, hist):
1449
1450         # check that all tx in hist are relevant
1451         if hist != ['*']:
1452             for tx_hash, height in hist:
1453                 tx = self.transactions.get(tx_hash)
1454                 if not tx: continue
1455                 if not tx.has_address(addr):
1456                     return False
1457
1458         # check that we are not "orphaning" a transaction
1459         old_hist = self.history.get(addr,[])
1460         if old_hist == ['*']: return True
1461
1462         for tx_hash, height in old_hist:
1463             if tx_hash in map(lambda x:x[0], hist): continue
1464             found = False
1465             for _addr, _hist in self.history.items():
1466                 if _addr == addr: continue
1467                 if _hist == ['*']: continue
1468                 _tx_hist = map(lambda x:x[0], _hist)
1469                 if tx_hash in _tx_hist:
1470                     found = True
1471                     break
1472
1473             if not found:
1474                 tx = self.transactions.get(tx_hash)
1475                 # tx might not be there
1476                 if not tx: continue
1477
1478                 # already verified?
1479                 if self.verifier.get_height(tx_hash):
1480                     continue
1481                 # unconfirmed tx
1482                 print_error("new history is orphaning transaction:", tx_hash)
1483                 # check that all outputs are not mine, request histories
1484                 ext_requests = []
1485                 for _addr, _v in tx.outputs:
1486                     # assert not self.is_mine(_addr)
1487                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1488
1489                 ext_h = self.network.synchronous_get(ext_requests)
1490                 print_error("sync:", ext_requests, ext_h)
1491                 height = None
1492                 for h in ext_h:
1493                     if h == ['*']: continue
1494                     for item in h:
1495                         if item.get('tx_hash') == tx_hash:
1496                             height = item.get('height')
1497                 if height:
1498                     print_error("found height for", tx_hash, height)
1499                     self.verifier.add(tx_hash, height)
1500                 else:
1501                     print_error("removing orphaned tx from history", tx_hash)
1502                     self.transactions.pop(tx_hash)
1503
1504         return True
1505
1506
1507
1508     def check_new_tx(self, tx_hash, tx):
1509         # 1 check that tx is referenced in addr_history.
1510         addresses = []
1511         for addr, hist in self.history.items():
1512             if hist == ['*']:continue
1513             for txh, height in hist:
1514                 if txh == tx_hash:
1515                     addresses.append(addr)
1516
1517         if not addresses:
1518             return False
1519
1520         # 2 check that referencing addresses are in the tx
1521         for addr in addresses:
1522             if not tx.has_address(addr):
1523                 return False
1524
1525         return True
1526
1527
1528     def start_threads(self, network):
1529         from verifier import TxVerifier
1530         self.network = network
1531         self.verifier = TxVerifier(self.network, self.storage)
1532         self.verifier.start()
1533         self.set_verifier(self.verifier)
1534         self.synchronizer = WalletSynchronizer(self, network)
1535         self.synchronizer.start()
1536
1537     def stop_threads(self):
1538         self.verifier.stop()
1539         self.synchronizer.stop()
1540
1541
1542
1543     def restore(self, callback):
1544         from i18n import _
1545         def wait_for_wallet():
1546             self.set_up_to_date(False)
1547             while not self.is_up_to_date():
1548                 msg = "%s\n%s %d\n%s %.1f"%(
1549                     _("Please wait..."),
1550                     _("Addresses generated:"),
1551                     len(self.addresses(True)),_("Kilobytes received:"),
1552                     self.network.interface.bytes_received/1024.)
1553
1554                 apply(callback, (msg,))
1555                 time.sleep(0.1)
1556
1557         def wait_for_network():
1558             while not self.network.interface.is_connected:
1559                 msg = "%s \n" % (_("Connecting..."))
1560                 apply(callback, (msg,))
1561                 time.sleep(0.1)
1562
1563         # wait until we are connected, because the user might have selected another server
1564         wait_for_network()
1565
1566
1567         self.create_accounts()
1568         wait_for_wallet()
1569
1570
1571
1572
1573 class WalletSynchronizer(threading.Thread):
1574
1575
1576     def __init__(self, wallet, network):
1577         threading.Thread.__init__(self)
1578         self.daemon = True
1579         self.wallet = wallet
1580         self.network = network
1581         self.was_updated = True
1582         self.running = False
1583         self.lock = threading.Lock()
1584         self.queue = Queue.Queue()
1585
1586     def stop(self):
1587         with self.lock: self.running = False
1588
1589     def is_running(self):
1590         with self.lock: return self.running
1591
1592
1593     def subscribe_to_addresses(self, addresses):
1594         messages = []
1595         for addr in addresses:
1596             messages.append(('blockchain.address.subscribe', [addr]))
1597         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1598
1599
1600     def run(self):
1601         with self.lock:
1602             self.running = True
1603
1604         while self.is_running():
1605
1606             if not self.network.is_connected():
1607                 print_error("synchronizer: waiting for interface")
1608                 self.network.wait_until_connected()
1609
1610             self.run_interface(self.network.interface)
1611
1612
1613     def run_interface(self, interface):
1614
1615         print_error("synchronizer: connected to", interface.server)
1616
1617         requested_tx = []
1618         missing_tx = []
1619         requested_histories = {}
1620
1621         # request any missing transactions
1622         for history in self.wallet.history.values():
1623             if history == ['*']: continue
1624             for tx_hash, tx_height in history:
1625                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1626                     missing_tx.append( (tx_hash, tx_height) )
1627
1628         if missing_tx:
1629             print_error("missing tx", missing_tx)
1630
1631         # subscriptions
1632         self.subscribe_to_addresses(self.wallet.addresses(True))
1633
1634         while self.is_running():
1635             # 1. create new addresses
1636             new_addresses = self.wallet.synchronize()
1637
1638             # request missing addresses
1639             if new_addresses:
1640                 self.subscribe_to_addresses(new_addresses)
1641
1642             # request missing transactions
1643             for tx_hash, tx_height in missing_tx:
1644                 if (tx_hash, tx_height) not in requested_tx:
1645                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1646                     requested_tx.append( (tx_hash, tx_height) )
1647             missing_tx = []
1648
1649             # detect if situation has changed
1650             if interface.is_up_to_date() and self.queue.empty():
1651                 if not self.wallet.is_up_to_date():
1652                     self.wallet.set_up_to_date(True)
1653                     self.was_updated = True
1654             else:
1655                 if self.wallet.is_up_to_date():
1656                     self.wallet.set_up_to_date(False)
1657                     self.was_updated = True
1658
1659             if self.was_updated:
1660                 self.wallet.network.trigger_callback('updated')
1661                 self.was_updated = False
1662
1663             # 2. get a response
1664             try:
1665                 r = self.queue.get(block=True, timeout=1)
1666             except Queue.Empty:
1667                 continue
1668
1669             if interface != self.network.interface:
1670                 break
1671
1672             if not r:
1673                 continue
1674
1675             # 3. handle response
1676             method = r['method']
1677             params = r['params']
1678             result = r.get('result')
1679             error = r.get('error')
1680             if error:
1681                 print "error", r
1682                 continue
1683
1684             if method == 'blockchain.address.subscribe':
1685                 addr = params[0]
1686                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1687                     if requested_histories.get(addr) is None:
1688                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1689                         requested_histories[addr] = result
1690
1691             elif method == 'blockchain.address.get_history':
1692                 addr = params[0]
1693                 print_error("receiving history", addr, result)
1694                 if result == ['*']:
1695                     assert requested_histories.pop(addr) == '*'
1696                     self.wallet.receive_history_callback(addr, result)
1697                 else:
1698                     hist = []
1699                     # check that txids are unique
1700                     txids = []
1701                     for item in result:
1702                         tx_hash = item['tx_hash']
1703                         if tx_hash not in txids:
1704                             txids.append(tx_hash)
1705                             hist.append( (tx_hash, item['height']) )
1706
1707                     if len(hist) != len(result):
1708                         raise BaseException("error: server sent history with non-unique txid", result)
1709
1710                     # check that the status corresponds to what was announced
1711                     rs = requested_histories.pop(addr)
1712                     if self.wallet.get_status(hist) != rs:
1713                         raise BaseException("error: status mismatch: %s"%addr)
1714
1715                     # store received history
1716                     self.wallet.receive_history_callback(addr, hist)
1717
1718                     # request transactions that we don't have
1719                     for tx_hash, tx_height in hist:
1720                         if self.wallet.transactions.get(tx_hash) is None:
1721                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1722                                 missing_tx.append( (tx_hash, tx_height) )
1723
1724             elif method == 'blockchain.transaction.get':
1725                 tx_hash = params[0]
1726                 tx_height = params[1]
1727                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1728                 tx = Transaction(result)
1729                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1730                 self.was_updated = True
1731                 requested_tx.remove( (tx_hash, tx_height) )
1732                 print_error("received tx:", tx_hash, len(tx.raw))
1733
1734             else:
1735                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1736
1737             if self.was_updated and not requested_tx:
1738                 self.wallet.network.trigger_callback('updated')
1739                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1740
1741                 self.was_updated = False