lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32 import math
  33
  34 from util import print_msg, print_error, format_satoshis
  35 from bitcoin import *
  36 from account import *
  37 from transaction import Transaction
  38 from plugins import run_hook
  39
  40 COINBASE_MATURITY = 100
  41 DUST_THRESHOLD = 5430
  42
  43 # AES encryption
  44 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  45 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  46
  47 def pw_encode(s, password):
  48     if password:
  49         secret = Hash(password)
  50         return EncodeAES(secret, s)
  51     else:
  52         return s
  53
  54 def pw_decode(s, password):
  55     if password is not None:
  56         secret = Hash(password)
  57         try:
  58             d = DecodeAES(secret, s)
  59         except Exception:
  60             raise Exception('Invalid password')
  61         return d
  62     else:
  63         return s
  64
  65
  66
  67
  68
  69 from version import *
  70
  71
  72 class WalletStorage:
  73
  74     def __init__(self, config):
  75         self.lock = threading.Lock()
  76         self.config = config
  77         self.data = {}
  78         self.file_exists = False
  79         self.path = self.init_path(config)
  80         print_error( "wallet path", self.path )
  81         if self.path:
  82             self.read(self.path)
  83
  84
  85     def init_path(self, config):
  86         """Set the path of the wallet."""
  87
  88         # command line -w option
  89         path = config.get('wallet_path')
  90         if path:
  91             return path
  92
  93         # path in config file
  94         path = config.get('default_wallet_path')
  95         if path:
  96             return path
  97
  98         # default path
  99         dirpath = os.path.join(config.path, "wallets")
 100         if not os.path.exists(dirpath):
 101             os.mkdir(dirpath)
 102
 103         new_path = os.path.join(config.path, "wallets", "default_wallet")
 104
 105         # default path in pre 1.9 versions
 106         old_path = os.path.join(config.path, "electrum.dat")
 107         if os.path.exists(old_path) and not os.path.exists(new_path):
 108             os.rename(old_path, new_path)
 109
 110         return new_path
 111
 112
 113     def read(self, path):
 114         """Read the contents of the wallet file."""
 115         try:
 116             with open(self.path, "r") as f:
 117                 data = f.read()
 118         except IOError:
 119             return
 120         try:
 121             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 122         except Exception:
 123             raise IOError("Cannot read wallet file.")
 124
 125         self.data = d
 126         self.file_exists = True
 127
 128
 129     def get(self, key, default=None):
 130         v = self.data.get(key)
 131         if v is None:
 132             v = default
 133         return v
 134
 135     def put(self, key, value, save = True):
 136
 137         with self.lock:
 138             if value is not None:
 139                 self.data[key] = value
 140             else:
 141                 self.data.pop(key)
 142             if save:
 143                 self.write()
 144
 145     def write(self):
 146         s = repr(self.data)
 147         f = open(self.path,"w")
 148         f.write( s )
 149         f.close()
 150         if 'ANDROID_DATA' not in os.environ:
 151             import stat
 152             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 153
 154
 155
 156
 157
 158 class NewWallet:
 159
 160     def __init__(self, storage):
 161
 162         self.storage = storage
 163         self.electrum_version = ELECTRUM_VERSION
 164         self.gap_limit_for_change = 3 # constant
 165
 166         # saved fields
 167         self.seed_version          = storage.get('seed_version', NEW_SEED_VERSION)
 168
 169         self.gap_limit             = storage.get('gap_limit', 5)
 170         self.use_change            = storage.get('use_change',True)
 171         self.use_encryption        = storage.get('use_encryption', False)
 172         self.seed                  = storage.get('seed', '')               # encrypted
 173         self.labels                = storage.get('labels', {})
 174         self.frozen_addresses      = storage.get('frozen_addresses',[])
 175         self.addressbook           = storage.get('contacts', [])
 176
 177         self.imported_keys         = storage.get('imported_keys',{})
 178         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 179
 180         self.fee                   = int(storage.get('fee_per_kb',20000))
 181
 182         self.master_public_keys = storage.get('master_public_keys',{})
 183         self.master_private_keys = storage.get('master_private_keys', {})
 184
 185         self.next_addresses = storage.get('next_addresses',{})
 186
 187
 188         # This attribute is set when wallet.start_threads is called.
 189         self.synchronizer = None
 190
 191         self.load_accounts()
 192
 193         self.transactions = {}
 194         tx_list = self.storage.get('transactions',{})
 195         for k,v in tx_list.items():
 196             try:
 197                 tx = Transaction(v)
 198             except Exception:
 199                 print_msg("Warning: Cannot deserialize transactions. skipping")
 200                 continue
 201
 202             self.add_extra_addresses(tx)
 203             self.transactions[k] = tx
 204
 205         for h,tx in self.transactions.items():
 206             if not self.check_new_tx(h, tx):
 207                 print_error("removing unreferenced tx", h)
 208                 self.transactions.pop(h)
 209
 210
 211         # not saved
 212         self.prevout_values = {}     # my own transaction outputs
 213         self.spent_outputs = []
 214
 215         # spv
 216         self.verifier = None
 217
 218         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 219         # interface.is_up_to_date() returns true when all requests have been answered and processed
 220         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 221
 222         self.up_to_date = False
 223         self.lock = threading.Lock()
 224         self.transaction_lock = threading.Lock()
 225         self.tx_event = threading.Event()
 226
 227         for tx_hash, tx in self.transactions.items():
 228             self.update_tx_outputs(tx_hash)
 229
 230
 231     def add_extra_addresses(self, tx):
 232         h = tx.hash()
 233         # find the address corresponding to pay-to-pubkey inputs
 234         tx.add_extra_addresses(self.transactions)
 235         for o in tx.d.get('outputs'):
 236             if o.get('is_pubkey'):
 237                 for tx2 in self.transactions.values():
 238                     tx2.add_extra_addresses({h:tx})
 239
 240
 241
 242
 243     def set_up_to_date(self,b):
 244         with self.lock: self.up_to_date = b
 245
 246
 247     def is_up_to_date(self):
 248         with self.lock: return self.up_to_date
 249
 250
 251     def update(self):
 252         self.up_to_date = False
 253         while not self.is_up_to_date():
 254             time.sleep(0.1)
 255
 256
 257     def import_key(self, sec, password):
 258         # check password
 259         seed = self.get_seed(password)
 260         try:
 261             address = address_from_private_key(sec)
 262         except Exception:
 263             raise Exception('Invalid private key')
 264
 265         if self.is_mine(address):
 266             raise Exception('Address already in wallet')
 267
 268         # store the originally requested keypair into the imported keys table
 269         self.imported_keys[address] = pw_encode(sec, password )
 270         self.storage.put('imported_keys', self.imported_keys, True)
 271         if self.synchronizer:
 272             self.synchronizer.subscribe_to_addresses([address])
 273         return address
 274
 275     def delete_imported_key(self, addr):
 276         if addr in self.imported_keys:
 277             self.imported_keys.pop(addr)
 278             self.storage.put('imported_keys', self.imported_keys, True)
 279
 280
 281     def make_seed(self):
 282         import mnemonic, ecdsa
 283         entropy = ecdsa.util.randrange( pow(2,160) )
 284         nonce = 0
 285         while True:
 286             ss = "%040x"%(entropy+nonce)
 287             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 288             # we keep only 13 words, that's approximately 139 bits of entropy
 289             words = mnemonic.mn_encode(s)[0:13]
 290             seed = ' '.join(words)
 291             if is_seed(seed):
 292                 break  # this will remove 8 bits of entropy
 293             nonce += 1
 294
 295         return seed
 296
 297
 298     def init_seed(self, seed):
 299         import mnemonic, unicodedata
 300
 301         if self.seed:
 302             raise Exception("a seed exists")
 303
 304         self.seed_version = NEW_SEED_VERSION
 305
 306         if not seed:
 307             self.seed = self.make_seed()
 308             return
 309
 310         self.seed = unicodedata.normalize('NFC', unicode(seed.strip()))
 311
 312
 313
 314     def save_seed(self, password):
 315         if password:
 316             self.seed = pw_encode( self.seed, password)
 317             self.use_encryption = True
 318         self.storage.put('seed', self.seed, True)
 319         self.storage.put('seed_version', self.seed_version, True)
 320         self.storage.put('use_encryption', self.use_encryption,True)
 321         self.create_accounts(password)
 322
 323
 324     def create_watching_only_wallet(self, K0, c0):
 325         cK0 = "" #FIXME
 326         self.master_public_keys = {
 327             "m/0'/": (c0, K0, cK0),
 328             }
 329         self.storage.put('master_public_keys', self.master_public_keys, True)
 330         self.storage.put('seed_version', self.seed_version, True)
 331         self.create_account('1of1','Main account')
 332
 333
 334     def create_accounts(self, password):
 335         seed = pw_decode(self.seed, password)
 336         # create default account
 337         self.create_master_keys('1of1', password)
 338         self.create_account('1of1','Main account')
 339
 340
 341     def create_master_keys(self, account_type, password):
 342         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(password))
 343         if account_type == '1of1':
 344             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 345             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 346             self.master_private_keys["m/0'/"] = pw_encode(k0, password)
 347         elif account_type == '2of2':
 348             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 349             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 350             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 351             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 352             self.master_private_keys["m/1'/"] = pw_encode(k1, password)
 353             self.master_private_keys["m/2'/"] = pw_encode(k2, password)
 354         elif account_type == '2of3':
 355             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 356             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 357             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 358             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 359             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 360             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 361             self.master_private_keys["m/3'/"] = pw_encode(k3, password)
 362             self.master_private_keys["m/4'/"] = pw_encode(k4, password)
 363             self.master_private_keys["m/5'/"] = pw_encode(k5, password)
 364
 365         self.storage.put('master_public_keys', self.master_public_keys, True)
 366         self.storage.put('master_private_keys', self.master_private_keys, True)
 367
 368     def has_master_public_keys(self, account_type):
 369         if account_type == '1of1':
 370             return "m/0'/" in self.master_public_keys
 371         elif account_type == '2of2':
 372             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 373         elif account_type == '2of3':
 374             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 375
 376     def find_root_by_master_key(self, c, K):
 377         for key, v in self.master_public_keys.items():
 378             if key == "m/":continue
 379             cc, KK, _ = v
 380             if (c == cc) and (K == KK):
 381                 return key
 382
 383     def deseed_root(self, seed, password):
 384         # for safety, we ask the user to enter their seed
 385         assert seed == self.get_seed(password)
 386         self.seed = ''
 387         self.storage.put('seed', '', True)
 388
 389
 390     def deseed_branch(self, k):
 391         # check that parent has no seed
 392         # assert self.seed == ''
 393         self.master_private_keys.pop(k)
 394         self.storage.put('master_private_keys', self.master_private_keys, True)
 395
 396
 397     def is_watching_only(self):
 398         return (self.seed == '') and (self.master_private_keys == {})
 399
 400
 401
 402     def account_id(self, account_type, i):
 403         if account_type == '1of1':
 404             return "m/0'/%d"%i
 405         elif account_type == '2of2':
 406             return "m/1'/%d & m/2'/%d"%(i,i)
 407         elif account_type == '2of3':
 408             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 409         else:
 410             raise Exception('unknown account type')
 411
 412
 413     def num_accounts(self, account_type):
 414         keys = self.accounts.keys()
 415         i = 0
 416         while True:
 417             account_id = self.account_id(account_type, i)
 418             if account_id not in keys: break
 419             i += 1
 420         return i
 421
 422
 423     def new_account_address(self, account_type = '1of1'):
 424         i = self.num_accounts(account_type)
 425         k = self.account_id(account_type,i)
 426
 427         addr = self.next_addresses.get(k)
 428         if not addr:
 429             account_id, account = self.next_account(account_type)
 430             addr = account.first_address()
 431             self.next_addresses[k] = addr
 432             self.storage.put('next_addresses',self.next_addresses)
 433
 434         return k, addr
 435
 436
 437     def next_account(self, account_type = '1of1'):
 438
 439         i = self.num_accounts(account_type)
 440         account_id = self.account_id(account_type,i)
 441
 442         if account_type is '1of1':
 443             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 444             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 445             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 446
 447         elif account_type == '2of2':
 448             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 449             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 450             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 451             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 452             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 453
 454         elif account_type == '2of3':
 455             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 456             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 457             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 458             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 459             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 460             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 461             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 462
 463         return account_id, account
 464
 465
 466     def set_label(self, name, text = None):
 467         changed = False
 468         old_text = self.labels.get(name)
 469         if text:
 470             if old_text != text:
 471                 self.labels[name] = text
 472                 changed = True
 473         else:
 474             if old_text:
 475                 self.labels.pop(name)
 476                 changed = True
 477
 478         if changed:
 479             self.storage.put('labels', self.labels, True)
 480
 481         run_hook('set_label', name, text, changed)
 482         return changed
 483
 484
 485
 486     def create_account(self, account_type = '1of1', name = None):
 487         k, account = self.next_account(account_type)
 488         if k in self.pending_accounts:
 489             self.pending_accounts.pop(k)
 490             self.storage.put('pending_accounts', self.pending_accounts)
 491
 492         self.accounts[k] = account
 493         self.save_accounts()
 494         if name:
 495             self.set_label(k, name)
 496
 497
 498     def save_accounts(self):
 499         d = {}
 500         for k, v in self.accounts.items():
 501             d[k] = v.dump()
 502         self.storage.put('accounts', d, True)
 503
 504
 505
 506     def load_accounts(self):
 507         d = self.storage.get('accounts', {})
 508         self.accounts = {}
 509         for k, v in d.items():
 510             if k == 0:
 511                 v['mpk'] = self.storage.get('master_public_key')
 512                 self.accounts[k] = OldAccount(v)
 513             elif '&' in k:
 514                 self.accounts[k] = BIP32_Account_2of2(v)
 515             else:
 516                 self.accounts[k] = BIP32_Account(v)
 517
 518         self.pending_accounts = self.storage.get('pending_accounts',{})
 519
 520
 521     def delete_pending_account(self, k):
 522         self.pending_accounts.pop(k)
 523         self.storage.put('pending_accounts', self.pending_accounts)
 524
 525     def account_is_pending(self, k):
 526         return k in self.pending_accounts
 527
 528     def create_pending_account(self, acct_type, name):
 529         k, addr = self.new_account_address(acct_type)
 530         self.set_label(k, name)
 531         self.pending_accounts[k] = addr
 532         self.storage.put('pending_accounts', self.pending_accounts)
 533
 534     def get_pending_accounts(self):
 535         return self.pending_accounts.items()
 536
 537
 538     def addresses(self, include_change = True, _next=True):
 539         o = self.get_account_addresses(-1, include_change)
 540         for a in self.accounts.keys():
 541             o += self.get_account_addresses(a, include_change)
 542
 543         if _next:
 544             for addr in self.next_addresses.values():
 545                 if addr not in o:
 546                     o += [addr]
 547         return o
 548
 549
 550     def is_mine(self, address):
 551         return address in self.addresses(True)
 552
 553
 554     def is_change(self, address):
 555         if not self.is_mine(address): return False
 556         if address in self.imported_keys.keys(): return False
 557         acct, s = self.get_address_index(address)
 558         if s is None: return False
 559         return s[0] == 1
 560
 561     def get_master_public_key(self):
 562         c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 563         return repr((c, K))
 564
 565     def get_master_private_key(self, account, password):
 566         k = self.master_private_keys.get(account)
 567         if not k: return
 568         master_k = pw_decode( k, password)
 569         master_c, master_K, master_Kc = self.master_public_keys[account]
 570         try:
 571             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 572             assert K.encode('hex') == master_K
 573         except Exception:
 574             raise Exception("Invalid password")
 575         return master_k
 576
 577
 578     def get_address_index(self, address):
 579         if address in self.imported_keys.keys():
 580             return -1, None
 581
 582         for account in self.accounts.keys():
 583             for for_change in [0,1]:
 584                 addresses = self.accounts[account].get_addresses(for_change)
 585                 for addr in addresses:
 586                     if address == addr:
 587                         return account, (for_change, addresses.index(addr))
 588
 589         for k,v in self.next_addresses.items():
 590             if v == address:
 591                 return k, (0,0)
 592
 593         raise Exception("Address not found", address)
 594
 595
 596     def getpubkeys(self, addr):
 597         assert is_valid(addr) and self.is_mine(addr)
 598         account, sequence = self.get_address_index(addr)
 599         if account != -1:
 600             a = self.accounts[account]
 601             return a.get_pubkeys( sequence )
 602
 603
 604     def get_roots(self, account):
 605         roots = []
 606         for a in account.split('&'):
 607             s = a.strip()
 608             m = re.match("(m/\d+'/)(\d+)", s)
 609             roots.append( m.group(1) )
 610         return roots
 611
 612     def is_seeded(self, account):
 613         if type(account) is int:
 614             return self.seed is not None
 615
 616         for root in self.get_roots(account):
 617             if root not in self.master_private_keys.keys():
 618                 return False
 619         return True
 620
 621     def rebase_sequence(self, account, sequence):
 622         c, i = sequence
 623         dd = []
 624         for a in account.split('&'):
 625             s = a.strip()
 626             m = re.match("(m/\d+'/)(\d+)", s)
 627             root = m.group(1)
 628             num = int(m.group(2))
 629             dd.append( (root, [num,c,i] ) )
 630         return dd
 631
 632
 633     def get_keyID(self, account, sequence):
 634         if account == 0:
 635             a, b = sequence
 636             mpk = self.storage.get('master_public_key')
 637             return 'old(%s,%d,%d)'%(mpk,a,b)
 638
 639         rs = self.rebase_sequence(account, sequence)
 640         dd = []
 641         for root, public_sequence in rs:
 642             c, K, cK = self.master_public_keys[root]
 643             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 644             dd.append( 'bip32(%s,%s,%s)'%(c, cK, s) )
 645         return '&'.join(dd)
 646
 647
 648     def get_seed(self, password):
 649         s = pw_decode(self.seed, password)
 650         seed = mnemonic_to_seed(s,'').encode('hex')
 651         return seed
 652
 653
 654     def get_mnemonic(self, password):
 655         return pw_decode(self.seed, password)
 656
 657
 658     def get_private_key(self, address, password):
 659         if self.is_watching_only():
 660             return []
 661
 662         # first check the provided password
 663         seed = self.get_seed(password)
 664
 665         out = []
 666         if address in self.imported_keys.keys():
 667             out.append( pw_decode( self.imported_keys[address], password ) )
 668         else:
 669             account, sequence = self.get_address_index(address)
 670             if account == 0:
 671                 pk = self.accounts[account].get_private_key(seed, sequence)
 672                 out.append(pk)
 673                 return out
 674
 675             # assert address == self.accounts[account].get_address(*sequence)
 676             rs = self.rebase_sequence( account, sequence)
 677             for root, public_sequence in rs:
 678
 679                 if root not in self.master_private_keys.keys(): continue
 680                 master_k = self.get_master_private_key(root, password)
 681                 master_c, _, _ = self.master_public_keys[root]
 682                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 683                 out.append(pk)
 684
 685         return out
 686
 687
 688     def add_keypairs_from_wallet(self, tx, keypairs, password):
 689         for txin in tx.inputs:
 690             address = txin['address']
 691             if not self.is_mine(address):
 692                 continue
 693             private_keys = self.get_private_key(address, password)
 694             for sec in private_keys:
 695                 pubkey = public_key_from_private_key(sec)
 696                 keypairs[ pubkey ] = sec
 697                 if address in self.imported_keys.keys():
 698                     txin['redeemPubkey'] = pubkey
 699
 700
 701     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 702         # first check the provided password
 703         seed = self.get_seed(password)
 704
 705         for txin in tx.inputs:
 706             keyid = txin.get('KeyID')
 707             if keyid:
 708                 roots = []
 709                 for s in keyid.split('&'):
 710                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 711                     if not m: continue
 712                     c = m.group(1)
 713                     K = m.group(2)
 714                     sequence = m.group(3)
 715                     root = self.find_root_by_master_key(c,K)
 716                     if not root: continue
 717                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 718                     root = root + '%d'%sequence[0]
 719                     sequence = sequence[1:]
 720                     roots.append((root,sequence))
 721
 722                 account_id = " & ".join( map(lambda x:x[0], roots) )
 723                 account = self.accounts.get(account_id)
 724                 if not account: continue
 725                 addr = account.get_address(*sequence)
 726                 txin['address'] = addr # fixme: side effect
 727                 pk = self.get_private_key(addr, password)
 728                 for sec in pk:
 729                     pubkey = public_key_from_private_key(sec)
 730                     keypairs[pubkey] = sec
 731
 732
 733
 734     def signrawtransaction(self, tx, input_info, private_keys, password):
 735
 736         # check that the password is correct
 737         seed = self.get_seed(password)
 738
 739         # add input info
 740         tx.add_input_info(input_info)
 741
 742         # add redeem script for coins that are in the wallet
 743         # FIXME: add redeemPubkey too!
 744
 745         try:
 746             unspent_coins = self.get_unspent_coins()
 747         except:
 748             # an exception may be raised is the wallet is not synchronized
 749             unspent_coins = []
 750
 751         for txin in tx.inputs:
 752             for item in unspent_coins:
 753                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 754                     print_error( "tx input is in unspent coins" )
 755                     txin['scriptPubKey'] = item['scriptPubKey']
 756                     account, sequence = self.get_address_index(item['address'])
 757                     if account != -1:
 758                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 759                         print_error("added redeemScript", txin['redeemScript'])
 760                     break
 761
 762
 763         # build a list of public/private keys
 764         keypairs = {}
 765
 766         # add private keys from parameter
 767         for sec in private_keys:
 768             pubkey = public_key_from_private_key(sec)
 769             keypairs[ pubkey ] = sec
 770
 771         # add private_keys from KeyID
 772         self.add_keypairs_from_KeyID(tx, keypairs, password)
 773
 774         # add private keys from wallet
 775         self.add_keypairs_from_wallet(tx, keypairs, password)
 776         self.sign_transaction(tx, keypairs, password)
 777
 778
 779     def sign_message(self, address, message, password):
 780         keys = self.get_private_key(address, password)
 781         assert len(keys) == 1
 782         sec = keys[0]
 783         key = regenerate_key(sec)
 784         compressed = is_compressed(sec)
 785         return key.sign_message(message, compressed, address)
 786
 787
 788
 789     def decrypt_message(self, pubkey, message, password):
 790         address = public_key_to_bc_address(pubkey.decode('hex'))
 791         keys = self.get_private_key(address, password)
 792         secret = keys[0]
 793         ec = regenerate_key(secret)
 794         decrypted = ec.decrypt_message(message)
 795         return decrypted[0]
 796
 797
 798     def change_gap_limit(self, value):
 799         if value >= self.gap_limit:
 800             self.gap_limit = value
 801             self.storage.put('gap_limit', self.gap_limit, True)
 802             #self.interface.poke('synchronizer')
 803             return True
 804
 805         elif value >= self.min_acceptable_gap():
 806             for key, account in self.accounts.items():
 807                 addresses = account[0]
 808                 k = self.num_unused_trailing_addresses(addresses)
 809                 n = len(addresses) - k + value
 810                 addresses = addresses[0:n]
 811                 self.accounts[key][0] = addresses
 812
 813             self.gap_limit = value
 814             self.storage.put('gap_limit', self.gap_limit, True)
 815             self.save_accounts()
 816             return True
 817         else:
 818             return False
 819
 820     def num_unused_trailing_addresses(self, addresses):
 821         k = 0
 822         for a in addresses[::-1]:
 823             if self.history.get(a):break
 824             k = k + 1
 825         return k
 826
 827     def min_acceptable_gap(self):
 828         # fixme: this assumes wallet is synchronized
 829         n = 0
 830         nmax = 0
 831
 832         for account in self.accounts.values():
 833             addresses = account.get_addresses(0)
 834             k = self.num_unused_trailing_addresses(addresses)
 835             for a in addresses[0:-k]:
 836                 if self.history.get(a):
 837                     n = 0
 838                 else:
 839                     n += 1
 840                     if n > nmax: nmax = n
 841         return nmax + 1
 842
 843
 844     def address_is_old(self, address):
 845         age = -1
 846         h = self.history.get(address, [])
 847         if h == ['*']:
 848             return True
 849         for tx_hash, tx_height in h:
 850             if tx_height == 0:
 851                 tx_age = 0
 852             else:
 853                 tx_age = self.network.get_local_height() - tx_height + 1
 854             if tx_age > age:
 855                 age = tx_age
 856         return age > 2
 857
 858
 859     def synchronize_sequence(self, account, for_change):
 860         limit = self.gap_limit_for_change if for_change else self.gap_limit
 861         new_addresses = []
 862         while True:
 863             addresses = account.get_addresses(for_change)
 864             if len(addresses) < limit:
 865                 address = account.create_new_address(for_change)
 866                 self.history[address] = []
 867                 new_addresses.append( address )
 868                 continue
 869
 870             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 871                 break
 872             else:
 873                 address = account.create_new_address(for_change)
 874                 self.history[address] = []
 875                 new_addresses.append( address )
 876
 877         return new_addresses
 878
 879
 880
 881     def create_pending_accounts(self):
 882         for account_type in ['1of1','2of2','2of3']:
 883             if not self.has_master_public_keys(account_type):
 884                 continue
 885             k, a = self.new_account_address(account_type)
 886             if self.address_is_old(a):
 887                 print_error( "creating account", a )
 888                 self.create_account(account_type)
 889                 self.next_addresses.pop(k)
 890
 891
 892     def synchronize_account(self, account):
 893         new = []
 894         new += self.synchronize_sequence(account, 0)
 895         new += self.synchronize_sequence(account, 1)
 896         return new
 897
 898
 899     def synchronize(self):
 900         if self.master_public_keys:
 901             self.create_pending_accounts()
 902         new = []
 903         for account in self.accounts.values():
 904             new += self.synchronize_account(account)
 905         if new:
 906             self.save_accounts()
 907             self.storage.put('addr_history', self.history, True)
 908         return new
 909
 910
 911     def is_found(self):
 912         return self.history.values() != [[]] * len(self.history)
 913
 914
 915     def add_contact(self, address, label=None):
 916         self.addressbook.append(address)
 917         self.storage.put('contacts', self.addressbook, True)
 918         if label:
 919             self.set_label(address, label)
 920
 921
 922     def delete_contact(self, addr):
 923         if addr in self.addressbook:
 924             self.addressbook.remove(addr)
 925             self.storage.put('addressbook', self.addressbook, True)
 926
 927
 928     def fill_addressbook(self):
 929         for tx_hash, tx in self.transactions.items():
 930             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 931             if is_send:
 932                 for addr, v in tx.outputs:
 933                     if not self.is_mine(addr) and addr not in self.addressbook:
 934                         self.addressbook.append(addr)
 935         # redo labels
 936         # self.update_tx_labels()
 937
 938     def get_num_tx(self, address):
 939         n = 0
 940         for tx in self.transactions.values():
 941             if address in map(lambda x:x[0], tx.outputs): n += 1
 942         return n
 943
 944
 945     def get_address_flags(self, addr):
 946         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 947         flags += "F" if addr in self.frozen_addresses else "-"
 948         return flags
 949
 950
 951     def get_tx_value(self, tx, account=None):
 952         domain = self.get_account_addresses(account)
 953         return tx.get_value(domain, self.prevout_values)
 954
 955
 956     def update_tx_outputs(self, tx_hash):
 957         tx = self.transactions.get(tx_hash)
 958
 959         for i, (addr, value) in enumerate(tx.outputs):
 960             key = tx_hash+ ':%d'%i
 961             self.prevout_values[key] = value
 962
 963         for item in tx.inputs:
 964             if self.is_mine(item.get('address')):
 965                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 966                 self.spent_outputs.append(key)
 967
 968
 969     def get_addr_balance(self, address):
 970         assert self.is_mine(address)
 971         h = self.history.get(address,[])
 972         if h == ['*']: return 0,0
 973         c = u = 0
 974         received_coins = []   # list of coins received at address
 975
 976         for tx_hash, tx_height in h:
 977             tx = self.transactions.get(tx_hash)
 978             if not tx: continue
 979
 980             for i, (addr, value) in enumerate(tx.outputs):
 981                 if addr == address:
 982                     key = tx_hash + ':%d'%i
 983                     received_coins.append(key)
 984
 985         for tx_hash, tx_height in h:
 986             tx = self.transactions.get(tx_hash)
 987             if not tx: continue
 988             v = 0
 989
 990             for item in tx.inputs:
 991                 addr = item.get('address')
 992                 if addr == address:
 993                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 994                     value = self.prevout_values.get( key )
 995                     if key in received_coins:
 996                         v -= value
 997
 998             for i, (addr, value) in enumerate(tx.outputs):
 999                 key = tx_hash + ':%d'%i
1000                 if addr == address:
1001                     v += value
1002
1003             if tx_height:
1004                 c += v
1005             else:
1006                 u += v
1007         return c, u
1008
1009
1010     def get_account_name(self, k):
1011         default = "Unnamed account"
1012         m = re.match("m/0'/(\d+)", k)
1013         if m:
1014             num = m.group(1)
1015             if num == '0':
1016                 default = "Main account"
1017             else:
1018                 default = "Account %s"%num
1019
1020         m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1021         if m:
1022             num = m.group(1)
1023             default = "2of2 account %s"%num
1024         name = self.labels.get(k, default)
1025         return name
1026
1027
1028     def get_account_names(self):
1029         accounts = {}
1030         for k, account in self.accounts.items():
1031             accounts[k] = self.get_account_name(k)
1032         if self.imported_keys:
1033             accounts[-1] = 'Imported keys'
1034         return accounts
1035
1036
1037     def get_account_addresses(self, a, include_change=True):
1038         if a is None:
1039             o = self.addresses(True)
1040         elif a == -1:
1041             o = self.imported_keys.keys()
1042         else:
1043             ac = self.accounts[a]
1044             o = ac.get_addresses(0)
1045             if include_change: o += ac.get_addresses(1)
1046         return o
1047
1048     def get_imported_balance(self):
1049         return self.get_balance(self.imported_keys.keys())
1050
1051     def get_account_balance(self, account):
1052         return self.get_balance(self.get_account_addresses(account))
1053
1054     def get_frozen_balance(self):
1055         return self.get_balance(self.frozen_addresses)
1056
1057     def get_balance(self, domain=None):
1058         if domain is None: domain = self.addresses(True)
1059         cc = uu = 0
1060         for addr in domain:
1061             c, u = self.get_addr_balance(addr)
1062             cc += c
1063             uu += u
1064         return cc, uu
1065
1066
1067     def get_unspent_coins(self, domain=None):
1068         coins = []
1069         if domain is None: domain = self.addresses(True)
1070         for addr in domain:
1071             h = self.history.get(addr, [])
1072             if h == ['*']: continue
1073             for tx_hash, tx_height in h:
1074                 tx = self.transactions.get(tx_hash)
1075                 if tx is None: raise Exception("Wallet not synchronized")
1076                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1077                 for o in tx.d.get('outputs'):
1078                     output = o.copy()
1079                     if output.get('address') != addr: continue
1080                     key = tx_hash + ":%d" % output.get('prevout_n')
1081                     if key in self.spent_outputs: continue
1082                     output['prevout_hash'] = tx_hash
1083                     output['height'] = tx_height
1084                     output['coinbase'] = is_coinbase
1085                     coins.append((tx_height, output))
1086
1087         # sort by age
1088         if coins:
1089             coins = sorted(coins)
1090             if coins[-1][0] != 0:
1091                 while coins[0][0] == 0:
1092                     coins = coins[1:] + [ coins[0] ]
1093         return [x[1] for x in coins]
1094
1095
1096     def choose_tx_inputs( self, amount, fixed_fee, num_outputs, domain = None ):
1097         """ todo: minimize tx size """
1098         total = 0
1099         fee = self.fee if fixed_fee is None else fixed_fee
1100         if domain is None:
1101             domain = self.addresses(True)
1102
1103         for i in self.frozen_addresses:
1104             if i in domain: domain.remove(i)
1105
1106         coins = self.get_unspent_coins(domain)
1107         inputs = []
1108
1109         for item in coins:
1110             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.get_local_height():
1111                 continue
1112             addr = item.get('address')
1113             v = item.get('value')
1114             total += v
1115             inputs.append(item)
1116             fee = self.estimated_fee(inputs, num_outputs) if fixed_fee is None else fixed_fee
1117             if total >= amount + fee: break
1118         else:
1119             inputs = []
1120
1121         return inputs, total, fee
1122
1123
1124     def set_fee(self, fee):
1125         if self.fee != fee:
1126             self.fee = fee
1127             self.storage.put('fee_per_kb', self.fee, True)
1128
1129     def estimated_fee(self, inputs, num_outputs):
1130         estimated_size =  len(inputs) * 180 + num_outputs * 34    # this assumes non-compressed keys
1131         fee = self.fee * int(math.ceil(estimated_size/1000.))
1132         return fee
1133
1134
1135     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1136         "add change to a transaction"
1137         change_amount = total - ( amount + fee )
1138         if change_amount > DUST_THRESHOLD:
1139             if not change_addr:
1140
1141                 # send change to one of the accounts involved in the tx
1142                 address = inputs[0].get('address')
1143                 account, _ = self.get_address_index(address)
1144
1145                 if not self.use_change or account == -1:
1146                     change_addr = inputs[-1]['address']
1147                 else:
1148                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1149
1150             # Insert the change output at a random position in the outputs
1151             posn = random.randint(0, len(outputs))
1152             outputs[posn:posn] = [( change_addr,  change_amount)]
1153         return outputs
1154
1155
1156     def get_history(self, address):
1157         with self.lock:
1158             return self.history.get(address)
1159
1160
1161     def get_status(self, h):
1162         if not h: return None
1163         if h == ['*']: return '*'
1164         status = ''
1165         for tx_hash, height in h:
1166             status += tx_hash + ':%d:' % height
1167         return hashlib.sha256( status ).digest().encode('hex')
1168
1169
1170     def receive_tx_callback(self, tx_hash, tx, tx_height):
1171
1172         with self.transaction_lock:
1173             self.add_extra_addresses(tx)
1174             if not self.check_new_tx(tx_hash, tx):
1175                 # may happen due to pruning
1176                 print_error("received transaction that is no longer referenced in history", tx_hash)
1177                 return
1178             self.transactions[tx_hash] = tx
1179             self.network.interface.pending_transactions_for_notifications.append(tx)
1180             self.save_transactions()
1181             if self.verifier and tx_height>0:
1182                 self.verifier.add(tx_hash, tx_height)
1183             self.update_tx_outputs(tx_hash)
1184
1185
1186     def save_transactions(self):
1187         tx = {}
1188         for k,v in self.transactions.items():
1189             tx[k] = str(v)
1190         self.storage.put('transactions', tx, True)
1191
1192     def receive_history_callback(self, addr, hist):
1193
1194         if not self.check_new_history(addr, hist):
1195             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1196
1197         with self.lock:
1198             self.history[addr] = hist
1199             self.storage.put('addr_history', self.history, True)
1200
1201         if hist != ['*']:
1202             for tx_hash, tx_height in hist:
1203                 if tx_height>0:
1204                     # add it in case it was previously unconfirmed
1205                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1206
1207
1208     def get_tx_history(self, account=None):
1209         if not self.verifier:
1210             return []
1211
1212         with self.transaction_lock:
1213             history = self.transactions.items()
1214             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1215             result = []
1216
1217             balance = 0
1218             for tx_hash, tx in history:
1219                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1220                 if v is not None: balance += v
1221
1222             c, u = self.get_account_balance(account)
1223
1224             if balance != c+u:
1225                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1226
1227             balance = c + u - balance
1228             for tx_hash, tx in history:
1229                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1230                 if not is_relevant:
1231                     continue
1232                 if value is not None:
1233                     balance += value
1234
1235                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1236                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1237
1238         return result
1239
1240
1241     def get_label(self, tx_hash):
1242         label = self.labels.get(tx_hash)
1243         is_default = (label == '') or (label is None)
1244         if is_default: label = self.get_default_label(tx_hash)
1245         return label, is_default
1246
1247
1248     def get_default_label(self, tx_hash):
1249         tx = self.transactions.get(tx_hash)
1250         default_label = ''
1251         if tx:
1252             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1253             if is_mine:
1254                 for o in tx.outputs:
1255                     o_addr, _ = o
1256                     if not self.is_mine(o_addr):
1257                         try:
1258                             default_label = self.labels[o_addr]
1259                         except KeyError:
1260                             default_label = '>' + o_addr
1261                         break
1262                 else:
1263                     default_label = '(internal)'
1264             else:
1265                 for o in tx.outputs:
1266                     o_addr, _ = o
1267                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1268                         break
1269                 else:
1270                     for o in tx.outputs:
1271                         o_addr, _ = o
1272                         if self.is_mine(o_addr):
1273                             break
1274                     else:
1275                         o_addr = None
1276
1277                 if o_addr:
1278                     dest_label = self.labels.get(o_addr)
1279                     try:
1280                         default_label = self.labels[o_addr]
1281                     except KeyError:
1282                         default_label = '<' + o_addr
1283
1284         return default_label
1285
1286
1287     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1288         for address, x in outputs:
1289             assert is_valid(address), "Address " + address + " is invalid!"
1290         amount = sum( map(lambda x:x[1], outputs) )
1291         inputs, total, fee = self.choose_tx_inputs( amount, fee, len(outputs), domain )
1292         if not inputs:
1293             raise ValueError("Not enough funds")
1294         self.add_input_info(inputs)
1295         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1296         return Transaction.from_io(inputs, outputs)
1297
1298
1299     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1300         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1301         keypairs = {}
1302         self.add_keypairs_from_wallet(tx, keypairs, password)
1303         if keypairs:
1304             self.sign_transaction(tx, keypairs, password)
1305         return tx
1306
1307
1308     def add_input_info(self, inputs):
1309         for txin in inputs:
1310             address = txin['address']
1311             if address in self.imported_keys.keys():
1312                 continue
1313             account, sequence = self.get_address_index(address)
1314             txin['KeyID'] = self.get_keyID(account, sequence)
1315             redeemScript = self.accounts[account].redeem_script(sequence)
1316             if redeemScript:
1317                 txin['redeemScript'] = redeemScript
1318             else:
1319                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1320
1321
1322     def sign_transaction(self, tx, keypairs, password):
1323         tx.sign(keypairs)
1324         run_hook('sign_transaction', tx, password)
1325
1326
1327     def sendtx(self, tx):
1328         # synchronous
1329         h = self.send_tx(tx)
1330         self.tx_event.wait()
1331         return self.receive_tx(h, tx)
1332
1333     def send_tx(self, tx):
1334         # asynchronous
1335         self.tx_event.clear()
1336         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1337         return tx.hash()
1338
1339     def on_broadcast(self, i, r):
1340         self.tx_result = r.get('result')
1341         self.tx_event.set()
1342
1343     def receive_tx(self, tx_hash, tx):
1344         out = self.tx_result
1345         if out != tx_hash:
1346             return False, "error: " + out
1347         run_hook('receive_tx', tx, self)
1348         return True, out
1349
1350
1351
1352     def update_password(self, old_password, new_password):
1353         if new_password == '': new_password = None
1354         decoded = self.get_seed(old_password)
1355         self.seed = pw_encode( decoded, new_password)
1356         self.storage.put('seed', self.seed, True)
1357         self.use_encryption = (new_password != None)
1358         self.storage.put('use_encryption', self.use_encryption,True)
1359         for k in self.imported_keys.keys():
1360             a = self.imported_keys[k]
1361             b = pw_decode(a, old_password)
1362             c = pw_encode(b, new_password)
1363             self.imported_keys[k] = c
1364         self.storage.put('imported_keys', self.imported_keys, True)
1365
1366         for k, v in self.master_private_keys.items():
1367             b = pw_decode(v, old_password)
1368             c = pw_encode(b, new_password)
1369             self.master_private_keys[k] = c
1370         self.storage.put('master_private_keys', self.master_private_keys, True)
1371
1372
1373     def freeze(self,addr):
1374         if self.is_mine(addr) and addr not in self.frozen_addresses:
1375             self.frozen_addresses.append(addr)
1376             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1377             return True
1378         else:
1379             return False
1380
1381     def unfreeze(self,addr):
1382         if self.is_mine(addr) and addr in self.frozen_addresses:
1383             self.frozen_addresses.remove(addr)
1384             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1385             return True
1386         else:
1387             return False
1388
1389
1390     def set_verifier(self, verifier):
1391         self.verifier = verifier
1392
1393         # review transactions that are in the history
1394         for addr, hist in self.history.items():
1395             if hist == ['*']: continue
1396             for tx_hash, tx_height in hist:
1397                 if tx_height>0:
1398                     # add it in case it was previously unconfirmed
1399                     self.verifier.add(tx_hash, tx_height)
1400
1401
1402         # if we are on a pruning server, remove unverified transactions
1403         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1404         for tx_hash in self.transactions.keys():
1405             if tx_hash not in vr:
1406                 self.transactions.pop(tx_hash)
1407
1408
1409
1410     def check_new_history(self, addr, hist):
1411
1412         # check that all tx in hist are relevant
1413         if hist != ['*']:
1414             for tx_hash, height in hist:
1415                 tx = self.transactions.get(tx_hash)
1416                 if not tx: continue
1417                 if not tx.has_address(addr):
1418                     return False
1419
1420         # check that we are not "orphaning" a transaction
1421         old_hist = self.history.get(addr,[])
1422         if old_hist == ['*']: return True
1423
1424         for tx_hash, height in old_hist:
1425             if tx_hash in map(lambda x:x[0], hist): continue
1426             found = False
1427             for _addr, _hist in self.history.items():
1428                 if _addr == addr: continue
1429                 if _hist == ['*']: continue
1430                 _tx_hist = map(lambda x:x[0], _hist)
1431                 if tx_hash in _tx_hist:
1432                     found = True
1433                     break
1434
1435             if not found:
1436                 tx = self.transactions.get(tx_hash)
1437                 # tx might not be there
1438                 if not tx: continue
1439
1440                 # already verified?
1441                 if self.verifier.get_height(tx_hash):
1442                     continue
1443                 # unconfirmed tx
1444                 print_error("new history is orphaning transaction:", tx_hash)
1445                 # check that all outputs are not mine, request histories
1446                 ext_requests = []
1447                 for _addr, _v in tx.outputs:
1448                     # assert not self.is_mine(_addr)
1449                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1450
1451                 ext_h = self.network.synchronous_get(ext_requests)
1452                 print_error("sync:", ext_requests, ext_h)
1453                 height = None
1454                 for h in ext_h:
1455                     if h == ['*']: continue
1456                     for item in h:
1457                         if item.get('tx_hash') == tx_hash:
1458                             height = item.get('height')
1459                 if height:
1460                     print_error("found height for", tx_hash, height)
1461                     self.verifier.add(tx_hash, height)
1462                 else:
1463                     print_error("removing orphaned tx from history", tx_hash)
1464                     self.transactions.pop(tx_hash)
1465
1466         return True
1467
1468
1469
1470     def check_new_tx(self, tx_hash, tx):
1471         # 1 check that tx is referenced in addr_history.
1472         addresses = []
1473         for addr, hist in self.history.items():
1474             if hist == ['*']:continue
1475             for txh, height in hist:
1476                 if txh == tx_hash:
1477                     addresses.append(addr)
1478
1479         if not addresses:
1480             return False
1481
1482         # 2 check that referencing addresses are in the tx
1483         for addr in addresses:
1484             if not tx.has_address(addr):
1485                 return False
1486
1487         return True
1488
1489
1490     def start_threads(self, network):
1491         from verifier import TxVerifier
1492         self.network = network
1493         if self.network is not None:
1494             self.verifier = TxVerifier(self.network, self.storage)
1495             self.verifier.start()
1496             self.set_verifier(self.verifier)
1497             self.synchronizer = WalletSynchronizer(self, network)
1498             self.synchronizer.start()
1499         else:
1500             self.verifier = None
1501             self.synchronizer =None
1502
1503     def stop_threads(self):
1504         if self.network:
1505             self.verifier.stop()
1506             self.synchronizer.stop()
1507
1508
1509     def restore(self, callback):
1510         from i18n import _
1511         def wait_for_wallet():
1512             self.set_up_to_date(False)
1513             while not self.is_up_to_date():
1514                 msg = "%s\n%s %d\n%s %.1f"%(
1515                     _("Please wait..."),
1516                     _("Addresses generated:"),
1517                     len(self.addresses(True)),
1518                     _("Kilobytes received:"),
1519                     self.network.interface.bytes_received/1024.)
1520
1521                 apply(callback, (msg,))
1522                 time.sleep(0.1)
1523
1524         def wait_for_network():
1525             while not self.network.is_connected():
1526                 msg = "%s \n" % (_("Connecting..."))
1527                 apply(callback, (msg,))
1528                 time.sleep(0.1)
1529
1530         # wait until we are connected, because the user might have selected another server
1531         if self.network:
1532             wait_for_network()
1533             wait_for_wallet()
1534         else:
1535             self.synchronize()
1536
1537         self.fill_addressbook()
1538
1539
1540
1541
1542 class WalletSynchronizer(threading.Thread):
1543
1544
1545     def __init__(self, wallet, network):
1546         threading.Thread.__init__(self)
1547         self.daemon = True
1548         self.wallet = wallet
1549         self.network = network
1550         self.was_updated = True
1551         self.running = False
1552         self.lock = threading.Lock()
1553         self.queue = Queue.Queue()
1554
1555     def stop(self):
1556         with self.lock: self.running = False
1557
1558     def is_running(self):
1559         with self.lock: return self.running
1560
1561
1562     def subscribe_to_addresses(self, addresses):
1563         messages = []
1564         for addr in addresses:
1565             messages.append(('blockchain.address.subscribe', [addr]))
1566         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1567
1568
1569     def run(self):
1570         with self.lock:
1571             self.running = True
1572
1573         while self.is_running():
1574
1575             if not self.network.is_connected():
1576                 self.network.wait_until_connected()
1577
1578             self.run_interface()
1579
1580
1581     def run_interface(self):
1582
1583         print_error("synchronizer: connected to", self.network.main_server())
1584
1585         requested_tx = []
1586         missing_tx = []
1587         requested_histories = {}
1588
1589         # request any missing transactions
1590         for history in self.wallet.history.values():
1591             if history == ['*']: continue
1592             for tx_hash, tx_height in history:
1593                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1594                     missing_tx.append( (tx_hash, tx_height) )
1595
1596         if missing_tx:
1597             print_error("missing tx", missing_tx)
1598
1599         # subscriptions
1600         self.subscribe_to_addresses(self.wallet.addresses(True))
1601
1602         while self.is_running():
1603             # 1. create new addresses
1604             new_addresses = self.wallet.synchronize()
1605
1606             # request missing addresses
1607             if new_addresses:
1608                 self.subscribe_to_addresses(new_addresses)
1609
1610             # request missing transactions
1611             for tx_hash, tx_height in missing_tx:
1612                 if (tx_hash, tx_height) not in requested_tx:
1613                     self.network.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1614                     requested_tx.append( (tx_hash, tx_height) )
1615             missing_tx = []
1616
1617             # detect if situation has changed
1618             if self.network.is_up_to_date() and self.queue.empty():
1619                 if not self.wallet.is_up_to_date():
1620                     self.wallet.set_up_to_date(True)
1621                     self.was_updated = True
1622             else:
1623                 if self.wallet.is_up_to_date():
1624                     self.wallet.set_up_to_date(False)
1625                     self.was_updated = True
1626
1627             if self.was_updated:
1628                 self.network.trigger_callback('updated')
1629                 self.was_updated = False
1630
1631             # 2. get a response
1632             try:
1633                 r = self.queue.get(block=True, timeout=1)
1634             except Queue.Empty:
1635                 continue
1636
1637             # see if it changed
1638             #if interface != self.network.interface:
1639             #    break
1640
1641             if not r:
1642                 continue
1643
1644             # 3. handle response
1645             method = r['method']
1646             params = r['params']
1647             result = r.get('result')
1648             error = r.get('error')
1649             if error:
1650                 print "error", r
1651                 continue
1652
1653             if method == 'blockchain.address.subscribe':
1654                 addr = params[0]
1655                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1656                     if requested_histories.get(addr) is None:
1657                         self.network.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1658                         requested_histories[addr] = result
1659
1660             elif method == 'blockchain.address.get_history':
1661                 addr = params[0]
1662                 print_error("receiving history", addr, result)
1663                 if result == ['*']:
1664                     assert requested_histories.pop(addr) == '*'
1665                     self.wallet.receive_history_callback(addr, result)
1666                 else:
1667                     hist = []
1668                     # check that txids are unique
1669                     txids = []
1670                     for item in result:
1671                         tx_hash = item['tx_hash']
1672                         if tx_hash not in txids:
1673                             txids.append(tx_hash)
1674                             hist.append( (tx_hash, item['height']) )
1675
1676                     if len(hist) != len(result):
1677                         raise Exception("error: server sent history with non-unique txid", result)
1678
1679                     # check that the status corresponds to what was announced
1680                     rs = requested_histories.pop(addr)
1681                     if self.wallet.get_status(hist) != rs:
1682                         raise Exception("error: status mismatch: %s"%addr)
1683
1684                     # store received history
1685                     self.wallet.receive_history_callback(addr, hist)
1686
1687                     # request transactions that we don't have
1688                     for tx_hash, tx_height in hist:
1689                         if self.wallet.transactions.get(tx_hash) is None:
1690                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1691                                 missing_tx.append( (tx_hash, tx_height) )
1692
1693             elif method == 'blockchain.transaction.get':
1694                 tx_hash = params[0]
1695                 tx_height = params[1]
1696                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1697                 tx = Transaction(result)
1698                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1699                 self.was_updated = True
1700                 requested_tx.remove( (tx_hash, tx_height) )
1701                 print_error("received tx:", tx_hash, len(tx.raw))
1702
1703             else:
1704                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1705
1706             if self.was_updated and not requested_tx:
1707                 self.network.trigger_callback('updated')
1708                 # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1709                 self.network.trigger_callback("new_transaction")
1710                 self.was_updated = False
1711
1712
1713
1714
1715 class OldWallet(NewWallet):
1716
1717     def init_seed(self, seed):
1718         import mnemonic
1719
1720         if self.seed:
1721             raise Exception("a seed exists")
1722
1723         if not seed:
1724             seed = random_seed(128)
1725
1726         self.seed_version = OLD_SEED_VERSION
1727
1728         # see if seed was entered as hex
1729         seed = seed.strip()
1730         try:
1731             assert seed
1732             seed.decode('hex')
1733             self.seed = str(seed)
1734             return
1735         except Exception:
1736             pass
1737
1738         words = seed.split()
1739         try:
1740             mnemonic.mn_decode(words)
1741         except Exception:
1742             raise
1743
1744         self.seed = mnemonic.mn_decode(words)
1745
1746         if not self.seed:
1747             raise Exception("Invalid seed")
1748
1749
1750
1751     def get_master_public_key(self):
1752         return self.storage.get("master_public_key")
1753
1754     def create_accounts(self, password):
1755         seed = pw_decode(self.seed, password)
1756         mpk = OldAccount.mpk_from_seed(seed)
1757         self.create_account(mpk)
1758
1759     def create_account(self, mpk):
1760         self.storage.put('master_public_key', mpk, True)
1761         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
1762         self.save_accounts()
1763
1764     def create_watching_only_wallet(self, K0):
1765         self.seed_version = OLD_SEED_VERSION
1766         self.storage.put('seed_version', self.seed_version, True)
1767         self.create_account(K0)
1768
1769     def get_seed(self, password):
1770         seed = pw_decode(self.seed, password)
1771         self.accounts[0].check_seed(seed)
1772         return seed
1773
1774     def get_mnemonic(self, password):
1775         import mnemonic
1776         s = pw_decode(self.seed, password)
1777         return ' '.join(mnemonic.mn_encode(s))
1778
1779
1780     def add_keypairs_from_KeyID(self, tx, keypairs, password):
1781         # first check the provided password
1782         seed = self.get_seed(password)
1783         for txin in tx.inputs:
1784             keyid = txin.get('KeyID')
1785             if keyid:
1786                 m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
1787                 if not m: continue
1788                 mpk = m.group(1)
1789                 if mpk != self.storage.get('master_public_key'): continue
1790                 for_change = int(m.group(2))
1791                 num = int(m.group(3))
1792                 account = self.accounts[0]
1793                 addr = account.get_address(for_change, num)
1794                 txin['address'] = addr # fixme: side effect
1795                 pk = account.get_private_key(seed, (for_change, num))
1796                 pubkey = public_key_from_private_key(pk)
1797                 keypairs[pubkey] = pk
1798
1799
1800     def get_account_name(self, k):
1801         assert k == 0
1802         return 'Main account'
1803
1804
1805
1806
1807 # former WalletFactory
1808 class Wallet(object):
1809
1810     def __new__(self, storage):
1811         config = storage.config
1812         if config.get('bitkey', False):
1813             # if user requested support for Bitkey device,
1814             # import Bitkey driver
1815             from wallet_bitkey import WalletBitkey
1816             return WalletBitkey(config)
1817
1818         if not storage.file_exists:
1819             seed_version = NEW_SEED_VERSION if config.get('bip32') is True else OLD_SEED_VERSION
1820         else:
1821             seed_version = storage.get('seed_version')
1822             if not seed_version:
1823                 seed_version = OLD_SEED_VERSION if len(storage.get('master_public_key')) == 128 else NEW_SEED_VERSION
1824
1825         if seed_version == OLD_SEED_VERSION:
1826             return OldWallet(storage)
1827         elif seed_version == NEW_SEED_VERSION:
1828             return NewWallet(storage)
1829         else:
1830             msg = "This wallet seed is not supported."
1831             if seed_version in [5]:
1832                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%seed_version
1833             print msg
1834             sys.exit(1)
1835
1836
1837
1838     @classmethod
1839     def from_seed(self, seed, storage):
1840         import mnemonic
1841         if not seed:
1842             return
1843
1844         words = seed.strip().split()
1845         try:
1846             mnemonic.mn_decode(words)
1847             uses_electrum_words = True
1848         except Exception:
1849             uses_electrum_words = False
1850
1851         try:
1852             seed.decode('hex')
1853             is_hex = True
1854         except Exception:
1855             is_hex = False
1856
1857         if is_hex or (uses_electrum_words and len(words) != 13):
1858             #print "old style wallet", len(words), words
1859             w = OldWallet(storage)
1860             w.init_seed(seed) #hex
1861         else:
1862             #assert is_seed(seed)
1863             w = NewWallet(storage)
1864             w.init_seed(seed)
1865
1866         return w
1867
1868
1869     @classmethod
1870     def from_mpk(self, s, storage):
1871         try:
1872             mpk, chain = s.split(':')
1873         except:
1874             mpk = s
1875             chain = False
1876
1877         if chain:
1878             w = NewWallet(storage)
1879             w.create_watching_only_wallet(mpk, chain)
1880         else:
1881             w = OldWallet(storage)
1882             w.seed = ''
1883             w.create_watching_only_wallet(mpk)
1884
1885         return w