lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40
  41 # AES encryption
  42 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  43 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  44
  45 def pw_encode(s, password):
  46     if password:
  47         secret = Hash(password)
  48         return EncodeAES(secret, s)
  49     else:
  50         return s
  51
  52 def pw_decode(s, password):
  53     if password is not None:
  54         secret = Hash(password)
  55         try:
  56             d = DecodeAES(secret, s)
  57         except:
  58             raise BaseException('Invalid password')
  59         return d
  60     else:
  61         return s
  62
  63
  64
  65
  66
  67 from version import *
  68
  69
  70 class WalletStorage:
  71
  72     def __init__(self, config):
  73         self.lock = threading.Lock()
  74         self.data = {}
  75         self.file_exists = False
  76         self.path = self.init_path(config)
  77         print_error( "wallet path", self.path )
  78         if self.path:
  79             self.read(self.path)
  80
  81
  82     def init_path(self, config):
  83         """Set the path of the wallet."""
  84
  85         # command line -w option
  86         path = config.get('wallet_path')
  87         if path:
  88             return path
  89
  90         # path in config file
  91         path = config.get('default_wallet_path')
  92         if path:
  93             return path
  94
  95         # default path
  96         dirpath = os.path.join(config.path, "wallets")
  97         if not os.path.exists(dirpath):
  98             os.mkdir(dirpath)
  99
 100         new_path = os.path.join(config.path, "wallets", "default_wallet")
 101
 102         # default path in pre 1.9 versions
 103         old_path = os.path.join(config.path, "electrum.dat")
 104         if os.path.exists(old_path) and not os.path.exists(new_path):
 105             os.rename(old_path, new_path)
 106
 107         return new_path
 108
 109
 110     def read(self, path):
 111         """Read the contents of the wallet file."""
 112         try:
 113             with open(self.path, "r") as f:
 114                 data = f.read()
 115         except IOError:
 116             return
 117         try:
 118             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 119         except:
 120             raise IOError("Cannot read wallet file.")
 121
 122         self.data = d
 123         self.file_exists = True
 124
 125
 126     def get(self, key, default=None):
 127         return self.data.get(key, default)
 128
 129     def put(self, key, value, save = True):
 130
 131         with self.lock:
 132             if value is not None:
 133                 self.data[key] = value
 134             else:
 135                 self.data.pop[key]
 136             if save:
 137                 self.write()
 138
 139     def write(self):
 140         s = repr(self.data)
 141         f = open(self.path,"w")
 142         f.write( s )
 143         f.close()
 144         if 'ANDROID_DATA' not in os.environ:
 145             import stat
 146             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 147
 148
 149 class Wallet:
 150
 151     def __init__(self, storage):
 152
 153         self.storage = storage
 154         self.electrum_version = ELECTRUM_VERSION
 155         self.gap_limit_for_change = 3 # constant
 156
 157         # saved fields
 158         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 159
 160         self.gap_limit             = storage.get('gap_limit', 5)
 161         self.use_change            = storage.get('use_change',True)
 162         self.use_encryption        = storage.get('use_encryption', False)
 163         self.seed                  = storage.get('seed', '')               # encrypted
 164         self.labels                = storage.get('labels', {})
 165         self.frozen_addresses      = storage.get('frozen_addresses',[])
 166         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 167         self.addressbook           = storage.get('contacts', [])
 168
 169         self.imported_keys         = storage.get('imported_keys',{})
 170         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 171
 172         self.fee                   = int(storage.get('fee_per_kb',20000))
 173
 174         self.master_public_keys = storage.get('master_public_keys',{})
 175         self.master_private_keys = storage.get('master_private_keys', {})
 176
 177         self.next_addresses = storage.get('next_addresses',{})
 178
 179         if self.seed_version not in [4, 6]:
 180             msg = "This wallet seed is not supported."
 181             if self.seed_version in [5]:
 182                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 183                 print msg
 184                 sys.exit(1)
 185
 186
 187         self.load_accounts()
 188
 189         self.transactions = {}
 190         tx_list = self.storage.get('transactions',{})
 191         for k,v in tx_list.items():
 192             try:
 193                 tx = Transaction(v)
 194             except:
 195                 print_msg("Warning: Cannot deserialize transactions. skipping")
 196                 continue
 197
 198             self.add_extra_addresses(tx)
 199             self.transactions[k] = tx
 200
 201         for h,tx in self.transactions.items():
 202             if not self.check_new_tx(h, tx):
 203                 print_error("removing unreferenced tx", h)
 204                 self.transactions.pop(h)
 205
 206
 207         # not saved
 208         self.prevout_values = {}     # my own transaction outputs
 209         self.spent_outputs = []
 210
 211         # spv
 212         self.verifier = None
 213
 214         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 215         # interface.is_up_to_date() returns true when all requests have been answered and processed
 216         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 217
 218         self.up_to_date = False
 219         self.lock = threading.Lock()
 220         self.transaction_lock = threading.Lock()
 221         self.tx_event = threading.Event()
 222
 223         for tx_hash, tx in self.transactions.items():
 224             self.update_tx_outputs(tx_hash)
 225
 226
 227     def add_extra_addresses(self, tx):
 228         h = tx.hash()
 229         # find the address corresponding to pay-to-pubkey inputs
 230         tx.add_extra_addresses(self.transactions)
 231         for o in tx.d.get('outputs'):
 232             if o.get('is_pubkey'):
 233                 for tx2 in self.transactions.values():
 234                     tx2.add_extra_addresses({h:tx})
 235
 236
 237
 238
 239     def set_up_to_date(self,b):
 240         with self.lock: self.up_to_date = b
 241
 242
 243     def is_up_to_date(self):
 244         with self.lock: return self.up_to_date
 245
 246
 247     def update(self):
 248         self.up_to_date = False
 249         while not self.is_up_to_date():
 250             time.sleep(0.1)
 251
 252
 253     def import_key(self, sec, password):
 254         # check password
 255         seed = self.get_seed(password)
 256         try:
 257             address = address_from_private_key(sec)
 258         except:
 259             raise BaseException('Invalid private key')
 260
 261         if self.is_mine(address):
 262             raise BaseException('Address already in wallet')
 263
 264         # store the originally requested keypair into the imported keys table
 265         self.imported_keys[address] = pw_encode(sec, password )
 266         self.storage.put('imported_keys', self.imported_keys, True)
 267         if self.synchronizer:
 268             self.synchronizer.subscribe_to_addresses([address])
 269         return address
 270
 271     def delete_imported_key(self, addr):
 272         if addr in self.imported_keys:
 273             self.imported_keys.pop(addr)
 274             self.storage.put('imported_keys', self.imported_keys, True)
 275
 276
 277     def make_seed(self):
 278         import mnemonic, ecdsa
 279         entropy = ecdsa.util.randrange( pow(2,160) )
 280         nonce = 0
 281         while True:
 282             ss = "%040x"%(entropy+nonce)
 283             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 284             # we keep only 13 words, that's approximately 139 bits of entropy
 285             words = mnemonic.mn_encode(s)[0:13]
 286             seed = ' '.join(words)
 287             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 288                 break  # this removes 12 bits of entropy
 289             nonce += 1
 290
 291         return seed
 292
 293
 294     def init_seed(self, seed):
 295         import mnemonic
 296
 297         if self.seed:
 298             raise BaseException("a seed exists")
 299
 300         if not seed:
 301             self.seed = random_seed(128)
 302             self.seed_version = 4
 303             return
 304
 305         #if not seed:
 306         #    self.seed = self.make_seed()
 307         #    self.seed_version = SEED_VERSION
 308         #    return
 309
 310         # find out what kind of wallet we are
 311         try:
 312             seed.decode('hex')
 313             self.seed_version = 4
 314             self.seed = str(seed)
 315             return
 316         except:
 317             pass
 318
 319         words = seed.split()
 320         self.seed_version = 4
 321         self.seed = mnemonic.mn_decode(words)
 322
 323         #try:
 324         #    mnemonic.mn_decode(words)
 325         #    uses_electrum_words = True
 326         #except:
 327         #    uses_electrum_words = False
 328         #
 329         #if uses_electrum_words and len(words) != 13:
 330         #    self.seed_version = 4
 331         #    self.seed = mnemonic.mn_decode(words)
 332         #else:
 333         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 334         #    self.seed_version = SEED_VERSION
 335         #    self.seed = seed
 336
 337
 338     def save_seed(self):
 339         self.storage.put('seed', self.seed, True)
 340         self.storage.put('seed_version', self.seed_version, True)
 341
 342     def create_watching_only_wallet(self, c0, K0):
 343         cK0 = ""
 344         self.master_public_keys = {
 345             "m/0'/": (c0, K0, cK0),
 346             }
 347         self.storage.put('master_public_keys', self.master_public_keys, True)
 348         self.create_account('1','Main account')
 349
 350
 351     def create_accounts(self):
 352         if self.seed_version == 4:
 353             self.create_old_account()
 354         else:
 355             # create default account
 356             self.create_master_keys('1')
 357             self.create_account('1','Main account')
 358
 359
 360     def create_master_keys(self, account_type):
 361         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 362         if account_type == '1':
 363             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 364             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 365             self.master_private_keys["m/0'/"] = k0
 366         elif account_type == '2of2':
 367             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 368             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 369             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 370             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 371             self.master_private_keys["m/1'/"] = k1
 372             self.master_private_keys["m/2'/"] = k2
 373         elif account_type == '2of3':
 374             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 375             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 376             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 377             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 378             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 379             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 380             self.master_private_keys["m/3'/"] = k3
 381             self.master_private_keys["m/4'/"] = k4
 382             self.master_private_keys["m/5'/"] = k5
 383
 384         self.storage.put('master_public_keys', self.master_public_keys, True)
 385         self.storage.put('master_private_keys', self.master_private_keys, True)
 386
 387     def has_master_public_keys(self, account_type):
 388         if account_type == '1':
 389             return "m/0'/" in self.master_public_keys
 390         elif account_type == '2of2':
 391             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 392         elif account_type == '2of3':
 393             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 394
 395     def find_root_by_master_key(self, c, K):
 396         for key, v in self.master_public_keys.items():
 397             if key == "m/":continue
 398             cc, KK, _ = v
 399             if (c == cc) and (K == KK):
 400                 return key
 401
 402     def deseed_root(self, seed, password):
 403         # for safety, we ask the user to enter their seed
 404         assert seed == self.get_seed(password)
 405         self.seed = ''
 406         self.storage.put('seed', '', True)
 407
 408
 409     def deseed_branch(self, k):
 410         # check that parent has no seed
 411         assert self.seed == ''
 412         self.master_private_keys.pop(k)
 413         self.storage.put('master_private_keys', self.master_private_keys, True)
 414
 415     def is_watching_only(self):
 416         return (self.seed == '') and (self.master_private_keys == {})
 417
 418
 419
 420     def account_id(self, account_type, i):
 421         if account_type == '1':
 422             return "m/0'/%d"%i
 423         elif account_type == '2of2':
 424             return "m/1'/%d & m/2'/%d"%(i,i)
 425         elif account_type == '2of3':
 426             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 427         else:
 428             raise BaseException('unknown account type')
 429
 430
 431     def num_accounts(self, account_type):
 432         keys = self.accounts.keys()
 433         i = 0
 434         while True:
 435             account_id = self.account_id(account_type, i)
 436             if account_id not in keys: break
 437             i += 1
 438         return i
 439
 440
 441     def new_account_address(self, account_type = '1'):
 442         i = self.num_accounts(account_type)
 443         k = self.account_id(account_type,i)
 444
 445         addr = self.next_addresses.get(k)
 446         if not addr:
 447             account_id, account = self.next_account(account_type)
 448             addr = account.first_address()
 449             self.next_addresses[k] = addr
 450             self.storage.put('next_addresses',self.next_addresses)
 451
 452         return k, addr
 453
 454
 455     def next_account(self, account_type = '1'):
 456
 457         i = self.num_accounts(account_type)
 458         account_id = self.account_id(account_type,i)
 459
 460         if account_type is '1':
 461             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 462             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 463             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 464
 465         elif account_type == '2of2':
 466             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 467             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 468             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 469             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 470             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 471
 472         elif account_type == '2of3':
 473             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 474             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 475             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 476             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 477             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 478             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 479             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 480
 481         return account_id, account
 482
 483
 484     def set_label(self, name, text = None):
 485         changed = False
 486         old_text = self.labels.get(name)
 487         if text:
 488             if old_text != text:
 489                 self.labels[name] = text
 490                 changed = True
 491         else:
 492             if old_text:
 493                 self.labels.pop(name)
 494                 changed = True
 495
 496         if changed:
 497             self.storage.put('labels', self.labels, True)
 498
 499         run_hook('set_label', name, text, changed)
 500         return changed
 501
 502
 503
 504     def create_account(self, account_type = '1', name = None):
 505         k, account = self.next_account(account_type)
 506         if k in self.pending_accounts:
 507             self.pending_accounts.pop(k)
 508             self.storage.put('pending_accounts', self.pending_accounts)
 509
 510         self.accounts[k] = account
 511         self.save_accounts()
 512         if name:
 513             self.set_label(k, name)
 514
 515
 516     def create_old_account(self):
 517         mpk = OldAccount.mpk_from_seed(self.seed)
 518         self.storage.put('master_public_key', mpk, True)
 519         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 520         self.save_accounts()
 521
 522
 523     def save_accounts(self):
 524         d = {}
 525         for k, v in self.accounts.items():
 526             d[k] = v.dump()
 527         self.storage.put('accounts', d, True)
 528
 529
 530
 531     def load_accounts(self):
 532         d = self.storage.get('accounts', {})
 533         self.accounts = {}
 534         for k, v in d.items():
 535             if k == 0:
 536                 v['mpk'] = self.storage.get('master_public_key')
 537                 self.accounts[k] = OldAccount(v)
 538             elif '&' in k:
 539                 self.accounts[k] = BIP32_Account_2of2(v)
 540             else:
 541                 self.accounts[k] = BIP32_Account(v)
 542
 543         self.pending_accounts = self.storage.get('pending_accounts',{})
 544
 545
 546     def delete_pending_account(self, k):
 547         self.pending_accounts.pop(k)
 548         self.storage.put('pending_accounts', self.pending_accounts)
 549
 550     def account_is_pending(self, k):
 551         return k in self.pending_accounts
 552
 553     def create_pending_account(self, acct_type, name):
 554         k, addr = self.new_account_address(acct_type)
 555         self.set_label(k, name)
 556         self.pending_accounts[k] = addr
 557         self.storage.put('pending_accounts', self.pending_accounts)
 558
 559     def get_pending_accounts(self):
 560         return self.pending_accounts.items()
 561
 562
 563     def addresses(self, include_change = True, _next=True):
 564         o = self.get_account_addresses(-1, include_change)
 565         for a in self.accounts.keys():
 566             o += self.get_account_addresses(a, include_change)
 567
 568         if _next:
 569             for addr in self.next_addresses.values():
 570                 if addr not in o:
 571                     o += [addr]
 572         return o
 573
 574
 575     def is_mine(self, address):
 576         return address in self.addresses(True)
 577
 578
 579     def is_change(self, address):
 580         if not self.is_mine(address): return False
 581         if address in self.imported_keys.keys(): return False
 582         acct, s = self.get_address_index(address)
 583         if s is None: return False
 584         return s[0] == 1
 585
 586     def get_master_public_key(self):
 587         if self.seed_version == 4:
 588             return self.storage.get("master_public_key")
 589         else:
 590             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 591             return repr((c, K))
 592
 593     def get_master_private_key(self, account, password):
 594         master_k = pw_decode( self.master_private_keys[account], password)
 595         master_c, master_K, master_Kc = self.master_public_keys[account]
 596         try:
 597             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 598             assert K.encode('hex') == master_K
 599         except:
 600             raise BaseException("Invalid password")
 601         return master_k
 602
 603
 604     def get_address_index(self, address):
 605         if address in self.imported_keys.keys():
 606             return -1, None
 607
 608         for account in self.accounts.keys():
 609             for for_change in [0,1]:
 610                 addresses = self.accounts[account].get_addresses(for_change)
 611                 for addr in addresses:
 612                     if address == addr:
 613                         return account, (for_change, addresses.index(addr))
 614
 615         for k,v in self.next_addresses.items():
 616             if v == address:
 617                 return k, (0,0)
 618
 619         raise BaseException("Address not found", address)
 620
 621
 622     def get_roots(self, account):
 623         roots = []
 624         for a in account.split('&'):
 625             s = a.strip()
 626             m = re.match("(m/\d+'/)(\d+)", s)
 627             roots.append( m.group(1) )
 628         return roots
 629
 630     def is_seeded(self, account):
 631         if type(account) is int:
 632             return self.seed is not None
 633
 634         for root in self.get_roots(account):
 635             if root not in self.master_private_keys.keys():
 636                 return False
 637         return True
 638
 639     def rebase_sequence(self, account, sequence):
 640         c, i = sequence
 641         dd = []
 642         for a in account.split('&'):
 643             s = a.strip()
 644             m = re.match("(m/\d+'/)(\d+)", s)
 645             root = m.group(1)
 646             num = int(m.group(2))
 647             dd.append( (root, [num,c,i] ) )
 648         return dd
 649
 650
 651     def get_keyID(self, account, sequence):
 652         if account == 0:
 653             a, b = sequence
 654             mpk = self.storage.get('master_public_key')
 655             return 'old(%s,%d,%d)'%(mpk,a,b)
 656
 657         rs = self.rebase_sequence(account, sequence)
 658         dd = []
 659         for root, public_sequence in rs:
 660             c, K, _ = self.master_public_keys[root]
 661             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 662             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 663         return '&'.join(dd)
 664
 665
 666
 667     def get_seed(self, password):
 668         s = pw_decode(self.seed, password)
 669         if self.seed_version == 4:
 670             seed = s
 671         else:
 672             seed = mnemonic_hash(s)
 673         #todo:  #self.sequences[0].check_seed(seed)
 674         return seed
 675
 676
 677     def get_mnemonic(self, password):
 678         import mnemonic
 679         s = pw_decode(self.seed, password)
 680         if self.seed_version == 4:
 681             return ' '.join(mnemonic.mn_encode(s))
 682         else:
 683             return s
 684
 685
 686
 687     def get_private_key(self, address, password):
 688         out = []
 689         if address in self.imported_keys.keys():
 690             out.append( pw_decode( self.imported_keys[address], password ) )
 691         else:
 692             account, sequence = self.get_address_index(address)
 693             if account == 0:
 694                 seed = self.get_seed(password)
 695                 pk = self.accounts[account].get_private_key(seed, sequence)
 696                 out.append(pk)
 697                 return out
 698
 699             # assert address == self.accounts[account].get_address(*sequence)
 700             rs = self.rebase_sequence( account, sequence)
 701             for root, public_sequence in rs:
 702
 703                 if root not in self.master_private_keys.keys(): continue
 704                 master_k = self.get_master_private_key(root, password)
 705                 master_c, _, _ = self.master_public_keys[root]
 706                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 707                 out.append(pk)
 708
 709         return out
 710
 711
 712     def add_keypairs_from_wallet(self, tx, keypairs, password):
 713         for txin in tx.inputs:
 714             address = txin['address']
 715             private_keys = self.get_private_key(address, password)
 716             for sec in private_keys:
 717                 pubkey = public_key_from_private_key(sec)
 718                 keypairs[ pubkey ] = sec
 719                 if address in self.imported_keys.keys():
 720                     txin['redeemPubkey'] = pubkey
 721
 722
 723     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 724         for txin in tx.inputs:
 725             keyid = txin.get('KeyID')
 726             if keyid:
 727
 728                 if self.seed_version==4:
 729                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 730                     if not m: continue
 731                     mpk = m.group(1)
 732                     if mpk != self.storage.get('master_public_key'): continue
 733                     index = int(m.group(2))
 734                     num = int(m.group(3))
 735                     account = self.accounts[0]
 736                     addr = account.get_address(index, num)
 737                     txin['address'] = addr # fixme: side effect
 738                     pk = self.get_private_key(addr, password)
 739                     for sec in pk:
 740                         pubkey = public_key_from_private_key(sec)
 741                         keypairs[pubkey] = sec
 742                     continue
 743
 744
 745                 roots = []
 746                 for s in keyid.split('&'):
 747                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 748                     if not m: continue
 749                     c = m.group(1)
 750                     K = m.group(2)
 751                     sequence = m.group(3)
 752                     root = self.find_root_by_master_key(c,K)
 753                     if not root: continue
 754                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 755                     root = root + '%d'%sequence[0]
 756                     sequence = sequence[1:]
 757                     roots.append((root,sequence))
 758
 759                 account_id = " & ".join( map(lambda x:x[0], roots) )
 760                 account = self.accounts.get(account_id)
 761                 if not account: continue
 762                 addr = account.get_address(*sequence)
 763                 txin['address'] = addr # fixme: side effect
 764                 pk = self.get_private_key(addr, password)
 765                 for sec in pk:
 766                     pubkey = public_key_from_private_key(sec)
 767                     keypairs[pubkey] = sec
 768
 769
 770
 771     def signrawtransaction(self, tx, input_info, private_keys, password):
 772
 773         # check that the password is correct
 774         seed = self.get_seed(password)
 775
 776         # add input info
 777         tx.add_input_info(input_info)
 778
 779         # add redeem script for coins that are in the wallet
 780         # FIXME: add redeemPubkey too!
 781         unspent_coins = self.get_unspent_coins()
 782         for txin in tx.inputs:
 783             for item in unspent_coins:
 784                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 785                     print_error( "tx input is in unspent coins" )
 786                     txin['scriptPubKey'] = item['scriptPubKey']
 787                     account, sequence = self.get_address_index(item['address'])
 788                     if account != -1:
 789                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 790                         print_error("added redeemScript", txin['redeemScript'])
 791                     break
 792
 793
 794         # build a list of public/private keys
 795         keypairs = {}
 796
 797         # add private keys from parameter
 798         for sec in private_keys:
 799             pubkey = public_key_from_private_key(sec)
 800             keypairs[ pubkey ] = sec
 801
 802         # add private_keys from KeyID
 803         self.add_keypairs_from_KeyID(tx, keypairs, password)
 804
 805         # add private keys from wallet
 806         self.add_keypairs_from_wallet(tx, keypairs, password)
 807         self.sign_transaction(tx, keypairs)
 808
 809
 810     def sign_message(self, address, message, password):
 811         keys = self.get_private_key(address, password)
 812         assert len(keys) == 1
 813         sec = keys[0]
 814         key = regenerate_key(sec)
 815         compressed = is_compressed(sec)
 816         return key.sign_message(message, compressed, address)
 817
 818
 819     def change_gap_limit(self, value):
 820         if value >= self.gap_limit:
 821             self.gap_limit = value
 822             self.storage.put('gap_limit', self.gap_limit, True)
 823             #self.interface.poke('synchronizer')
 824             return True
 825
 826         elif value >= self.min_acceptable_gap():
 827             for key, account in self.accounts.items():
 828                 addresses = account[0]
 829                 k = self.num_unused_trailing_addresses(addresses)
 830                 n = len(addresses) - k + value
 831                 addresses = addresses[0:n]
 832                 self.accounts[key][0] = addresses
 833
 834             self.gap_limit = value
 835             self.storage.put('gap_limit', self.gap_limit, True)
 836             self.save_accounts()
 837             return True
 838         else:
 839             return False
 840
 841     def num_unused_trailing_addresses(self, addresses):
 842         k = 0
 843         for a in addresses[::-1]:
 844             if self.history.get(a):break
 845             k = k + 1
 846         return k
 847
 848     def min_acceptable_gap(self):
 849         # fixme: this assumes wallet is synchronized
 850         n = 0
 851         nmax = 0
 852
 853         for account in self.accounts.values():
 854             addresses = account.get_addresses(0)
 855             k = self.num_unused_trailing_addresses(addresses)
 856             for a in addresses[0:-k]:
 857                 if self.history.get(a):
 858                     n = 0
 859                 else:
 860                     n += 1
 861                     if n > nmax: nmax = n
 862         return nmax + 1
 863
 864
 865     def address_is_old(self, address):
 866         age = -1
 867         h = self.history.get(address, [])
 868         if h == ['*']:
 869             return True
 870         for tx_hash, tx_height in h:
 871             if tx_height == 0:
 872                 tx_age = 0
 873             else:
 874                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 875             if tx_age > age:
 876                 age = tx_age
 877         return age > 2
 878
 879
 880     def synchronize_sequence(self, account, for_change):
 881         limit = self.gap_limit_for_change if for_change else self.gap_limit
 882         new_addresses = []
 883         while True:
 884             addresses = account.get_addresses(for_change)
 885             if len(addresses) < limit:
 886                 address = account.create_new_address(for_change)
 887                 self.history[address] = []
 888                 new_addresses.append( address )
 889                 continue
 890
 891             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 892                 break
 893             else:
 894                 address = account.create_new_address(for_change)
 895                 self.history[address] = []
 896                 new_addresses.append( address )
 897
 898         return new_addresses
 899
 900
 901
 902     def create_pending_accounts(self):
 903         for account_type in ['1','2of2','2of3']:
 904             if not self.has_master_public_keys(account_type):
 905                 continue
 906             k, a = self.new_account_address(account_type)
 907             if self.address_is_old(a):
 908                 print_error( "creating account", a )
 909                 self.create_account(account_type)
 910                 self.next_addresses.pop(k)
 911
 912
 913     def synchronize_account(self, account):
 914         new = []
 915         new += self.synchronize_sequence(account, 0)
 916         new += self.synchronize_sequence(account, 1)
 917         return new
 918
 919
 920     def synchronize(self):
 921         if self.master_public_keys:
 922             self.create_pending_accounts()
 923         new = []
 924         for account in self.accounts.values():
 925             new += self.synchronize_account(account)
 926         if new:
 927             self.save_accounts()
 928             self.storage.put('addr_history', self.history, True)
 929         return new
 930
 931
 932     def is_found(self):
 933         return self.history.values() != [[]] * len(self.history)
 934
 935
 936     def add_contact(self, address, label=None):
 937         self.addressbook.append(address)
 938         self.storage.put('contacts', self.addressbook, True)
 939         if label:
 940             self.set_label(address, label)
 941
 942
 943     def delete_contact(self, addr):
 944         if addr in self.addressbook:
 945             self.addressbook.remove(addr)
 946             self.storage.put('addressbook', self.addressbook, True)
 947
 948
 949     def fill_addressbook(self):
 950         for tx_hash, tx in self.transactions.items():
 951             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 952             if is_send:
 953                 for addr, v in tx.outputs:
 954                     if not self.is_mine(addr) and addr not in self.addressbook:
 955                         self.addressbook.append(addr)
 956         # redo labels
 957         # self.update_tx_labels()
 958
 959     def get_num_tx(self, address):
 960         n = 0
 961         for tx in self.transactions.values():
 962             if address in map(lambda x:x[0], tx.outputs): n += 1
 963         return n
 964
 965
 966     def get_address_flags(self, addr):
 967         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 968         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 969         return flags
 970
 971
 972     def get_tx_value(self, tx, account=None):
 973         domain = self.get_account_addresses(account)
 974         return tx.get_value(domain, self.prevout_values)
 975
 976
 977     def update_tx_outputs(self, tx_hash):
 978         tx = self.transactions.get(tx_hash)
 979
 980         for i, (addr, value) in enumerate(tx.outputs):
 981             key = tx_hash+ ':%d'%i
 982             self.prevout_values[key] = value
 983
 984         for item in tx.inputs:
 985             if self.is_mine(item.get('address')):
 986                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 987                 self.spent_outputs.append(key)
 988
 989
 990     def get_addr_balance(self, address):
 991         assert self.is_mine(address)
 992         h = self.history.get(address,[])
 993         if h == ['*']: return 0,0
 994         c = u = 0
 995         received_coins = []   # list of coins received at address
 996
 997         for tx_hash, tx_height in h:
 998             tx = self.transactions.get(tx_hash)
 999             if not tx: continue
1000
1001             for i, (addr, value) in enumerate(tx.outputs):
1002                 if addr == address:
1003                     key = tx_hash + ':%d'%i
1004                     received_coins.append(key)
1005
1006         for tx_hash, tx_height in h:
1007             tx = self.transactions.get(tx_hash)
1008             if not tx: continue
1009             v = 0
1010
1011             for item in tx.inputs:
1012                 addr = item.get('address')
1013                 if addr == address:
1014                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1015                     value = self.prevout_values.get( key )
1016                     if key in received_coins:
1017                         v -= value
1018
1019             for i, (addr, value) in enumerate(tx.outputs):
1020                 key = tx_hash + ':%d'%i
1021                 if addr == address:
1022                     v += value
1023
1024             if tx_height:
1025                 c += v
1026             else:
1027                 u += v
1028         return c, u
1029
1030
1031     def get_account_name(self, k):
1032         if k == 0:
1033             if self.seed_version == 4:
1034                 name = 'Main account'
1035             else:
1036                 name = 'Old account'
1037         else:
1038             default = "Unnamed account"
1039             m = re.match("m/0'/(\d+)", k)
1040             if m:
1041                 num = m.group(1)
1042                 if num == '0':
1043                     default = "Main account"
1044                 else:
1045                     default = "Account %s"%num
1046
1047             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1048             if m:
1049                 num = m.group(1)
1050                 default = "2of2 account %s"%num
1051             name = self.labels.get(k, default)
1052
1053         return name
1054
1055     def get_account_names(self):
1056         accounts = {}
1057         for k, account in self.accounts.items():
1058             accounts[k] = self.get_account_name(k)
1059         if self.imported_keys:
1060             accounts[-1] = 'Imported keys'
1061         return accounts
1062
1063     def get_account_addresses(self, a, include_change=True):
1064         if a is None:
1065             o = self.addresses(True)
1066         elif a == -1:
1067             o = self.imported_keys.keys()
1068         else:
1069             ac = self.accounts[a]
1070             o = ac.get_addresses(0)
1071             if include_change: o += ac.get_addresses(1)
1072         return o
1073
1074     def get_imported_balance(self):
1075         return self.get_balance(self.imported_keys.keys())
1076
1077     def get_account_balance(self, account):
1078         return self.get_balance(self.get_account_addresses(account))
1079
1080     def get_frozen_balance(self):
1081         return self.get_balance(self.frozen_addresses)
1082
1083     def get_balance(self, domain=None):
1084         if domain is None: domain = self.addresses(True)
1085         cc = uu = 0
1086         for addr in domain:
1087             c, u = self.get_addr_balance(addr)
1088             cc += c
1089             uu += u
1090         return cc, uu
1091
1092
1093     def get_unspent_coins(self, domain=None):
1094         coins = []
1095         if domain is None: domain = self.addresses(True)
1096         for addr in domain:
1097             h = self.history.get(addr, [])
1098             if h == ['*']: continue
1099             for tx_hash, tx_height in h:
1100                 tx = self.transactions.get(tx_hash)
1101                 if tx is None: raise BaseException("Wallet not synchronized")
1102                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1103                 for output in tx.d.get('outputs'):
1104                     if output.get('address') != addr: continue
1105                     key = tx_hash + ":%d" % output.get('prevout_n')
1106                     if key in self.spent_outputs: continue
1107                     output['prevout_hash'] = tx_hash
1108                     output['height'] = tx_height
1109                     output['coinbase'] = is_coinbase
1110                     coins.append((tx_height, output))
1111
1112         # sort by age
1113         if coins:
1114             coins = sorted(coins)
1115             if coins[-1][0] != 0:
1116                 while coins[0][0] == 0:
1117                     coins = coins[1:] + [ coins[0] ]
1118         return [x[1] for x in coins]
1119
1120
1121
1122     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1123         domain = self.get_account_addresses(account) if account else None
1124         return self.choose_tx_inputs( amount, fixed_fee, domain )
1125
1126
1127     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1128         """ todo: minimize tx size """
1129         total = 0
1130         fee = self.fee if fixed_fee is None else fixed_fee
1131         if domain is None:
1132             domain = self.addresses(True)
1133
1134         for i in self.frozen_addresses:
1135             if i in domain: domain.remove(i)
1136
1137         prioritized = []
1138         for i in self.prioritized_addresses:
1139             if i in domain:
1140                 domain.remove(i)
1141                 prioritized.append(i)
1142
1143         coins = self.get_unspent_coins(domain)
1144         prioritized_coins = self.get_unspent_coins(prioritized)
1145
1146         inputs = []
1147         coins = prioritized_coins + coins
1148
1149         for item in coins:
1150             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1151                 continue
1152             addr = item.get('address')
1153             v = item.get('value')
1154             total += v
1155             inputs.append(item)
1156             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1157             if total >= amount + fee: break
1158         else:
1159             inputs = []
1160
1161         return inputs, total, fee
1162
1163
1164     def set_fee(self, fee):
1165         if self.fee != fee:
1166             self.fee = fee
1167             self.storage.put('fee_per_kb', self.fee, True)
1168
1169     def estimated_fee(self, inputs):
1170         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1171         fee = self.fee * int(round(estimated_size/1024.))
1172         if fee == 0: fee = self.fee
1173         return fee
1174
1175
1176     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1177         "add change to a transaction"
1178         change_amount = total - ( amount + fee )
1179         if change_amount != 0:
1180             if not change_addr:
1181
1182                 # send change to one of the accounts involved in the tx
1183                 address = inputs[0].get('address')
1184                 account, _ = self.get_address_index(address)
1185
1186                 if not self.use_change or account == -1:
1187                     change_addr = inputs[-1]['address']
1188                 else:
1189                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1190
1191             # Insert the change output at a random position in the outputs
1192             posn = random.randint(0, len(outputs))
1193             outputs[posn:posn] = [( change_addr,  change_amount)]
1194         return outputs
1195
1196
1197     def get_history(self, address):
1198         with self.lock:
1199             return self.history.get(address)
1200
1201
1202     def get_status(self, h):
1203         if not h: return None
1204         if h == ['*']: return '*'
1205         status = ''
1206         for tx_hash, height in h:
1207             status += tx_hash + ':%d:' % height
1208         return hashlib.sha256( status ).digest().encode('hex')
1209
1210
1211     def receive_tx_callback(self, tx_hash, tx, tx_height):
1212
1213         with self.transaction_lock:
1214             self.add_extra_addresses(tx)
1215             if not self.check_new_tx(tx_hash, tx):
1216                 # may happen due to pruning
1217                 print_error("received transaction that is no longer referenced in history", tx_hash)
1218                 return
1219             self.transactions[tx_hash] = tx
1220             self.network.interface.pending_transactions_for_notifications.append(tx)
1221             self.save_transactions()
1222             if self.verifier and tx_height>0:
1223                 self.verifier.add(tx_hash, tx_height)
1224             self.update_tx_outputs(tx_hash)
1225
1226
1227     def save_transactions(self):
1228         tx = {}
1229         for k,v in self.transactions.items():
1230             tx[k] = str(v)
1231         self.storage.put('transactions', tx, True)
1232
1233     def receive_history_callback(self, addr, hist):
1234
1235         if not self.check_new_history(addr, hist):
1236             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1237
1238         with self.lock:
1239             self.history[addr] = hist
1240             self.storage.put('addr_history', self.history, True)
1241
1242         if hist != ['*']:
1243             for tx_hash, tx_height in hist:
1244                 if tx_height>0:
1245                     # add it in case it was previously unconfirmed
1246                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1247
1248
1249     def get_tx_history(self, account=None):
1250         with self.transaction_lock:
1251             history = self.transactions.items()
1252             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1253             result = []
1254
1255             balance = 0
1256             for tx_hash, tx in history:
1257                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1258                 if v is not None: balance += v
1259
1260             c, u = self.get_account_balance(account)
1261
1262             if balance != c+u:
1263                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1264
1265             balance = c + u - balance
1266             for tx_hash, tx in history:
1267                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1268                 if not is_relevant:
1269                     continue
1270                 if value is not None:
1271                     balance += value
1272
1273                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1274                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1275
1276         return result
1277
1278
1279     def get_label(self, tx_hash):
1280         label = self.labels.get(tx_hash)
1281         is_default = (label == '') or (label is None)
1282         if is_default: label = self.get_default_label(tx_hash)
1283         return label, is_default
1284
1285
1286     def get_default_label(self, tx_hash):
1287         tx = self.transactions.get(tx_hash)
1288         default_label = ''
1289         if tx:
1290             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1291             if is_mine:
1292                 for o in tx.outputs:
1293                     o_addr, _ = o
1294                     if not self.is_mine(o_addr):
1295                         try:
1296                             default_label = self.labels[o_addr]
1297                         except KeyError:
1298                             default_label = o_addr
1299                         break
1300                 else:
1301                     default_label = '(internal)'
1302             else:
1303                 for o in tx.outputs:
1304                     o_addr, _ = o
1305                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1306                         break
1307                 else:
1308                     for o in tx.outputs:
1309                         o_addr, _ = o
1310                         if self.is_mine(o_addr):
1311                             break
1312                     else:
1313                         o_addr = None
1314
1315                 if o_addr:
1316                     dest_label = self.labels.get(o_addr)
1317                     try:
1318                         default_label = self.labels[o_addr]
1319                     except KeyError:
1320                         default_label = o_addr
1321
1322         return default_label
1323
1324
1325     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1326         for address, x in outputs:
1327             assert is_valid(address)
1328         amount = sum( map(lambda x:x[1], outputs) )
1329         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1330         if not inputs:
1331             raise ValueError("Not enough funds")
1332         self.add_input_info(inputs)
1333         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1334         return Transaction.from_io(inputs, outputs)
1335
1336
1337     def mktx_from_account(self, outputs, password, fee=None, account=None):
1338         domain = self.get_account_addresses(account) if account else None
1339         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1340
1341
1342     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1343         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1344         keypairs = {}
1345         self.add_keypairs_from_wallet(tx, keypairs, password)
1346         if keypairs:
1347             self.sign_transaction(tx, keypairs)
1348         return tx
1349
1350
1351     def add_input_info(self, inputs):
1352         for txin in inputs:
1353             address = txin['address']
1354             if address in self.imported_keys.keys():
1355                 continue
1356             account, sequence = self.get_address_index(address)
1357             txin['KeyID'] = self.get_keyID(account, sequence)
1358             redeemScript = self.accounts[account].redeem_script(sequence)
1359             if redeemScript:
1360                 txin['redeemScript'] = redeemScript
1361             else:
1362                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1363
1364
1365     def sign_transaction(self, tx, keypairs):
1366         tx.sign(keypairs)
1367         run_hook('sign_transaction', tx)
1368
1369
1370     def sendtx(self, tx):
1371         # synchronous
1372         h = self.send_tx(tx)
1373         self.tx_event.wait()
1374         return self.receive_tx(h)
1375
1376     def send_tx(self, tx):
1377         # asynchronous
1378         self.tx_event.clear()
1379         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1380         return tx.hash()
1381
1382     def on_broadcast(self, i, r):
1383         self.tx_result = r.get('result')
1384         self.tx_event.set()
1385
1386     def receive_tx(self,tx_hash):
1387         out = self.tx_result
1388         if out != tx_hash:
1389             return False, "error: " + out
1390         return True, out
1391
1392
1393
1394     def update_password(self, old_password, new_password):
1395         if new_password == '': new_password = None
1396         # this will throw an exception if unicode cannot be converted
1397         decoded = pw_decode(self.seed, old_password)
1398         self.seed = pw_encode( decoded, new_password)
1399         self.storage.put('seed', self.seed, True)
1400         self.use_encryption = (new_password != None)
1401         self.storage.put('use_encryption', self.use_encryption,True)
1402         for k in self.imported_keys.keys():
1403             a = self.imported_keys[k]
1404             b = pw_decode(a, old_password)
1405             c = pw_encode(b, new_password)
1406             self.imported_keys[k] = c
1407         self.storage.put('imported_keys', self.imported_keys, True)
1408
1409         for k, v in self.master_private_keys.items():
1410             b = pw_decode(v, old_password)
1411             c = pw_encode(b, new_password)
1412             self.master_private_keys[k] = c
1413         self.storage.put('master_private_keys', self.master_private_keys, True)
1414
1415
1416     def freeze(self,addr):
1417         if self.is_mine(addr) and addr not in self.frozen_addresses:
1418             self.unprioritize(addr)
1419             self.frozen_addresses.append(addr)
1420             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1421             return True
1422         else:
1423             return False
1424
1425     def unfreeze(self,addr):
1426         if self.is_mine(addr) and addr in self.frozen_addresses:
1427             self.frozen_addresses.remove(addr)
1428             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1429             return True
1430         else:
1431             return False
1432
1433     def prioritize(self,addr):
1434         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1435             self.unfreeze(addr)
1436             self.prioritized_addresses.append(addr)
1437             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1438             return True
1439         else:
1440             return False
1441
1442     def unprioritize(self,addr):
1443         if self.is_mine(addr) and addr in self.prioritized_addresses:
1444             self.prioritized_addresses.remove(addr)
1445             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1446             return True
1447         else:
1448             return False
1449
1450
1451     def set_verifier(self, verifier):
1452         self.verifier = verifier
1453
1454         # review transactions that are in the history
1455         for addr, hist in self.history.items():
1456             if hist == ['*']: continue
1457             for tx_hash, tx_height in hist:
1458                 if tx_height>0:
1459                     # add it in case it was previously unconfirmed
1460                     self.verifier.add(tx_hash, tx_height)
1461
1462
1463         # if we are on a pruning server, remove unverified transactions
1464         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1465         for tx_hash in self.transactions.keys():
1466             if tx_hash not in vr:
1467                 self.transactions.pop(tx_hash)
1468
1469
1470
1471     def check_new_history(self, addr, hist):
1472
1473         # check that all tx in hist are relevant
1474         if hist != ['*']:
1475             for tx_hash, height in hist:
1476                 tx = self.transactions.get(tx_hash)
1477                 if not tx: continue
1478                 if not tx.has_address(addr):
1479                     return False
1480
1481         # check that we are not "orphaning" a transaction
1482         old_hist = self.history.get(addr,[])
1483         if old_hist == ['*']: return True
1484
1485         for tx_hash, height in old_hist:
1486             if tx_hash in map(lambda x:x[0], hist): continue
1487             found = False
1488             for _addr, _hist in self.history.items():
1489                 if _addr == addr: continue
1490                 if _hist == ['*']: continue
1491                 _tx_hist = map(lambda x:x[0], _hist)
1492                 if tx_hash in _tx_hist:
1493                     found = True
1494                     break
1495
1496             if not found:
1497                 tx = self.transactions.get(tx_hash)
1498                 # tx might not be there
1499                 if not tx: continue
1500
1501                 # already verified?
1502                 if self.verifier.get_height(tx_hash):
1503                     continue
1504                 # unconfirmed tx
1505                 print_error("new history is orphaning transaction:", tx_hash)
1506                 # check that all outputs are not mine, request histories
1507                 ext_requests = []
1508                 for _addr, _v in tx.outputs:
1509                     # assert not self.is_mine(_addr)
1510                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1511
1512                 ext_h = self.network.synchronous_get(ext_requests)
1513                 print_error("sync:", ext_requests, ext_h)
1514                 height = None
1515                 for h in ext_h:
1516                     if h == ['*']: continue
1517                     for item in h:
1518                         if item.get('tx_hash') == tx_hash:
1519                             height = item.get('height')
1520                 if height:
1521                     print_error("found height for", tx_hash, height)
1522                     self.verifier.add(tx_hash, height)
1523                 else:
1524                     print_error("removing orphaned tx from history", tx_hash)
1525                     self.transactions.pop(tx_hash)
1526
1527         return True
1528
1529
1530
1531     def check_new_tx(self, tx_hash, tx):
1532         # 1 check that tx is referenced in addr_history.
1533         addresses = []
1534         for addr, hist in self.history.items():
1535             if hist == ['*']:continue
1536             for txh, height in hist:
1537                 if txh == tx_hash:
1538                     addresses.append(addr)
1539
1540         if not addresses:
1541             return False
1542
1543         # 2 check that referencing addresses are in the tx
1544         for addr in addresses:
1545             if not tx.has_address(addr):
1546                 return False
1547
1548         return True
1549
1550
1551     def start_threads(self, network):
1552         from verifier import TxVerifier
1553         self.network = network
1554         self.verifier = TxVerifier(self.network, self.storage)
1555         self.verifier.start()
1556         self.set_verifier(self.verifier)
1557         self.synchronizer = WalletSynchronizer(self, network)
1558         self.synchronizer.start()
1559
1560     def stop_threads(self):
1561         self.verifier.stop()
1562         self.synchronizer.stop()
1563
1564
1565
1566     def restore(self, callback):
1567         from i18n import _
1568         def wait_for_wallet():
1569             self.set_up_to_date(False)
1570             while not self.is_up_to_date():
1571                 msg = "%s\n%s %d\n%s %.1f"%(
1572                     _("Please wait..."),
1573                     _("Addresses generated:"),
1574                     len(self.addresses(True)),_("Kilobytes received:"),
1575                     self.network.interface.bytes_received/1024.)
1576
1577                 apply(callback, (msg,))
1578                 time.sleep(0.1)
1579
1580         def wait_for_network():
1581             while not self.network.is_connected():
1582                 msg = "%s \n" % (_("Connecting..."))
1583                 apply(callback, (msg,))
1584                 time.sleep(0.1)
1585
1586         # wait until we are connected, because the user might have selected another server
1587         wait_for_network()
1588
1589
1590         self.create_accounts()
1591         wait_for_wallet()
1592
1593
1594
1595
1596 class WalletSynchronizer(threading.Thread):
1597
1598
1599     def __init__(self, wallet, network):
1600         threading.Thread.__init__(self)
1601         self.daemon = True
1602         self.wallet = wallet
1603         self.network = network
1604         self.was_updated = True
1605         self.running = False
1606         self.lock = threading.Lock()
1607         self.queue = Queue.Queue()
1608
1609     def stop(self):
1610         with self.lock: self.running = False
1611
1612     def is_running(self):
1613         with self.lock: return self.running
1614
1615
1616     def subscribe_to_addresses(self, addresses):
1617         messages = []
1618         for addr in addresses:
1619             messages.append(('blockchain.address.subscribe', [addr]))
1620         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1621
1622
1623     def run(self):
1624         with self.lock:
1625             self.running = True
1626
1627         while self.is_running():
1628
1629             if not self.network.is_connected():
1630                 print_error("synchronizer: waiting for interface")
1631                 self.network.wait_until_connected()
1632
1633             self.run_interface(self.network.interface)
1634
1635
1636     def run_interface(self, interface):
1637
1638         print_error("synchronizer: connected to", interface.server)
1639
1640         requested_tx = []
1641         missing_tx = []
1642         requested_histories = {}
1643
1644         # request any missing transactions
1645         for history in self.wallet.history.values():
1646             if history == ['*']: continue
1647             for tx_hash, tx_height in history:
1648                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1649                     missing_tx.append( (tx_hash, tx_height) )
1650
1651         if missing_tx:
1652             print_error("missing tx", missing_tx)
1653
1654         # subscriptions
1655         self.subscribe_to_addresses(self.wallet.addresses(True))
1656
1657         while self.is_running():
1658             # 1. create new addresses
1659             new_addresses = self.wallet.synchronize()
1660
1661             # request missing addresses
1662             if new_addresses:
1663                 self.subscribe_to_addresses(new_addresses)
1664
1665             # request missing transactions
1666             for tx_hash, tx_height in missing_tx:
1667                 if (tx_hash, tx_height) not in requested_tx:
1668                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1669                     requested_tx.append( (tx_hash, tx_height) )
1670             missing_tx = []
1671
1672             # detect if situation has changed
1673             if interface.is_up_to_date() and self.queue.empty():
1674                 if not self.wallet.is_up_to_date():
1675                     self.wallet.set_up_to_date(True)
1676                     self.was_updated = True
1677             else:
1678                 if self.wallet.is_up_to_date():
1679                     self.wallet.set_up_to_date(False)
1680                     self.was_updated = True
1681
1682             if self.was_updated:
1683                 self.wallet.network.trigger_callback('updated')
1684                 self.was_updated = False
1685
1686             # 2. get a response
1687             try:
1688                 r = self.queue.get(block=True, timeout=1)
1689             except Queue.Empty:
1690                 continue
1691
1692             if interface != self.network.interface:
1693                 break
1694
1695             if not r:
1696                 continue
1697
1698             # 3. handle response
1699             method = r['method']
1700             params = r['params']
1701             result = r.get('result')
1702             error = r.get('error')
1703             if error:
1704                 print "error", r
1705                 continue
1706
1707             if method == 'blockchain.address.subscribe':
1708                 addr = params[0]
1709                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1710                     if requested_histories.get(addr) is None:
1711                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1712                         requested_histories[addr] = result
1713
1714             elif method == 'blockchain.address.get_history':
1715                 addr = params[0]
1716                 print_error("receiving history", addr, result)
1717                 if result == ['*']:
1718                     assert requested_histories.pop(addr) == '*'
1719                     self.wallet.receive_history_callback(addr, result)
1720                 else:
1721                     hist = []
1722                     # check that txids are unique
1723                     txids = []
1724                     for item in result:
1725                         tx_hash = item['tx_hash']
1726                         if tx_hash not in txids:
1727                             txids.append(tx_hash)
1728                             hist.append( (tx_hash, item['height']) )
1729
1730                     if len(hist) != len(result):
1731                         raise BaseException("error: server sent history with non-unique txid", result)
1732
1733                     # check that the status corresponds to what was announced
1734                     rs = requested_histories.pop(addr)
1735                     if self.wallet.get_status(hist) != rs:
1736                         raise BaseException("error: status mismatch: %s"%addr)
1737
1738                     # store received history
1739                     self.wallet.receive_history_callback(addr, hist)
1740
1741                     # request transactions that we don't have
1742                     for tx_hash, tx_height in hist:
1743                         if self.wallet.transactions.get(tx_hash) is None:
1744                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1745                                 missing_tx.append( (tx_hash, tx_height) )
1746
1747             elif method == 'blockchain.transaction.get':
1748                 tx_hash = params[0]
1749                 tx_height = params[1]
1750                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1751                 tx = Transaction(result)
1752                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1753                 self.was_updated = True
1754                 requested_tx.remove( (tx_hash, tx_height) )
1755                 print_error("received tx:", tx_hash, len(tx.raw))
1756
1757             else:
1758                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1759
1760             if self.was_updated and not requested_tx:
1761                 self.wallet.network.trigger_callback('updated')
1762                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1763
1764                 self.was_updated = False