lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40
  41 # AES encryption
  42 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  43 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  44
  45 def pw_encode(s, password):
  46     if password:
  47         secret = Hash(password)
  48         return EncodeAES(secret, s)
  49     else:
  50         return s
  51
  52 def pw_decode(s, password):
  53     if password is not None:
  54         secret = Hash(password)
  55         try:
  56             d = DecodeAES(secret, s)
  57         except:
  58             raise BaseException('Invalid password')
  59         return d
  60     else:
  61         return s
  62
  63
  64
  65
  66
  67 from version import *
  68
  69
  70 class WalletStorage:
  71
  72     def __init__(self, config):
  73         self.lock = threading.Lock()
  74         self.data = {}
  75         self.file_exists = False
  76         self.path = self.init_path(config)
  77         print_error( "wallet path", self.path )
  78         if self.path:
  79             self.read(self.path)
  80
  81
  82     def init_path(self, config):
  83         """Set the path of the wallet."""
  84
  85         # command line -w option
  86         path = config.get('wallet_path')
  87         if path:
  88             return path
  89
  90         # path in config file
  91         path = config.get('default_wallet_path')
  92         if path:
  93             return path
  94
  95         # default path
  96         dirpath = os.path.join(config.path, "wallets")
  97         if not os.path.exists(dirpath):
  98             os.mkdir(dirpath)
  99
 100         new_path = os.path.join(config.path, "wallets", "default_wallet")
 101
 102         # default path in pre 1.9 versions
 103         old_path = os.path.join(config.path, "electrum.dat")
 104         if os.path.exists(old_path) and not os.path.exists(new_path):
 105             os.rename(old_path, new_path)
 106
 107         return new_path
 108
 109
 110     def read(self, path):
 111         """Read the contents of the wallet file."""
 112         try:
 113             with open(self.path, "r") as f:
 114                 data = f.read()
 115         except IOError:
 116             return
 117         try:
 118             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 119         except:
 120             raise IOError("Cannot read wallet file.")
 121
 122         self.data = d
 123         self.file_exists = True
 124
 125
 126     def get(self, key, default=None):
 127         return self.data.get(key, default)
 128
 129     def put(self, key, value, save = True):
 130
 131         with self.lock:
 132             if value is not None:
 133                 self.data[key] = value
 134             else:
 135                 self.data.pop[key]
 136             if save:
 137                 self.write()
 138
 139     def write(self):
 140         s = repr(self.data)
 141         f = open(self.path,"w")
 142         f.write( s )
 143         f.close()
 144         if 'ANDROID_DATA' not in os.environ:
 145             import stat
 146             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 147
 148
 149 class Wallet:
 150
 151     def __init__(self, storage):
 152
 153         self.storage = storage
 154         self.electrum_version = ELECTRUM_VERSION
 155         self.gap_limit_for_change = 3 # constant
 156
 157         # saved fields
 158         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 159
 160         self.gap_limit             = storage.get('gap_limit', 5)
 161         self.use_change            = storage.get('use_change',True)
 162         self.use_encryption        = storage.get('use_encryption', False)
 163         self.seed                  = storage.get('seed', '')               # encrypted
 164         self.labels                = storage.get('labels', {})
 165         self.frozen_addresses      = storage.get('frozen_addresses',[])
 166         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 167         self.addressbook           = storage.get('contacts', [])
 168
 169         self.imported_keys         = storage.get('imported_keys',{})
 170         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 171
 172         self.fee                   = int(storage.get('fee_per_kb',20000))
 173
 174         self.master_public_keys = storage.get('master_public_keys',{})
 175         self.master_private_keys = storage.get('master_private_keys', {})
 176
 177         self.next_addresses = storage.get('next_addresses',{})
 178
 179         if self.seed_version not in [4, 6]:
 180             msg = "This wallet seed is not supported."
 181             if self.seed_version in [5]:
 182                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 183                 print msg
 184                 sys.exit(1)
 185
 186
 187         self.load_accounts()
 188
 189         self.transactions = {}
 190         tx_list = self.storage.get('transactions',{})
 191         for k,v in tx_list.items():
 192             try:
 193                 tx = Transaction(v)
 194             except:
 195                 print_msg("Warning: Cannot deserialize transactions. skipping")
 196                 continue
 197
 198             self.add_extra_addresses(tx)
 199             self.transactions[k] = tx
 200
 201         for h,tx in self.transactions.items():
 202             if not self.check_new_tx(h, tx):
 203                 print_error("removing unreferenced tx", h)
 204                 self.transactions.pop(h)
 205
 206
 207         # not saved
 208         self.prevout_values = {}     # my own transaction outputs
 209         self.spent_outputs = []
 210
 211         # spv
 212         self.verifier = None
 213
 214         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 215         # interface.is_up_to_date() returns true when all requests have been answered and processed
 216         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 217
 218         self.up_to_date = False
 219         self.lock = threading.Lock()
 220         self.transaction_lock = threading.Lock()
 221         self.tx_event = threading.Event()
 222
 223         for tx_hash, tx in self.transactions.items():
 224             self.update_tx_outputs(tx_hash)
 225
 226
 227     def add_extra_addresses(self, tx):
 228         h = tx.hash()
 229         # find the address corresponding to pay-to-pubkey inputs
 230         tx.add_extra_addresses(self.transactions)
 231         for o in tx.d.get('outputs'):
 232             if o.get('is_pubkey'):
 233                 for tx2 in self.transactions.values():
 234                     tx2.add_extra_addresses({h:tx})
 235
 236
 237
 238
 239     def set_up_to_date(self,b):
 240         with self.lock: self.up_to_date = b
 241
 242
 243     def is_up_to_date(self):
 244         with self.lock: return self.up_to_date
 245
 246
 247     def update(self):
 248         self.up_to_date = False
 249         while not self.is_up_to_date():
 250             time.sleep(0.1)
 251
 252
 253     def import_key(self, sec, password):
 254         # check password
 255         seed = self.get_seed(password)
 256         try:
 257             address = address_from_private_key(sec)
 258         except:
 259             raise BaseException('Invalid private key')
 260
 261         if self.is_mine(address):
 262             raise BaseException('Address already in wallet')
 263
 264         # store the originally requested keypair into the imported keys table
 265         self.imported_keys[address] = pw_encode(sec, password )
 266         self.storage.put('imported_keys', self.imported_keys, True)
 267         if self.synchronizer:
 268             self.synchronizer.subscribe_to_addresses([address])
 269         return address
 270
 271     def delete_imported_key(self, addr):
 272         if addr in self.imported_keys:
 273             self.imported_keys.pop(addr)
 274             self.storage.put('imported_keys', self.imported_keys, True)
 275
 276
 277     def make_seed(self):
 278         import mnemonic, ecdsa
 279         entropy = ecdsa.util.randrange( pow(2,160) )
 280         nonce = 0
 281         while True:
 282             ss = "%040x"%(entropy+nonce)
 283             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 284             # we keep only 13 words, that's approximately 139 bits of entropy
 285             words = mnemonic.mn_encode(s)[0:13]
 286             seed = ' '.join(words)
 287             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 288                 break  # this removes 12 bits of entropy
 289             nonce += 1
 290
 291         return seed
 292
 293
 294     def init_seed(self, seed):
 295         import mnemonic
 296
 297         if self.seed:
 298             raise BaseException("a seed exists")
 299
 300         if not seed:
 301             self.seed = random_seed(128)
 302             self.seed_version = 4
 303             return
 304
 305         #if not seed:
 306         #    self.seed = self.make_seed()
 307         #    self.seed_version = SEED_VERSION
 308         #    return
 309
 310         # find out what kind of wallet we are
 311         try:
 312             seed.decode('hex')
 313             self.seed_version = 4
 314             self.seed = str(seed)
 315             return
 316         except:
 317             pass
 318
 319         words = seed.split()
 320         self.seed_version = 4
 321         self.seed = mnemonic.mn_decode(words)
 322
 323         #try:
 324         #    mnemonic.mn_decode(words)
 325         #    uses_electrum_words = True
 326         #except:
 327         #    uses_electrum_words = False
 328         #
 329         #if uses_electrum_words and len(words) != 13:
 330         #    self.seed_version = 4
 331         #    self.seed = mnemonic.mn_decode(words)
 332         #else:
 333         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 334         #    self.seed_version = SEED_VERSION
 335         #    self.seed = seed
 336
 337
 338     def save_seed(self):
 339         self.storage.put('seed', self.seed, True)
 340         self.storage.put('seed_version', self.seed_version, True)
 341
 342     def create_watching_only_wallet(self, c0, K0):
 343         cK0 = ""
 344         self.master_public_keys = {
 345             "m/0'/": (c0, K0, cK0),
 346             }
 347         self.storage.put('master_public_keys', self.master_public_keys, True)
 348         self.create_account('1','Main account')
 349
 350
 351     def create_accounts(self):
 352         if self.seed_version == 4:
 353             self.create_old_account()
 354         else:
 355             # create default account
 356             self.create_master_keys('1')
 357             self.create_account('1','Main account')
 358
 359
 360     def create_master_keys(self, account_type):
 361         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 362         if account_type == '1':
 363             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 364             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 365             self.master_private_keys["m/0'/"] = k0
 366         elif account_type == '2of2':
 367             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 368             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 369             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 370             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 371             self.master_private_keys["m/1'/"] = k1
 372             self.master_private_keys["m/2'/"] = k2
 373         elif account_type == '2of3':
 374             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 375             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 376             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 377             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 378             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 379             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 380             self.master_private_keys["m/3'/"] = k3
 381             self.master_private_keys["m/4'/"] = k4
 382             self.master_private_keys["m/5'/"] = k5
 383
 384         self.storage.put('master_public_keys', self.master_public_keys, True)
 385         self.storage.put('master_private_keys', self.master_private_keys, True)
 386
 387     def has_master_public_keys(self, account_type):
 388         if account_type == '1':
 389             return "m/0'/" in self.master_public_keys
 390         elif account_type == '2of2':
 391             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 392         elif account_type == '2of3':
 393             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 394
 395     def find_root_by_master_key(self, c, K):
 396         for key, v in self.master_public_keys.items():
 397             if key == "m/":continue
 398             cc, KK, _ = v
 399             if (c == cc) and (K == KK):
 400                 return key
 401
 402     def deseed_root(self, seed, password):
 403         # for safety, we ask the user to enter their seed
 404         assert seed == self.get_seed(password)
 405         self.seed = ''
 406         self.storage.put('seed', '', True)
 407
 408
 409     def deseed_branch(self, k):
 410         # check that parent has no seed
 411         assert self.seed == ''
 412         self.master_private_keys.pop(k)
 413         self.storage.put('master_private_keys', self.master_private_keys, True)
 414
 415     def is_watching_only(self):
 416         return (self.seed == '') and (self.master_private_keys == {})
 417
 418
 419
 420     def account_id(self, account_type, i):
 421         if account_type == '1':
 422             return "m/0'/%d"%i
 423         elif account_type == '2of2':
 424             return "m/1'/%d & m/2'/%d"%(i,i)
 425         elif account_type == '2of3':
 426             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 427         else:
 428             raise BaseException('unknown account type')
 429
 430
 431     def num_accounts(self, account_type):
 432         keys = self.accounts.keys()
 433         i = 0
 434         while True:
 435             account_id = self.account_id(account_type, i)
 436             if account_id not in keys: break
 437             i += 1
 438         return i
 439
 440
 441     def new_account_address(self, account_type = '1'):
 442         i = self.num_accounts(account_type)
 443         k = self.account_id(account_type,i)
 444
 445         addr = self.next_addresses.get(k)
 446         if not addr:
 447             account_id, account = self.next_account(account_type)
 448             addr = account.first_address()
 449             self.next_addresses[k] = addr
 450             self.storage.put('next_addresses',self.next_addresses)
 451
 452         return k, addr
 453
 454
 455     def next_account(self, account_type = '1'):
 456
 457         i = self.num_accounts(account_type)
 458         account_id = self.account_id(account_type,i)
 459
 460         if account_type is '1':
 461             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 462             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 463             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 464
 465         elif account_type == '2of2':
 466             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 467             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 468             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 469             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 470             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 471
 472         elif account_type == '2of3':
 473             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 474             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 475             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 476             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 477             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 478             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 479             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 480
 481         return account_id, account
 482
 483
 484     def set_label(self, name, text = None):
 485         changed = False
 486         old_text = self.labels.get(name)
 487         if text:
 488             if old_text != text:
 489                 self.labels[name] = text
 490                 changed = True
 491         else:
 492             if old_text:
 493                 self.labels.pop(name)
 494                 changed = True
 495
 496         if changed:
 497             self.storage.put('labels', self.labels, True)
 498
 499         run_hook('set_label', name, text, changed)
 500         return changed
 501
 502
 503
 504     def create_account(self, account_type = '1', name = None):
 505         k, account = self.next_account(account_type)
 506         if k in self.pending_accounts:
 507             self.pending_accounts.pop(k)
 508             self.storage.put('pending_accounts', self.pending_accounts)
 509
 510         self.accounts[k] = account
 511         self.save_accounts()
 512         if name:
 513             self.set_label(k, name)
 514
 515
 516     def create_old_account(self):
 517         mpk = OldAccount.mpk_from_seed(self.seed)
 518         self.storage.put('master_public_key', mpk, True)
 519         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 520         self.save_accounts()
 521
 522
 523     def save_accounts(self):
 524         d = {}
 525         for k, v in self.accounts.items():
 526             d[k] = v.dump()
 527         self.storage.put('accounts', d, True)
 528
 529
 530
 531     def load_accounts(self):
 532         d = self.storage.get('accounts', {})
 533         self.accounts = {}
 534         for k, v in d.items():
 535             if k == 0:
 536                 v['mpk'] = self.storage.get('master_public_key')
 537                 self.accounts[k] = OldAccount(v)
 538             elif '&' in k:
 539                 self.accounts[k] = BIP32_Account_2of2(v)
 540             else:
 541                 self.accounts[k] = BIP32_Account(v)
 542
 543         self.pending_accounts = self.storage.get('pending_accounts',{})
 544
 545
 546     def delete_pending_account(self, k):
 547         self.pending_accounts.pop(k)
 548         self.storage.put('pending_accounts', self.pending_accounts)
 549
 550     def account_is_pending(self, k):
 551         return k in self.pending_accounts
 552
 553     def create_pending_account(self, acct_type, name):
 554         k, addr = self.new_account_address(acct_type)
 555         self.set_label(k, name)
 556         self.pending_accounts[k] = addr
 557         self.storage.put('pending_accounts', self.pending_accounts)
 558
 559     def get_pending_accounts(self):
 560         return self.pending_accounts.items()
 561
 562
 563     def addresses(self, include_change = True, _next=True):
 564         o = self.get_account_addresses(-1, include_change)
 565         for a in self.accounts.keys():
 566             o += self.get_account_addresses(a, include_change)
 567
 568         if _next:
 569             for addr in self.next_addresses.values():
 570                 if addr not in o:
 571                     o += [addr]
 572         return o
 573
 574
 575     def is_mine(self, address):
 576         return address in self.addresses(True)
 577
 578
 579     def is_change(self, address):
 580         if not self.is_mine(address): return False
 581         if address in self.imported_keys.keys(): return False
 582         acct, s = self.get_address_index(address)
 583         if s is None: return False
 584         return s[0] == 1
 585
 586     def get_master_public_key(self):
 587         if self.seed_version == 4:
 588             return self.storage.get("master_public_key")
 589         else:
 590             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 591             return repr((c, K))
 592
 593     def get_master_private_key(self, account, password):
 594         master_k = pw_decode( self.master_private_keys[account], password)
 595         master_c, master_K, master_Kc = self.master_public_keys[account]
 596         try:
 597             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 598             assert K.encode('hex') == master_K
 599         except:
 600             raise BaseException("Invalid password")
 601         return master_k
 602
 603
 604     def get_address_index(self, address):
 605         if address in self.imported_keys.keys():
 606             return -1, None
 607
 608         for account in self.accounts.keys():
 609             for for_change in [0,1]:
 610                 addresses = self.accounts[account].get_addresses(for_change)
 611                 for addr in addresses:
 612                     if address == addr:
 613                         return account, (for_change, addresses.index(addr))
 614
 615         for k,v in self.next_addresses.items():
 616             if v == address:
 617                 return k, (0,0)
 618
 619         raise BaseException("Address not found", address)
 620
 621
 622     def get_roots(self, account):
 623         roots = []
 624         for a in account.split('&'):
 625             s = a.strip()
 626             m = re.match("(m/\d+'/)(\d+)", s)
 627             roots.append( m.group(1) )
 628         return roots
 629
 630     def is_seeded(self, account):
 631         if type(account) is int:
 632             return self.seed is not None
 633
 634         for root in self.get_roots(account):
 635             if root not in self.master_private_keys.keys():
 636                 return False
 637         return True
 638
 639     def rebase_sequence(self, account, sequence):
 640         c, i = sequence
 641         dd = []
 642         for a in account.split('&'):
 643             s = a.strip()
 644             m = re.match("(m/\d+'/)(\d+)", s)
 645             root = m.group(1)
 646             num = int(m.group(2))
 647             dd.append( (root, [num,c,i] ) )
 648         return dd
 649
 650
 651     def get_keyID(self, account, sequence):
 652         if account == 0:
 653             a, b = sequence
 654             mpk = self.storage.get('master_public_key')
 655             return 'old(%s,%d,%d)'%(mpk,a,b)
 656
 657         rs = self.rebase_sequence(account, sequence)
 658         dd = []
 659         for root, public_sequence in rs:
 660             c, K, _ = self.master_public_keys[root]
 661             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 662             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 663         return '&'.join(dd)
 664
 665
 666
 667     def get_seed(self, password):
 668         s = pw_decode(self.seed, password)
 669         if self.seed_version == 4:
 670             seed = s
 671         else:
 672             seed = mnemonic_hash(s)
 673         #todo:  #self.sequences[0].check_seed(seed)
 674         return seed
 675
 676
 677     def get_mnemonic(self, password):
 678         import mnemonic
 679         s = pw_decode(self.seed, password)
 680         if self.seed_version == 4:
 681             return ' '.join(mnemonic.mn_encode(s))
 682         else:
 683             return s
 684
 685
 686
 687     def get_private_key(self, address, password):
 688         out = []
 689         if address in self.imported_keys.keys():
 690             out.append( pw_decode( self.imported_keys[address], password ) )
 691         else:
 692             account, sequence = self.get_address_index(address)
 693             if account == 0:
 694                 seed = self.get_seed(password)
 695                 pk = self.accounts[account].get_private_key(seed, sequence)
 696                 out.append(pk)
 697                 return out
 698
 699             # assert address == self.accounts[account].get_address(*sequence)
 700             rs = self.rebase_sequence( account, sequence)
 701             for root, public_sequence in rs:
 702
 703                 if root not in self.master_private_keys.keys(): continue
 704                 master_k = self.get_master_private_key(root, password)
 705                 master_c, _, _ = self.master_public_keys[root]
 706                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 707                 out.append(pk)
 708
 709         return out
 710
 711
 712     def add_keypairs_from_wallet(self, tx, keypairs, password):
 713         for txin in tx.inputs:
 714             address = txin['address']
 715             private_keys = self.get_private_key(address, password)
 716             for sec in private_keys:
 717                 pubkey = public_key_from_private_key(sec)
 718                 keypairs[ pubkey ] = sec
 719
 720
 721     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 722         for txin in tx.inputs:
 723             keyid = txin.get('KeyID')
 724             if keyid:
 725
 726                 if self.seed_version==4:
 727                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 728                     if not m: continue
 729                     mpk = m.group(1)
 730                     if mpk != self.storage.get('master_public_key'): continue
 731                     index = int(m.group(2))
 732                     num = int(m.group(3))
 733                     account = self.accounts[0]
 734                     addr = account.get_address(index, num)
 735                     txin['address'] = addr # fixme: side effect
 736                     pk = self.get_private_key(addr, password)
 737                     for sec in pk:
 738                         pubkey = public_key_from_private_key(sec)
 739                         keypairs[pubkey] = sec
 740                     continue
 741
 742
 743                 roots = []
 744                 for s in keyid.split('&'):
 745                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 746                     if not m: continue
 747                     c = m.group(1)
 748                     K = m.group(2)
 749                     sequence = m.group(3)
 750                     root = self.find_root_by_master_key(c,K)
 751                     if not root: continue
 752                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 753                     root = root + '%d'%sequence[0]
 754                     sequence = sequence[1:]
 755                     roots.append((root,sequence))
 756
 757                 account_id = " & ".join( map(lambda x:x[0], roots) )
 758                 account = self.accounts.get(account_id)
 759                 if not account: continue
 760                 addr = account.get_address(*sequence)
 761                 txin['address'] = addr # fixme: side effect
 762                 pk = self.get_private_key(addr, password)
 763                 for sec in pk:
 764                     pubkey = public_key_from_private_key(sec)
 765                     keypairs[pubkey] = sec
 766
 767
 768
 769     def signrawtransaction(self, tx, input_info, private_keys, password):
 770
 771         # check that the password is correct
 772         seed = self.get_seed(password)
 773
 774         # add input info
 775         tx.add_input_info(input_info)
 776
 777         # add redeem script for coins that are in the wallet
 778         # FIXME: add redeemPubkey too!
 779         unspent_coins = self.get_unspent_coins()
 780         for txin in tx.inputs:
 781             for item in unspent_coins:
 782                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 783                     print_error( "tx input is in unspent coins" )
 784                     txin['scriptPubKey'] = item['scriptPubKey']
 785                     account, sequence = self.get_address_index(item['address'])
 786                     if account != -1:
 787                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 788                         print_error("added redeemScript", txin['redeemScript'])
 789                     break
 790
 791
 792         # build a list of public/private keys
 793         keypairs = {}
 794
 795         # add private keys from parameter
 796         for sec in private_keys:
 797             pubkey = public_key_from_private_key(sec)
 798             keypairs[ pubkey ] = sec
 799
 800         # add private_keys from KeyID
 801         self.add_keypairs_from_KeyID(tx, keypairs, password)
 802
 803         # add private keys from wallet
 804         self.add_keypairs_from_wallet(tx, keypairs, password)
 805         self.sign_transaction(tx, keypairs)
 806
 807
 808     def sign_message(self, address, message, password):
 809         keys = self.get_private_key(address, password)
 810         assert len(keys) == 1
 811         sec = keys[0]
 812         key = regenerate_key(sec)
 813         compressed = is_compressed(sec)
 814         return key.sign_message(message, compressed, address)
 815
 816
 817     def change_gap_limit(self, value):
 818         if value >= self.gap_limit:
 819             self.gap_limit = value
 820             self.storage.put('gap_limit', self.gap_limit, True)
 821             #self.interface.poke('synchronizer')
 822             return True
 823
 824         elif value >= self.min_acceptable_gap():
 825             for key, account in self.accounts.items():
 826                 addresses = account[0]
 827                 k = self.num_unused_trailing_addresses(addresses)
 828                 n = len(addresses) - k + value
 829                 addresses = addresses[0:n]
 830                 self.accounts[key][0] = addresses
 831
 832             self.gap_limit = value
 833             self.storage.put('gap_limit', self.gap_limit, True)
 834             self.save_accounts()
 835             return True
 836         else:
 837             return False
 838
 839     def num_unused_trailing_addresses(self, addresses):
 840         k = 0
 841         for a in addresses[::-1]:
 842             if self.history.get(a):break
 843             k = k + 1
 844         return k
 845
 846     def min_acceptable_gap(self):
 847         # fixme: this assumes wallet is synchronized
 848         n = 0
 849         nmax = 0
 850
 851         for account in self.accounts.values():
 852             addresses = account.get_addresses(0)
 853             k = self.num_unused_trailing_addresses(addresses)
 854             for a in addresses[0:-k]:
 855                 if self.history.get(a):
 856                     n = 0
 857                 else:
 858                     n += 1
 859                     if n > nmax: nmax = n
 860         return nmax + 1
 861
 862
 863     def address_is_old(self, address):
 864         age = -1
 865         h = self.history.get(address, [])
 866         if h == ['*']:
 867             return True
 868         for tx_hash, tx_height in h:
 869             if tx_height == 0:
 870                 tx_age = 0
 871             else:
 872                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 873             if tx_age > age:
 874                 age = tx_age
 875         return age > 2
 876
 877
 878     def synchronize_sequence(self, account, for_change):
 879         limit = self.gap_limit_for_change if for_change else self.gap_limit
 880         new_addresses = []
 881         while True:
 882             addresses = account.get_addresses(for_change)
 883             if len(addresses) < limit:
 884                 address = account.create_new_address(for_change)
 885                 self.history[address] = []
 886                 new_addresses.append( address )
 887                 continue
 888
 889             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 890                 break
 891             else:
 892                 address = account.create_new_address(for_change)
 893                 self.history[address] = []
 894                 new_addresses.append( address )
 895
 896         return new_addresses
 897
 898
 899
 900     def create_pending_accounts(self):
 901         for account_type in ['1','2of2','2of3']:
 902             if not self.has_master_public_keys(account_type):
 903                 continue
 904             k, a = self.new_account_address(account_type)
 905             if self.address_is_old(a):
 906                 print_error( "creating account", a )
 907                 self.create_account(account_type)
 908                 self.next_addresses.pop(k)
 909
 910
 911     def synchronize_account(self, account):
 912         new = []
 913         new += self.synchronize_sequence(account, 0)
 914         new += self.synchronize_sequence(account, 1)
 915         return new
 916
 917
 918     def synchronize(self):
 919         if self.master_public_keys:
 920             self.create_pending_accounts()
 921         new = []
 922         for account in self.accounts.values():
 923             new += self.synchronize_account(account)
 924         if new:
 925             self.save_accounts()
 926             self.storage.put('addr_history', self.history, True)
 927         return new
 928
 929
 930     def is_found(self):
 931         return self.history.values() != [[]] * len(self.history)
 932
 933
 934     def add_contact(self, address, label=None):
 935         self.addressbook.append(address)
 936         self.storage.put('contacts', self.addressbook, True)
 937         if label:
 938             self.set_label(address, label)
 939
 940
 941     def delete_contact(self, addr):
 942         if addr in self.addressbook:
 943             self.addressbook.remove(addr)
 944             self.storage.put('addressbook', self.addressbook, True)
 945
 946
 947     def fill_addressbook(self):
 948         for tx_hash, tx in self.transactions.items():
 949             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 950             if is_send:
 951                 for addr, v in tx.outputs:
 952                     if not self.is_mine(addr) and addr not in self.addressbook:
 953                         self.addressbook.append(addr)
 954         # redo labels
 955         # self.update_tx_labels()
 956
 957     def get_num_tx(self, address):
 958         n = 0
 959         for tx in self.transactions.values():
 960             if address in map(lambda x:x[0], tx.outputs): n += 1
 961         return n
 962
 963
 964     def get_address_flags(self, addr):
 965         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 966         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 967         return flags
 968
 969
 970     def get_tx_value(self, tx, account=None):
 971         domain = self.get_account_addresses(account)
 972         return tx.get_value(domain, self.prevout_values)
 973
 974
 975     def update_tx_outputs(self, tx_hash):
 976         tx = self.transactions.get(tx_hash)
 977
 978         for i, (addr, value) in enumerate(tx.outputs):
 979             key = tx_hash+ ':%d'%i
 980             self.prevout_values[key] = value
 981
 982         for item in tx.inputs:
 983             if self.is_mine(item.get('address')):
 984                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 985                 self.spent_outputs.append(key)
 986
 987
 988     def get_addr_balance(self, address):
 989         assert self.is_mine(address)
 990         h = self.history.get(address,[])
 991         if h == ['*']: return 0,0
 992         c = u = 0
 993         received_coins = []   # list of coins received at address
 994
 995         for tx_hash, tx_height in h:
 996             tx = self.transactions.get(tx_hash)
 997             if not tx: continue
 998
 999             for i, (addr, value) in enumerate(tx.outputs):
1000                 if addr == address:
1001                     key = tx_hash + ':%d'%i
1002                     received_coins.append(key)
1003
1004         for tx_hash, tx_height in h:
1005             tx = self.transactions.get(tx_hash)
1006             if not tx: continue
1007             v = 0
1008
1009             for item in tx.inputs:
1010                 addr = item.get('address')
1011                 if addr == address:
1012                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1013                     value = self.prevout_values.get( key )
1014                     if key in received_coins:
1015                         v -= value
1016
1017             for i, (addr, value) in enumerate(tx.outputs):
1018                 key = tx_hash + ':%d'%i
1019                 if addr == address:
1020                     v += value
1021
1022             if tx_height:
1023                 c += v
1024             else:
1025                 u += v
1026         return c, u
1027
1028
1029     def get_account_name(self, k):
1030         if k == 0:
1031             if self.seed_version == 4:
1032                 name = 'Main account'
1033             else:
1034                 name = 'Old account'
1035         else:
1036             default = "Unnamed account"
1037             m = re.match("m/0'/(\d+)", k)
1038             if m:
1039                 num = m.group(1)
1040                 if num == '0':
1041                     default = "Main account"
1042                 else:
1043                     default = "Account %s"%num
1044
1045             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1046             if m:
1047                 num = m.group(1)
1048                 default = "2of2 account %s"%num
1049             name = self.labels.get(k, default)
1050
1051         return name
1052
1053     def get_account_names(self):
1054         accounts = {}
1055         for k, account in self.accounts.items():
1056             accounts[k] = self.get_account_name(k)
1057         if self.imported_keys:
1058             accounts[-1] = 'Imported keys'
1059         return accounts
1060
1061     def get_account_addresses(self, a, include_change=True):
1062         if a is None:
1063             o = self.addresses(True)
1064         elif a == -1:
1065             o = self.imported_keys.keys()
1066         else:
1067             ac = self.accounts[a]
1068             o = ac.get_addresses(0)
1069             if include_change: o += ac.get_addresses(1)
1070         return o
1071
1072     def get_imported_balance(self):
1073         return self.get_balance(self.imported_keys.keys())
1074
1075     def get_account_balance(self, account):
1076         return self.get_balance(self.get_account_addresses(account))
1077
1078     def get_frozen_balance(self):
1079         return self.get_balance(self.frozen_addresses)
1080
1081     def get_balance(self, domain=None):
1082         if domain is None: domain = self.addresses(True)
1083         cc = uu = 0
1084         for addr in domain:
1085             c, u = self.get_addr_balance(addr)
1086             cc += c
1087             uu += u
1088         return cc, uu
1089
1090
1091     def get_unspent_coins(self, domain=None):
1092         coins = []
1093         if domain is None: domain = self.addresses(True)
1094         for addr in domain:
1095             h = self.history.get(addr, [])
1096             if h == ['*']: continue
1097             for tx_hash, tx_height in h:
1098                 tx = self.transactions.get(tx_hash)
1099                 if tx is None: raise BaseException("Wallet not synchronized")
1100                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1101                 for output in tx.d.get('outputs'):
1102                     if output.get('address') != addr: continue
1103                     key = tx_hash + ":%d" % output.get('prevout_n')
1104                     if key in self.spent_outputs: continue
1105                     output['prevout_hash'] = tx_hash
1106                     output['height'] = tx_height
1107                     output['coinbase'] = is_coinbase
1108                     coins.append((tx_height, output))
1109
1110         # sort by age
1111         if coins:
1112             coins = sorted(coins)
1113             if coins[-1][0] != 0:
1114                 while coins[0][0] == 0:
1115                     coins = coins[1:] + [ coins[0] ]
1116         return [x[1] for x in coins]
1117
1118
1119
1120     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1121         domain = self.get_account_addresses(account) if account else None
1122         return self.choose_tx_inputs( amount, fixed_fee, domain )
1123
1124
1125     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1126         """ todo: minimize tx size """
1127         total = 0
1128         fee = self.fee if fixed_fee is None else fixed_fee
1129         if domain is None:
1130             domain = self.addresses(True)
1131
1132         for i in self.frozen_addresses:
1133             if i in domain: domain.remove(i)
1134
1135         prioritized = []
1136         for i in self.prioritized_addresses:
1137             if i in domain:
1138                 domain.remove(i)
1139                 prioritized.append(i)
1140
1141         coins = self.get_unspent_coins(domain)
1142         prioritized_coins = self.get_unspent_coins(prioritized)
1143
1144         inputs = []
1145         coins = prioritized_coins + coins
1146
1147         for item in coins:
1148             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1149                 continue
1150             addr = item.get('address')
1151             v = item.get('value')
1152             total += v
1153             inputs.append(item)
1154             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1155             if total >= amount + fee: break
1156         else:
1157             inputs = []
1158
1159         return inputs, total, fee
1160
1161
1162     def set_fee(self, fee):
1163         if self.fee != fee:
1164             self.fee = fee
1165             self.storage.put('fee_per_kb', self.fee, True)
1166
1167     def estimated_fee(self, inputs):
1168         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1169         fee = self.fee * int(round(estimated_size/1024.))
1170         if fee == 0: fee = self.fee
1171         return fee
1172
1173
1174     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1175         "add change to a transaction"
1176         change_amount = total - ( amount + fee )
1177         if change_amount != 0:
1178             if not change_addr:
1179
1180                 # send change to one of the accounts involved in the tx
1181                 address = inputs[0].get('address')
1182                 account, _ = self.get_address_index(address)
1183
1184                 if not self.use_change or account == -1:
1185                     change_addr = inputs[-1]['address']
1186                 else:
1187                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1188
1189             # Insert the change output at a random position in the outputs
1190             posn = random.randint(0, len(outputs))
1191             outputs[posn:posn] = [( change_addr,  change_amount)]
1192         return outputs
1193
1194
1195     def get_history(self, address):
1196         with self.lock:
1197             return self.history.get(address)
1198
1199
1200     def get_status(self, h):
1201         if not h: return None
1202         if h == ['*']: return '*'
1203         status = ''
1204         for tx_hash, height in h:
1205             status += tx_hash + ':%d:' % height
1206         return hashlib.sha256( status ).digest().encode('hex')
1207
1208
1209     def receive_tx_callback(self, tx_hash, tx, tx_height):
1210
1211         with self.transaction_lock:
1212             self.add_extra_addresses(tx)
1213             if not self.check_new_tx(tx_hash, tx):
1214                 # may happen due to pruning
1215                 print_error("received transaction that is no longer referenced in history", tx_hash)
1216                 return
1217             self.transactions[tx_hash] = tx
1218             self.network.interface.pending_transactions_for_notifications.append(tx)
1219             self.save_transactions()
1220             if self.verifier and tx_height>0:
1221                 self.verifier.add(tx_hash, tx_height)
1222             self.update_tx_outputs(tx_hash)
1223
1224
1225     def save_transactions(self):
1226         tx = {}
1227         for k,v in self.transactions.items():
1228             tx[k] = str(v)
1229         self.storage.put('transactions', tx, True)
1230
1231     def receive_history_callback(self, addr, hist):
1232
1233         if not self.check_new_history(addr, hist):
1234             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1235
1236         with self.lock:
1237             self.history[addr] = hist
1238             self.storage.put('addr_history', self.history, True)
1239
1240         if hist != ['*']:
1241             for tx_hash, tx_height in hist:
1242                 if tx_height>0:
1243                     # add it in case it was previously unconfirmed
1244                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1245
1246
1247     def get_tx_history(self, account=None):
1248         with self.transaction_lock:
1249             history = self.transactions.items()
1250             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1251             result = []
1252
1253             balance = 0
1254             for tx_hash, tx in history:
1255                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1256                 if v is not None: balance += v
1257
1258             c, u = self.get_account_balance(account)
1259
1260             if balance != c+u:
1261                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1262
1263             balance = c + u - balance
1264             for tx_hash, tx in history:
1265                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1266                 if not is_relevant:
1267                     continue
1268                 if value is not None:
1269                     balance += value
1270
1271                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1272                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1273
1274         return result
1275
1276
1277     def get_label(self, tx_hash):
1278         label = self.labels.get(tx_hash)
1279         is_default = (label == '') or (label is None)
1280         if is_default: label = self.get_default_label(tx_hash)
1281         return label, is_default
1282
1283
1284     def get_default_label(self, tx_hash):
1285         tx = self.transactions.get(tx_hash)
1286         default_label = ''
1287         if tx:
1288             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1289             if is_mine:
1290                 for o in tx.outputs:
1291                     o_addr, _ = o
1292                     if not self.is_mine(o_addr):
1293                         try:
1294                             default_label = self.labels[o_addr]
1295                         except KeyError:
1296                             default_label = o_addr
1297                         break
1298                 else:
1299                     default_label = '(internal)'
1300             else:
1301                 for o in tx.outputs:
1302                     o_addr, _ = o
1303                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1304                         break
1305                 else:
1306                     for o in tx.outputs:
1307                         o_addr, _ = o
1308                         if self.is_mine(o_addr):
1309                             break
1310                     else:
1311                         o_addr = None
1312
1313                 if o_addr:
1314                     dest_label = self.labels.get(o_addr)
1315                     try:
1316                         default_label = self.labels[o_addr]
1317                     except KeyError:
1318                         default_label = o_addr
1319
1320         return default_label
1321
1322
1323     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1324         for address, x in outputs:
1325             assert is_valid(address)
1326         amount = sum( map(lambda x:x[1], outputs) )
1327         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1328         if not inputs:
1329             raise ValueError("Not enough funds")
1330         self.add_input_info(inputs)
1331         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1332         return Transaction.from_io(inputs, outputs)
1333
1334
1335     def mktx_from_account(self, outputs, password, fee=None, account=None):
1336         domain = self.get_account_addresses(account) if account else None
1337         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1338
1339
1340     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1341         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1342         keypairs = {}
1343         self.add_keypairs_from_wallet(tx, keypairs, password)
1344         if keypairs:
1345             self.sign_transaction(tx, keypairs)
1346         return tx
1347
1348
1349     def add_input_info(self, inputs):
1350         for txin in inputs:
1351             address = txin['address']
1352             if address in self.imported_keys.keys():
1353                 continue
1354             account, sequence = self.get_address_index(address)
1355             txin['KeyID'] = self.get_keyID(account, sequence)
1356             redeemScript = self.accounts[account].redeem_script(sequence)
1357             if redeemScript:
1358                 txin['redeemScript'] = redeemScript
1359             else:
1360                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1361
1362
1363     def sign_transaction(self, tx, keypairs):
1364         tx.sign(keypairs)
1365         run_hook('sign_transaction', tx)
1366
1367
1368     def sendtx(self, tx):
1369         # synchronous
1370         h = self.send_tx(tx)
1371         self.tx_event.wait()
1372         return self.receive_tx(h)
1373
1374     def send_tx(self, tx):
1375         # asynchronous
1376         self.tx_event.clear()
1377         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1378         return tx.hash()
1379
1380     def on_broadcast(self, i, r):
1381         self.tx_result = r.get('result')
1382         self.tx_event.set()
1383
1384     def receive_tx(self,tx_hash):
1385         out = self.tx_result
1386         if out != tx_hash:
1387             return False, "error: " + out
1388         return True, out
1389
1390
1391
1392     def update_password(self, old_password, new_password):
1393         if new_password == '': new_password = None
1394         # this will throw an exception if unicode cannot be converted
1395         decoded = pw_decode(self.seed, old_password)
1396         self.seed = pw_encode( decoded, new_password)
1397         self.storage.put('seed', self.seed, True)
1398         self.use_encryption = (new_password != None)
1399         self.storage.put('use_encryption', self.use_encryption,True)
1400         for k in self.imported_keys.keys():
1401             a = self.imported_keys[k]
1402             b = pw_decode(a, old_password)
1403             c = pw_encode(b, new_password)
1404             self.imported_keys[k] = c
1405         self.storage.put('imported_keys', self.imported_keys, True)
1406
1407         for k, v in self.master_private_keys.items():
1408             b = pw_decode(v, old_password)
1409             c = pw_encode(b, new_password)
1410             self.master_private_keys[k] = c
1411         self.storage.put('master_private_keys', self.master_private_keys, True)
1412
1413
1414     def freeze(self,addr):
1415         if self.is_mine(addr) and addr not in self.frozen_addresses:
1416             self.unprioritize(addr)
1417             self.frozen_addresses.append(addr)
1418             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1419             return True
1420         else:
1421             return False
1422
1423     def unfreeze(self,addr):
1424         if self.is_mine(addr) and addr in self.frozen_addresses:
1425             self.frozen_addresses.remove(addr)
1426             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1427             return True
1428         else:
1429             return False
1430
1431     def prioritize(self,addr):
1432         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1433             self.unfreeze(addr)
1434             self.prioritized_addresses.append(addr)
1435             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1436             return True
1437         else:
1438             return False
1439
1440     def unprioritize(self,addr):
1441         if self.is_mine(addr) and addr in self.prioritized_addresses:
1442             self.prioritized_addresses.remove(addr)
1443             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1444             return True
1445         else:
1446             return False
1447
1448
1449     def set_verifier(self, verifier):
1450         self.verifier = verifier
1451
1452         # review transactions that are in the history
1453         for addr, hist in self.history.items():
1454             if hist == ['*']: continue
1455             for tx_hash, tx_height in hist:
1456                 if tx_height>0:
1457                     # add it in case it was previously unconfirmed
1458                     self.verifier.add(tx_hash, tx_height)
1459
1460
1461         # if we are on a pruning server, remove unverified transactions
1462         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1463         for tx_hash in self.transactions.keys():
1464             if tx_hash not in vr:
1465                 self.transactions.pop(tx_hash)
1466
1467
1468
1469     def check_new_history(self, addr, hist):
1470
1471         # check that all tx in hist are relevant
1472         if hist != ['*']:
1473             for tx_hash, height in hist:
1474                 tx = self.transactions.get(tx_hash)
1475                 if not tx: continue
1476                 if not tx.has_address(addr):
1477                     return False
1478
1479         # check that we are not "orphaning" a transaction
1480         old_hist = self.history.get(addr,[])
1481         if old_hist == ['*']: return True
1482
1483         for tx_hash, height in old_hist:
1484             if tx_hash in map(lambda x:x[0], hist): continue
1485             found = False
1486             for _addr, _hist in self.history.items():
1487                 if _addr == addr: continue
1488                 if _hist == ['*']: continue
1489                 _tx_hist = map(lambda x:x[0], _hist)
1490                 if tx_hash in _tx_hist:
1491                     found = True
1492                     break
1493
1494             if not found:
1495                 tx = self.transactions.get(tx_hash)
1496                 # tx might not be there
1497                 if not tx: continue
1498
1499                 # already verified?
1500                 if self.verifier.get_height(tx_hash):
1501                     continue
1502                 # unconfirmed tx
1503                 print_error("new history is orphaning transaction:", tx_hash)
1504                 # check that all outputs are not mine, request histories
1505                 ext_requests = []
1506                 for _addr, _v in tx.outputs:
1507                     # assert not self.is_mine(_addr)
1508                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1509
1510                 ext_h = self.network.synchronous_get(ext_requests)
1511                 print_error("sync:", ext_requests, ext_h)
1512                 height = None
1513                 for h in ext_h:
1514                     if h == ['*']: continue
1515                     for item in h:
1516                         if item.get('tx_hash') == tx_hash:
1517                             height = item.get('height')
1518                 if height:
1519                     print_error("found height for", tx_hash, height)
1520                     self.verifier.add(tx_hash, height)
1521                 else:
1522                     print_error("removing orphaned tx from history", tx_hash)
1523                     self.transactions.pop(tx_hash)
1524
1525         return True
1526
1527
1528
1529     def check_new_tx(self, tx_hash, tx):
1530         # 1 check that tx is referenced in addr_history.
1531         addresses = []
1532         for addr, hist in self.history.items():
1533             if hist == ['*']:continue
1534             for txh, height in hist:
1535                 if txh == tx_hash:
1536                     addresses.append(addr)
1537
1538         if not addresses:
1539             return False
1540
1541         # 2 check that referencing addresses are in the tx
1542         for addr in addresses:
1543             if not tx.has_address(addr):
1544                 return False
1545
1546         return True
1547
1548
1549     def start_threads(self, network):
1550         from verifier import TxVerifier
1551         self.network = network
1552         self.verifier = TxVerifier(self.network, self.storage)
1553         self.verifier.start()
1554         self.set_verifier(self.verifier)
1555         self.synchronizer = WalletSynchronizer(self, network)
1556         self.synchronizer.start()
1557
1558     def stop_threads(self):
1559         self.verifier.stop()
1560         self.synchronizer.stop()
1561
1562
1563
1564     def restore(self, callback):
1565         from i18n import _
1566         def wait_for_wallet():
1567             self.set_up_to_date(False)
1568             while not self.is_up_to_date():
1569                 msg = "%s\n%s %d\n%s %.1f"%(
1570                     _("Please wait..."),
1571                     _("Addresses generated:"),
1572                     len(self.addresses(True)),_("Kilobytes received:"),
1573                     self.network.interface.bytes_received/1024.)
1574
1575                 apply(callback, (msg,))
1576                 time.sleep(0.1)
1577
1578         def wait_for_network():
1579             while not self.network.interface.is_connected:
1580                 msg = "%s \n" % (_("Connecting..."))
1581                 apply(callback, (msg,))
1582                 time.sleep(0.1)
1583
1584         # wait until we are connected, because the user might have selected another server
1585         wait_for_network()
1586
1587
1588         self.create_accounts()
1589         wait_for_wallet()
1590
1591
1592
1593
1594 class WalletSynchronizer(threading.Thread):
1595
1596
1597     def __init__(self, wallet, network):
1598         threading.Thread.__init__(self)
1599         self.daemon = True
1600         self.wallet = wallet
1601         self.network = network
1602         self.was_updated = True
1603         self.running = False
1604         self.lock = threading.Lock()
1605         self.queue = Queue.Queue()
1606
1607     def stop(self):
1608         with self.lock: self.running = False
1609
1610     def is_running(self):
1611         with self.lock: return self.running
1612
1613
1614     def subscribe_to_addresses(self, addresses):
1615         messages = []
1616         for addr in addresses:
1617             messages.append(('blockchain.address.subscribe', [addr]))
1618         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1619
1620
1621     def run(self):
1622         with self.lock:
1623             self.running = True
1624
1625         while self.is_running():
1626
1627             if not self.network.is_connected():
1628                 print_error("synchronizer: waiting for interface")
1629                 self.network.wait_until_connected()
1630
1631             self.run_interface(self.network.interface)
1632
1633
1634     def run_interface(self, interface):
1635
1636         print_error("synchronizer: connected to", interface.server)
1637
1638         requested_tx = []
1639         missing_tx = []
1640         requested_histories = {}
1641
1642         # request any missing transactions
1643         for history in self.wallet.history.values():
1644             if history == ['*']: continue
1645             for tx_hash, tx_height in history:
1646                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1647                     missing_tx.append( (tx_hash, tx_height) )
1648
1649         if missing_tx:
1650             print_error("missing tx", missing_tx)
1651
1652         # subscriptions
1653         self.subscribe_to_addresses(self.wallet.addresses(True))
1654
1655         while self.is_running():
1656             # 1. create new addresses
1657             new_addresses = self.wallet.synchronize()
1658
1659             # request missing addresses
1660             if new_addresses:
1661                 self.subscribe_to_addresses(new_addresses)
1662
1663             # request missing transactions
1664             for tx_hash, tx_height in missing_tx:
1665                 if (tx_hash, tx_height) not in requested_tx:
1666                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1667                     requested_tx.append( (tx_hash, tx_height) )
1668             missing_tx = []
1669
1670             # detect if situation has changed
1671             if interface.is_up_to_date() and self.queue.empty():
1672                 if not self.wallet.is_up_to_date():
1673                     self.wallet.set_up_to_date(True)
1674                     self.was_updated = True
1675             else:
1676                 if self.wallet.is_up_to_date():
1677                     self.wallet.set_up_to_date(False)
1678                     self.was_updated = True
1679
1680             if self.was_updated:
1681                 self.wallet.network.trigger_callback('updated')
1682                 self.was_updated = False
1683
1684             # 2. get a response
1685             try:
1686                 r = self.queue.get(block=True, timeout=1)
1687             except Queue.Empty:
1688                 continue
1689
1690             if interface != self.network.interface:
1691                 break
1692
1693             if not r:
1694                 continue
1695
1696             # 3. handle response
1697             method = r['method']
1698             params = r['params']
1699             result = r.get('result')
1700             error = r.get('error')
1701             if error:
1702                 print "error", r
1703                 continue
1704
1705             if method == 'blockchain.address.subscribe':
1706                 addr = params[0]
1707                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1708                     if requested_histories.get(addr) is None:
1709                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1710                         requested_histories[addr] = result
1711
1712             elif method == 'blockchain.address.get_history':
1713                 addr = params[0]
1714                 print_error("receiving history", addr, result)
1715                 if result == ['*']:
1716                     assert requested_histories.pop(addr) == '*'
1717                     self.wallet.receive_history_callback(addr, result)
1718                 else:
1719                     hist = []
1720                     # check that txids are unique
1721                     txids = []
1722                     for item in result:
1723                         tx_hash = item['tx_hash']
1724                         if tx_hash not in txids:
1725                             txids.append(tx_hash)
1726                             hist.append( (tx_hash, item['height']) )
1727
1728                     if len(hist) != len(result):
1729                         raise BaseException("error: server sent history with non-unique txid", result)
1730
1731                     # check that the status corresponds to what was announced
1732                     rs = requested_histories.pop(addr)
1733                     if self.wallet.get_status(hist) != rs:
1734                         raise BaseException("error: status mismatch: %s"%addr)
1735
1736                     # store received history
1737                     self.wallet.receive_history_callback(addr, hist)
1738
1739                     # request transactions that we don't have
1740                     for tx_hash, tx_height in hist:
1741                         if self.wallet.transactions.get(tx_hash) is None:
1742                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1743                                 missing_tx.append( (tx_hash, tx_height) )
1744
1745             elif method == 'blockchain.transaction.get':
1746                 tx_hash = params[0]
1747                 tx_height = params[1]
1748                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1749                 tx = Transaction(result)
1750                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1751                 self.was_updated = True
1752                 requested_tx.remove( (tx_hash, tx_height) )
1753                 print_error("received tx:", tx_hash, len(tx.raw))
1754
1755             else:
1756                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1757
1758             if self.was_updated and not requested_tx:
1759                 self.wallet.network.trigger_callback('updated')
1760                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1761
1762                 self.was_updated = False