lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except:
  59             raise BaseException('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         return self.data.get(key, default)
 129
 130     def put(self, key, value, save = True):
 131
 132         with self.lock:
 133             if value is not None:
 134                 self.data[key] = value
 135             else:
 136                 self.data.pop(key)
 137             if save:
 138                 self.write()
 139
 140     def write(self):
 141         s = repr(self.data)
 142         f = open(self.path,"w")
 143         f.write( s )
 144         f.close()
 145         if 'ANDROID_DATA' not in os.environ:
 146             import stat
 147             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 148
 149
 150 class Wallet:
 151
 152     def __init__(self, storage):
 153
 154         self.storage = storage
 155         self.electrum_version = ELECTRUM_VERSION
 156         self.gap_limit_for_change = 3 # constant
 157
 158         # saved fields
 159         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 160
 161         self.gap_limit             = storage.get('gap_limit', 5)
 162         self.use_change            = storage.get('use_change',True)
 163         self.use_encryption        = storage.get('use_encryption', False)
 164         self.seed                  = storage.get('seed', '')               # encrypted
 165         self.labels                = storage.get('labels', {})
 166         self.frozen_addresses      = storage.get('frozen_addresses',[])
 167         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 168         self.addressbook           = storage.get('contacts', [])
 169
 170         self.imported_keys         = storage.get('imported_keys',{})
 171         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 172
 173         self.fee                   = int(storage.get('fee_per_kb',20000))
 174
 175         self.master_public_keys = storage.get('master_public_keys',{})
 176         self.master_private_keys = storage.get('master_private_keys', {})
 177
 178         self.next_addresses = storage.get('next_addresses',{})
 179
 180         if self.seed_version not in [4, 6]:
 181             msg = "This wallet seed is not supported."
 182             if self.seed_version in [5]:
 183                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 184                 print msg
 185                 sys.exit(1)
 186
 187
 188         self.load_accounts()
 189
 190         self.transactions = {}
 191         tx_list = self.storage.get('transactions',{})
 192         for k,v in tx_list.items():
 193             try:
 194                 tx = Transaction(v)
 195             except:
 196                 print_msg("Warning: Cannot deserialize transactions. skipping")
 197                 continue
 198
 199             self.add_extra_addresses(tx)
 200             self.transactions[k] = tx
 201
 202         for h,tx in self.transactions.items():
 203             if not self.check_new_tx(h, tx):
 204                 print_error("removing unreferenced tx", h)
 205                 self.transactions.pop(h)
 206
 207
 208         # not saved
 209         self.prevout_values = {}     # my own transaction outputs
 210         self.spent_outputs = []
 211
 212         # spv
 213         self.verifier = None
 214
 215         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 216         # interface.is_up_to_date() returns true when all requests have been answered and processed
 217         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 218
 219         self.up_to_date = False
 220         self.lock = threading.Lock()
 221         self.transaction_lock = threading.Lock()
 222         self.tx_event = threading.Event()
 223
 224         for tx_hash, tx in self.transactions.items():
 225             self.update_tx_outputs(tx_hash)
 226
 227
 228     def add_extra_addresses(self, tx):
 229         h = tx.hash()
 230         # find the address corresponding to pay-to-pubkey inputs
 231         tx.add_extra_addresses(self.transactions)
 232         for o in tx.d.get('outputs'):
 233             if o.get('is_pubkey'):
 234                 for tx2 in self.transactions.values():
 235                     tx2.add_extra_addresses({h:tx})
 236
 237
 238
 239
 240     def set_up_to_date(self,b):
 241         with self.lock: self.up_to_date = b
 242
 243
 244     def is_up_to_date(self):
 245         with self.lock: return self.up_to_date
 246
 247
 248     def update(self):
 249         self.up_to_date = False
 250         while not self.is_up_to_date():
 251             time.sleep(0.1)
 252
 253
 254     def import_key(self, sec, password):
 255         # check password
 256         seed = self.get_seed(password)
 257         try:
 258             address = address_from_private_key(sec)
 259         except:
 260             raise BaseException('Invalid private key')
 261
 262         if self.is_mine(address):
 263             raise BaseException('Address already in wallet')
 264
 265         # store the originally requested keypair into the imported keys table
 266         self.imported_keys[address] = pw_encode(sec, password )
 267         self.storage.put('imported_keys', self.imported_keys, True)
 268         if self.synchronizer:
 269             self.synchronizer.subscribe_to_addresses([address])
 270         return address
 271
 272     def delete_imported_key(self, addr):
 273         if addr in self.imported_keys:
 274             self.imported_keys.pop(addr)
 275             self.storage.put('imported_keys', self.imported_keys, True)
 276
 277
 278     def make_seed(self):
 279         import mnemonic, ecdsa
 280         entropy = ecdsa.util.randrange( pow(2,160) )
 281         nonce = 0
 282         while True:
 283             ss = "%040x"%(entropy+nonce)
 284             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 285             # we keep only 13 words, that's approximately 139 bits of entropy
 286             words = mnemonic.mn_encode(s)[0:13]
 287             seed = ' '.join(words)
 288             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 289                 break  # this removes 12 bits of entropy
 290             nonce += 1
 291
 292         return seed
 293
 294
 295     def init_seed(self, seed):
 296         import mnemonic
 297
 298         if self.seed:
 299             raise BaseException("a seed exists")
 300
 301         if not seed:
 302             self.seed = random_seed(128)
 303             self.seed_version = 4
 304             return
 305
 306         #if not seed:
 307         #    self.seed = self.make_seed()
 308         #    self.seed_version = SEED_VERSION
 309         #    return
 310
 311         # find out what kind of wallet we are
 312         try:
 313             seed.decode('hex')
 314             self.seed_version = 4
 315             self.seed = str(seed)
 316             return
 317         except:
 318             pass
 319
 320         words = seed.split()
 321         self.seed_version = 4
 322         self.seed = mnemonic.mn_decode(words)
 323
 324         #try:
 325         #    mnemonic.mn_decode(words)
 326         #    uses_electrum_words = True
 327         #except:
 328         #    uses_electrum_words = False
 329         #
 330         #if uses_electrum_words and len(words) != 13:
 331         #    self.seed_version = 4
 332         #    self.seed = mnemonic.mn_decode(words)
 333         #else:
 334         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 335         #    self.seed_version = SEED_VERSION
 336         #    self.seed = seed
 337
 338
 339     def save_seed(self):
 340         self.storage.put('seed', self.seed, True)
 341         self.storage.put('seed_version', self.seed_version, True)
 342
 343     def create_watching_only_wallet(self, params):
 344         K0, c0 = params
 345         if not K0:
 346             return
 347
 348         if not c0:
 349             self.seed_version = 4
 350             self.storage.put('seed_version', self.seed_version, True)
 351             self.create_old_account(K0)
 352             return
 353
 354         cK0 = ""
 355         self.master_public_keys = {
 356             "m/0'/": (c0, K0, cK0),
 357             }
 358         self.storage.put('master_public_keys', self.master_public_keys, True)
 359         self.storage.put('seed_version', self.seed_version, True)
 360         self.create_account('1','Main account')
 361
 362
 363     def create_accounts(self):
 364         if self.seed_version == 4:
 365             mpk = OldAccount.mpk_from_seed(self.seed)
 366             self.create_old_account(mpk)
 367         else:
 368             # create default account
 369             self.create_master_keys('1')
 370             self.create_account('1','Main account')
 371
 372
 373     def create_master_keys(self, account_type):
 374         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 375         if account_type == '1':
 376             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 377             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 378             self.master_private_keys["m/0'/"] = k0
 379         elif account_type == '2of2':
 380             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 381             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 382             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 383             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 384             self.master_private_keys["m/1'/"] = k1
 385             self.master_private_keys["m/2'/"] = k2
 386         elif account_type == '2of3':
 387             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 388             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 389             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 390             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 391             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 392             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 393             self.master_private_keys["m/3'/"] = k3
 394             self.master_private_keys["m/4'/"] = k4
 395             self.master_private_keys["m/5'/"] = k5
 396
 397         self.storage.put('master_public_keys', self.master_public_keys, True)
 398         self.storage.put('master_private_keys', self.master_private_keys, True)
 399
 400     def has_master_public_keys(self, account_type):
 401         if account_type == '1':
 402             return "m/0'/" in self.master_public_keys
 403         elif account_type == '2of2':
 404             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 405         elif account_type == '2of3':
 406             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 407
 408     def find_root_by_master_key(self, c, K):
 409         for key, v in self.master_public_keys.items():
 410             if key == "m/":continue
 411             cc, KK, _ = v
 412             if (c == cc) and (K == KK):
 413                 return key
 414
 415     def deseed_root(self, seed, password):
 416         # for safety, we ask the user to enter their seed
 417         assert seed == self.get_seed(password)
 418         self.seed = ''
 419         self.storage.put('seed', '', True)
 420
 421
 422     def deseed_branch(self, k):
 423         # check that parent has no seed
 424         assert self.seed == ''
 425         self.master_private_keys.pop(k)
 426         self.storage.put('master_private_keys', self.master_private_keys, True)
 427
 428     def is_watching_only(self):
 429         return (self.seed == '') and (self.master_private_keys == {})
 430
 431
 432
 433     def account_id(self, account_type, i):
 434         if account_type == '1':
 435             return "m/0'/%d"%i
 436         elif account_type == '2of2':
 437             return "m/1'/%d & m/2'/%d"%(i,i)
 438         elif account_type == '2of3':
 439             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 440         else:
 441             raise BaseException('unknown account type')
 442
 443
 444     def num_accounts(self, account_type):
 445         keys = self.accounts.keys()
 446         i = 0
 447         while True:
 448             account_id = self.account_id(account_type, i)
 449             if account_id not in keys: break
 450             i += 1
 451         return i
 452
 453
 454     def new_account_address(self, account_type = '1'):
 455         i = self.num_accounts(account_type)
 456         k = self.account_id(account_type,i)
 457
 458         addr = self.next_addresses.get(k)
 459         if not addr:
 460             account_id, account = self.next_account(account_type)
 461             addr = account.first_address()
 462             self.next_addresses[k] = addr
 463             self.storage.put('next_addresses',self.next_addresses)
 464
 465         return k, addr
 466
 467
 468     def next_account(self, account_type = '1'):
 469
 470         i = self.num_accounts(account_type)
 471         account_id = self.account_id(account_type,i)
 472
 473         if account_type is '1':
 474             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 475             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 476             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 477
 478         elif account_type == '2of2':
 479             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 480             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 481             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 482             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 483             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 484
 485         elif account_type == '2of3':
 486             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 487             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 488             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 489             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 490             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 491             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 492             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 493
 494         return account_id, account
 495
 496
 497     def set_label(self, name, text = None):
 498         changed = False
 499         old_text = self.labels.get(name)
 500         if text:
 501             if old_text != text:
 502                 self.labels[name] = text
 503                 changed = True
 504         else:
 505             if old_text:
 506                 self.labels.pop(name)
 507                 changed = True
 508
 509         if changed:
 510             self.storage.put('labels', self.labels, True)
 511
 512         run_hook('set_label', name, text, changed)
 513         return changed
 514
 515
 516
 517     def create_account(self, account_type = '1', name = None):
 518         k, account = self.next_account(account_type)
 519         if k in self.pending_accounts:
 520             self.pending_accounts.pop(k)
 521             self.storage.put('pending_accounts', self.pending_accounts)
 522
 523         self.accounts[k] = account
 524         self.save_accounts()
 525         if name:
 526             self.set_label(k, name)
 527
 528
 529     def create_old_account(self, mpk):
 530         self.storage.put('master_public_key', mpk, True)
 531         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 532         self.save_accounts()
 533
 534
 535     def save_accounts(self):
 536         d = {}
 537         for k, v in self.accounts.items():
 538             d[k] = v.dump()
 539         self.storage.put('accounts', d, True)
 540
 541
 542
 543     def load_accounts(self):
 544         d = self.storage.get('accounts', {})
 545         self.accounts = {}
 546         for k, v in d.items():
 547             if k == 0:
 548                 v['mpk'] = self.storage.get('master_public_key')
 549                 self.accounts[k] = OldAccount(v)
 550             elif '&' in k:
 551                 self.accounts[k] = BIP32_Account_2of2(v)
 552             else:
 553                 self.accounts[k] = BIP32_Account(v)
 554
 555         self.pending_accounts = self.storage.get('pending_accounts',{})
 556
 557
 558     def delete_pending_account(self, k):
 559         self.pending_accounts.pop(k)
 560         self.storage.put('pending_accounts', self.pending_accounts)
 561
 562     def account_is_pending(self, k):
 563         return k in self.pending_accounts
 564
 565     def create_pending_account(self, acct_type, name):
 566         k, addr = self.new_account_address(acct_type)
 567         self.set_label(k, name)
 568         self.pending_accounts[k] = addr
 569         self.storage.put('pending_accounts', self.pending_accounts)
 570
 571     def get_pending_accounts(self):
 572         return self.pending_accounts.items()
 573
 574
 575     def addresses(self, include_change = True, _next=True):
 576         o = self.get_account_addresses(-1, include_change)
 577         for a in self.accounts.keys():
 578             o += self.get_account_addresses(a, include_change)
 579
 580         if _next:
 581             for addr in self.next_addresses.values():
 582                 if addr not in o:
 583                     o += [addr]
 584         return o
 585
 586
 587     def is_mine(self, address):
 588         return address in self.addresses(True)
 589
 590
 591     def is_change(self, address):
 592         if not self.is_mine(address): return False
 593         if address in self.imported_keys.keys(): return False
 594         acct, s = self.get_address_index(address)
 595         if s is None: return False
 596         return s[0] == 1
 597
 598     def get_master_public_key(self):
 599         if self.seed_version == 4:
 600             return self.storage.get("master_public_key")
 601         else:
 602             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 603             return repr((c, K))
 604
 605     def get_master_private_key(self, account, password):
 606         master_k = pw_decode( self.master_private_keys[account], password)
 607         master_c, master_K, master_Kc = self.master_public_keys[account]
 608         try:
 609             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 610             assert K.encode('hex') == master_K
 611         except:
 612             raise BaseException("Invalid password")
 613         return master_k
 614
 615
 616     def get_address_index(self, address):
 617         if address in self.imported_keys.keys():
 618             return -1, None
 619
 620         for account in self.accounts.keys():
 621             for for_change in [0,1]:
 622                 addresses = self.accounts[account].get_addresses(for_change)
 623                 for addr in addresses:
 624                     if address == addr:
 625                         return account, (for_change, addresses.index(addr))
 626
 627         for k,v in self.next_addresses.items():
 628             if v == address:
 629                 return k, (0,0)
 630
 631         raise BaseException("Address not found", address)
 632
 633
 634     def get_roots(self, account):
 635         roots = []
 636         for a in account.split('&'):
 637             s = a.strip()
 638             m = re.match("(m/\d+'/)(\d+)", s)
 639             roots.append( m.group(1) )
 640         return roots
 641
 642     def is_seeded(self, account):
 643         if type(account) is int:
 644             return self.seed is not None
 645
 646         for root in self.get_roots(account):
 647             if root not in self.master_private_keys.keys():
 648                 return False
 649         return True
 650
 651     def rebase_sequence(self, account, sequence):
 652         c, i = sequence
 653         dd = []
 654         for a in account.split('&'):
 655             s = a.strip()
 656             m = re.match("(m/\d+'/)(\d+)", s)
 657             root = m.group(1)
 658             num = int(m.group(2))
 659             dd.append( (root, [num,c,i] ) )
 660         return dd
 661
 662
 663     def get_keyID(self, account, sequence):
 664         if account == 0:
 665             a, b = sequence
 666             mpk = self.storage.get('master_public_key')
 667             return 'old(%s,%d,%d)'%(mpk,a,b)
 668
 669         rs = self.rebase_sequence(account, sequence)
 670         dd = []
 671         for root, public_sequence in rs:
 672             c, K, _ = self.master_public_keys[root]
 673             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 674             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 675         return '&'.join(dd)
 676
 677
 678
 679     def get_seed(self, password):
 680         s = pw_decode(self.seed, password)
 681         if self.seed_version == 4:
 682             seed = s
 683             self.accounts[0].check_seed(seed)
 684         else:
 685             seed = mnemonic_hash(s)
 686         return seed
 687
 688
 689     def get_mnemonic(self, password):
 690         import mnemonic
 691         s = pw_decode(self.seed, password)
 692         if self.seed_version == 4:
 693             return ' '.join(mnemonic.mn_encode(s))
 694         else:
 695             return s
 696
 697
 698
 699     def get_private_key(self, address, password):
 700         out = []
 701         if address in self.imported_keys.keys():
 702             out.append( pw_decode( self.imported_keys[address], password ) )
 703         else:
 704             account, sequence = self.get_address_index(address)
 705             if account == 0:
 706                 seed = self.get_seed(password)
 707                 pk = self.accounts[account].get_private_key(seed, sequence)
 708                 out.append(pk)
 709                 return out
 710
 711             # assert address == self.accounts[account].get_address(*sequence)
 712             rs = self.rebase_sequence( account, sequence)
 713             for root, public_sequence in rs:
 714
 715                 if root not in self.master_private_keys.keys(): continue
 716                 master_k = self.get_master_private_key(root, password)
 717                 master_c, _, _ = self.master_public_keys[root]
 718                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 719                 out.append(pk)
 720
 721         return out
 722
 723
 724     def add_keypairs_from_wallet(self, tx, keypairs, password):
 725         for txin in tx.inputs:
 726             address = txin['address']
 727             private_keys = self.get_private_key(address, password)
 728             for sec in private_keys:
 729                 pubkey = public_key_from_private_key(sec)
 730                 keypairs[ pubkey ] = sec
 731                 if address in self.imported_keys.keys():
 732                     txin['redeemPubkey'] = pubkey
 733
 734
 735     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 736         for txin in tx.inputs:
 737             keyid = txin.get('KeyID')
 738             if keyid:
 739
 740                 if self.seed_version==4:
 741                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 742                     if not m: continue
 743                     mpk = m.group(1)
 744                     if mpk != self.storage.get('master_public_key'): continue
 745                     index = int(m.group(2))
 746                     num = int(m.group(3))
 747                     account = self.accounts[0]
 748                     addr = account.get_address(index, num)
 749                     txin['address'] = addr # fixme: side effect
 750                     pk = self.get_private_key(addr, password)
 751                     for sec in pk:
 752                         pubkey = public_key_from_private_key(sec)
 753                         keypairs[pubkey] = sec
 754                     continue
 755
 756
 757                 roots = []
 758                 for s in keyid.split('&'):
 759                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 760                     if not m: continue
 761                     c = m.group(1)
 762                     K = m.group(2)
 763                     sequence = m.group(3)
 764                     root = self.find_root_by_master_key(c,K)
 765                     if not root: continue
 766                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 767                     root = root + '%d'%sequence[0]
 768                     sequence = sequence[1:]
 769                     roots.append((root,sequence))
 770
 771                 account_id = " & ".join( map(lambda x:x[0], roots) )
 772                 account = self.accounts.get(account_id)
 773                 if not account: continue
 774                 addr = account.get_address(*sequence)
 775                 txin['address'] = addr # fixme: side effect
 776                 pk = self.get_private_key(addr, password)
 777                 for sec in pk:
 778                     pubkey = public_key_from_private_key(sec)
 779                     keypairs[pubkey] = sec
 780
 781
 782
 783     def signrawtransaction(self, tx, input_info, private_keys, password):
 784
 785         # check that the password is correct
 786         seed = self.get_seed(password)
 787
 788         # add input info
 789         tx.add_input_info(input_info)
 790
 791         # add redeem script for coins that are in the wallet
 792         # FIXME: add redeemPubkey too!
 793         unspent_coins = self.get_unspent_coins()
 794         for txin in tx.inputs:
 795             for item in unspent_coins:
 796                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 797                     print_error( "tx input is in unspent coins" )
 798                     txin['scriptPubKey'] = item['scriptPubKey']
 799                     account, sequence = self.get_address_index(item['address'])
 800                     if account != -1:
 801                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 802                         print_error("added redeemScript", txin['redeemScript'])
 803                     break
 804
 805
 806         # build a list of public/private keys
 807         keypairs = {}
 808
 809         # add private keys from parameter
 810         for sec in private_keys:
 811             pubkey = public_key_from_private_key(sec)
 812             keypairs[ pubkey ] = sec
 813
 814         # add private_keys from KeyID
 815         self.add_keypairs_from_KeyID(tx, keypairs, password)
 816
 817         # add private keys from wallet
 818         self.add_keypairs_from_wallet(tx, keypairs, password)
 819         self.sign_transaction(tx, keypairs)
 820
 821
 822     def sign_message(self, address, message, password):
 823         keys = self.get_private_key(address, password)
 824         assert len(keys) == 1
 825         sec = keys[0]
 826         key = regenerate_key(sec)
 827         compressed = is_compressed(sec)
 828         return key.sign_message(message, compressed, address)
 829
 830
 831     def change_gap_limit(self, value):
 832         if value >= self.gap_limit:
 833             self.gap_limit = value
 834             self.storage.put('gap_limit', self.gap_limit, True)
 835             #self.interface.poke('synchronizer')
 836             return True
 837
 838         elif value >= self.min_acceptable_gap():
 839             for key, account in self.accounts.items():
 840                 addresses = account[0]
 841                 k = self.num_unused_trailing_addresses(addresses)
 842                 n = len(addresses) - k + value
 843                 addresses = addresses[0:n]
 844                 self.accounts[key][0] = addresses
 845
 846             self.gap_limit = value
 847             self.storage.put('gap_limit', self.gap_limit, True)
 848             self.save_accounts()
 849             return True
 850         else:
 851             return False
 852
 853     def num_unused_trailing_addresses(self, addresses):
 854         k = 0
 855         for a in addresses[::-1]:
 856             if self.history.get(a):break
 857             k = k + 1
 858         return k
 859
 860     def min_acceptable_gap(self):
 861         # fixme: this assumes wallet is synchronized
 862         n = 0
 863         nmax = 0
 864
 865         for account in self.accounts.values():
 866             addresses = account.get_addresses(0)
 867             k = self.num_unused_trailing_addresses(addresses)
 868             for a in addresses[0:-k]:
 869                 if self.history.get(a):
 870                     n = 0
 871                 else:
 872                     n += 1
 873                     if n > nmax: nmax = n
 874         return nmax + 1
 875
 876
 877     def address_is_old(self, address):
 878         age = -1
 879         h = self.history.get(address, [])
 880         if h == ['*']:
 881             return True
 882         for tx_hash, tx_height in h:
 883             if tx_height == 0:
 884                 tx_age = 0
 885             else:
 886                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 887             if tx_age > age:
 888                 age = tx_age
 889         return age > 2
 890
 891
 892     def synchronize_sequence(self, account, for_change):
 893         limit = self.gap_limit_for_change if for_change else self.gap_limit
 894         new_addresses = []
 895         while True:
 896             addresses = account.get_addresses(for_change)
 897             if len(addresses) < limit:
 898                 address = account.create_new_address(for_change)
 899                 self.history[address] = []
 900                 new_addresses.append( address )
 901                 continue
 902
 903             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 904                 break
 905             else:
 906                 address = account.create_new_address(for_change)
 907                 self.history[address] = []
 908                 new_addresses.append( address )
 909
 910         return new_addresses
 911
 912
 913
 914     def create_pending_accounts(self):
 915         for account_type in ['1','2of2','2of3']:
 916             if not self.has_master_public_keys(account_type):
 917                 continue
 918             k, a = self.new_account_address(account_type)
 919             if self.address_is_old(a):
 920                 print_error( "creating account", a )
 921                 self.create_account(account_type)
 922                 self.next_addresses.pop(k)
 923
 924
 925     def synchronize_account(self, account):
 926         new = []
 927         new += self.synchronize_sequence(account, 0)
 928         new += self.synchronize_sequence(account, 1)
 929         return new
 930
 931
 932     def synchronize(self):
 933         if self.master_public_keys:
 934             self.create_pending_accounts()
 935         new = []
 936         for account in self.accounts.values():
 937             new += self.synchronize_account(account)
 938         if new:
 939             self.save_accounts()
 940             self.storage.put('addr_history', self.history, True)
 941         return new
 942
 943
 944     def is_found(self):
 945         return self.history.values() != [[]] * len(self.history)
 946
 947
 948     def add_contact(self, address, label=None):
 949         self.addressbook.append(address)
 950         self.storage.put('contacts', self.addressbook, True)
 951         if label:
 952             self.set_label(address, label)
 953
 954
 955     def delete_contact(self, addr):
 956         if addr in self.addressbook:
 957             self.addressbook.remove(addr)
 958             self.storage.put('addressbook', self.addressbook, True)
 959
 960
 961     def fill_addressbook(self):
 962         for tx_hash, tx in self.transactions.items():
 963             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 964             if is_send:
 965                 for addr, v in tx.outputs:
 966                     if not self.is_mine(addr) and addr not in self.addressbook:
 967                         self.addressbook.append(addr)
 968         # redo labels
 969         # self.update_tx_labels()
 970
 971     def get_num_tx(self, address):
 972         n = 0
 973         for tx in self.transactions.values():
 974             if address in map(lambda x:x[0], tx.outputs): n += 1
 975         return n
 976
 977
 978     def get_address_flags(self, addr):
 979         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 980         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 981         return flags
 982
 983
 984     def get_tx_value(self, tx, account=None):
 985         domain = self.get_account_addresses(account)
 986         return tx.get_value(domain, self.prevout_values)
 987
 988
 989     def update_tx_outputs(self, tx_hash):
 990         tx = self.transactions.get(tx_hash)
 991
 992         for i, (addr, value) in enumerate(tx.outputs):
 993             key = tx_hash+ ':%d'%i
 994             self.prevout_values[key] = value
 995
 996         for item in tx.inputs:
 997             if self.is_mine(item.get('address')):
 998                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 999                 self.spent_outputs.append(key)
1000
1001
1002     def get_addr_balance(self, address):
1003         assert self.is_mine(address)
1004         h = self.history.get(address,[])
1005         if h == ['*']: return 0,0
1006         c = u = 0
1007         received_coins = []   # list of coins received at address
1008
1009         for tx_hash, tx_height in h:
1010             tx = self.transactions.get(tx_hash)
1011             if not tx: continue
1012
1013             for i, (addr, value) in enumerate(tx.outputs):
1014                 if addr == address:
1015                     key = tx_hash + ':%d'%i
1016                     received_coins.append(key)
1017
1018         for tx_hash, tx_height in h:
1019             tx = self.transactions.get(tx_hash)
1020             if not tx: continue
1021             v = 0
1022
1023             for item in tx.inputs:
1024                 addr = item.get('address')
1025                 if addr == address:
1026                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1027                     value = self.prevout_values.get( key )
1028                     if key in received_coins:
1029                         v -= value
1030
1031             for i, (addr, value) in enumerate(tx.outputs):
1032                 key = tx_hash + ':%d'%i
1033                 if addr == address:
1034                     v += value
1035
1036             if tx_height:
1037                 c += v
1038             else:
1039                 u += v
1040         return c, u
1041
1042
1043     def get_account_name(self, k):
1044         if k == 0:
1045             if self.seed_version == 4:
1046                 name = 'Main account'
1047             else:
1048                 name = 'Old account'
1049         else:
1050             default = "Unnamed account"
1051             m = re.match("m/0'/(\d+)", k)
1052             if m:
1053                 num = m.group(1)
1054                 if num == '0':
1055                     default = "Main account"
1056                 else:
1057                     default = "Account %s"%num
1058
1059             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1060             if m:
1061                 num = m.group(1)
1062                 default = "2of2 account %s"%num
1063             name = self.labels.get(k, default)
1064
1065         return name
1066
1067     def get_account_names(self):
1068         accounts = {}
1069         for k, account in self.accounts.items():
1070             accounts[k] = self.get_account_name(k)
1071         if self.imported_keys:
1072             accounts[-1] = 'Imported keys'
1073         return accounts
1074
1075     def get_account_addresses(self, a, include_change=True):
1076         if a is None:
1077             o = self.addresses(True)
1078         elif a == -1:
1079             o = self.imported_keys.keys()
1080         else:
1081             ac = self.accounts[a]
1082             o = ac.get_addresses(0)
1083             if include_change: o += ac.get_addresses(1)
1084         return o
1085
1086     def get_imported_balance(self):
1087         return self.get_balance(self.imported_keys.keys())
1088
1089     def get_account_balance(self, account):
1090         return self.get_balance(self.get_account_addresses(account))
1091
1092     def get_frozen_balance(self):
1093         return self.get_balance(self.frozen_addresses)
1094
1095     def get_balance(self, domain=None):
1096         if domain is None: domain = self.addresses(True)
1097         cc = uu = 0
1098         for addr in domain:
1099             c, u = self.get_addr_balance(addr)
1100             cc += c
1101             uu += u
1102         return cc, uu
1103
1104
1105     def get_unspent_coins(self, domain=None):
1106         coins = []
1107         if domain is None: domain = self.addresses(True)
1108         for addr in domain:
1109             h = self.history.get(addr, [])
1110             if h == ['*']: continue
1111             for tx_hash, tx_height in h:
1112                 tx = self.transactions.get(tx_hash)
1113                 if tx is None: raise BaseException("Wallet not synchronized")
1114                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1115                 for output in tx.d.get('outputs'):
1116                     if output.get('address') != addr: continue
1117                     key = tx_hash + ":%d" % output.get('prevout_n')
1118                     if key in self.spent_outputs: continue
1119                     output['prevout_hash'] = tx_hash
1120                     output['height'] = tx_height
1121                     output['coinbase'] = is_coinbase
1122                     coins.append((tx_height, output))
1123
1124         # sort by age
1125         if coins:
1126             coins = sorted(coins)
1127             if coins[-1][0] != 0:
1128                 while coins[0][0] == 0:
1129                     coins = coins[1:] + [ coins[0] ]
1130         return [x[1] for x in coins]
1131
1132
1133
1134     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1135         domain = self.get_account_addresses(account) if account else None
1136         return self.choose_tx_inputs( amount, fixed_fee, domain )
1137
1138
1139     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1140         """ todo: minimize tx size """
1141         total = 0
1142         fee = self.fee if fixed_fee is None else fixed_fee
1143         if domain is None:
1144             domain = self.addresses(True)
1145
1146         for i in self.frozen_addresses:
1147             if i in domain: domain.remove(i)
1148
1149         prioritized = []
1150         for i in self.prioritized_addresses:
1151             if i in domain:
1152                 domain.remove(i)
1153                 prioritized.append(i)
1154
1155         coins = self.get_unspent_coins(domain)
1156         prioritized_coins = self.get_unspent_coins(prioritized)
1157
1158         inputs = []
1159         coins = prioritized_coins + coins
1160
1161         for item in coins:
1162             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1163                 continue
1164             addr = item.get('address')
1165             v = item.get('value')
1166             total += v
1167             inputs.append(item)
1168             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1169             if total >= amount + fee: break
1170         else:
1171             inputs = []
1172
1173         return inputs, total, fee
1174
1175
1176     def set_fee(self, fee):
1177         if self.fee != fee:
1178             self.fee = fee
1179             self.storage.put('fee_per_kb', self.fee, True)
1180
1181     def estimated_fee(self, inputs):
1182         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1183         fee = self.fee * int(round(estimated_size/1024.))
1184         if fee == 0: fee = self.fee
1185         return fee
1186
1187
1188     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1189         "add change to a transaction"
1190         change_amount = total - ( amount + fee )
1191         if change_amount > DUST_THRESHOLD:
1192             if not change_addr:
1193
1194                 # send change to one of the accounts involved in the tx
1195                 address = inputs[0].get('address')
1196                 account, _ = self.get_address_index(address)
1197
1198                 if not self.use_change or account == -1:
1199                     change_addr = inputs[-1]['address']
1200                 else:
1201                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1202
1203             # Insert the change output at a random position in the outputs
1204             posn = random.randint(0, len(outputs))
1205             outputs[posn:posn] = [( change_addr,  change_amount)]
1206         return outputs
1207
1208
1209     def get_history(self, address):
1210         with self.lock:
1211             return self.history.get(address)
1212
1213
1214     def get_status(self, h):
1215         if not h: return None
1216         if h == ['*']: return '*'
1217         status = ''
1218         for tx_hash, height in h:
1219             status += tx_hash + ':%d:' % height
1220         return hashlib.sha256( status ).digest().encode('hex')
1221
1222
1223     def receive_tx_callback(self, tx_hash, tx, tx_height):
1224
1225         with self.transaction_lock:
1226             self.add_extra_addresses(tx)
1227             if not self.check_new_tx(tx_hash, tx):
1228                 # may happen due to pruning
1229                 print_error("received transaction that is no longer referenced in history", tx_hash)
1230                 return
1231             self.transactions[tx_hash] = tx
1232             self.network.interface.pending_transactions_for_notifications.append(tx)
1233             self.save_transactions()
1234             if self.verifier and tx_height>0:
1235                 self.verifier.add(tx_hash, tx_height)
1236             self.update_tx_outputs(tx_hash)
1237
1238
1239     def save_transactions(self):
1240         tx = {}
1241         for k,v in self.transactions.items():
1242             tx[k] = str(v)
1243         self.storage.put('transactions', tx, True)
1244
1245     def receive_history_callback(self, addr, hist):
1246
1247         if not self.check_new_history(addr, hist):
1248             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
1249
1250         with self.lock:
1251             self.history[addr] = hist
1252             self.storage.put('addr_history', self.history, True)
1253
1254         if hist != ['*']:
1255             for tx_hash, tx_height in hist:
1256                 if tx_height>0:
1257                     # add it in case it was previously unconfirmed
1258                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1259
1260
1261     def get_tx_history(self, account=None):
1262         if not self.verifier:
1263             return []
1264
1265         with self.transaction_lock:
1266             history = self.transactions.items()
1267             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1268             result = []
1269
1270             balance = 0
1271             for tx_hash, tx in history:
1272                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1273                 if v is not None: balance += v
1274
1275             c, u = self.get_account_balance(account)
1276
1277             if balance != c+u:
1278                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1279
1280             balance = c + u - balance
1281             for tx_hash, tx in history:
1282                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1283                 if not is_relevant:
1284                     continue
1285                 if value is not None:
1286                     balance += value
1287
1288                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1289                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1290
1291         return result
1292
1293
1294     def get_label(self, tx_hash):
1295         label = self.labels.get(tx_hash)
1296         is_default = (label == '') or (label is None)
1297         if is_default: label = self.get_default_label(tx_hash)
1298         return label, is_default
1299
1300
1301     def get_default_label(self, tx_hash):
1302         tx = self.transactions.get(tx_hash)
1303         default_label = ''
1304         if tx:
1305             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1306             if is_mine:
1307                 for o in tx.outputs:
1308                     o_addr, _ = o
1309                     if not self.is_mine(o_addr):
1310                         try:
1311                             default_label = self.labels[o_addr]
1312                         except KeyError:
1313                             default_label = o_addr
1314                         break
1315                 else:
1316                     default_label = '(internal)'
1317             else:
1318                 for o in tx.outputs:
1319                     o_addr, _ = o
1320                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1321                         break
1322                 else:
1323                     for o in tx.outputs:
1324                         o_addr, _ = o
1325                         if self.is_mine(o_addr):
1326                             break
1327                     else:
1328                         o_addr = None
1329
1330                 if o_addr:
1331                     dest_label = self.labels.get(o_addr)
1332                     try:
1333                         default_label = self.labels[o_addr]
1334                     except KeyError:
1335                         default_label = o_addr
1336
1337         return default_label
1338
1339
1340     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1341         for address, x in outputs:
1342             assert is_valid(address)
1343         amount = sum( map(lambda x:x[1], outputs) )
1344         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1345         if not inputs:
1346             raise ValueError("Not enough funds")
1347         self.add_input_info(inputs)
1348         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1349         return Transaction.from_io(inputs, outputs)
1350
1351
1352     def mktx_from_account(self, outputs, password, fee=None, account=None):
1353         domain = self.get_account_addresses(account) if account else None
1354         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1355
1356
1357     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1358         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1359         keypairs = {}
1360         self.add_keypairs_from_wallet(tx, keypairs, password)
1361         if keypairs:
1362             self.sign_transaction(tx, keypairs)
1363         return tx
1364
1365
1366     def add_input_info(self, inputs):
1367         for txin in inputs:
1368             address = txin['address']
1369             if address in self.imported_keys.keys():
1370                 continue
1371             account, sequence = self.get_address_index(address)
1372             txin['KeyID'] = self.get_keyID(account, sequence)
1373             redeemScript = self.accounts[account].redeem_script(sequence)
1374             if redeemScript:
1375                 txin['redeemScript'] = redeemScript
1376             else:
1377                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1378
1379
1380     def sign_transaction(self, tx, keypairs):
1381         tx.sign(keypairs)
1382         run_hook('sign_transaction', tx)
1383
1384
1385     def sendtx(self, tx):
1386         # synchronous
1387         h = self.send_tx(tx)
1388         self.tx_event.wait()
1389         return self.receive_tx(h)
1390
1391     def send_tx(self, tx):
1392         # asynchronous
1393         self.tx_event.clear()
1394         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1395         return tx.hash()
1396
1397     def on_broadcast(self, i, r):
1398         self.tx_result = r.get('result')
1399         self.tx_event.set()
1400
1401     def receive_tx(self,tx_hash):
1402         out = self.tx_result
1403         if out != tx_hash:
1404             return False, "error: " + out
1405         return True, out
1406
1407
1408
1409     def update_password(self, old_password, new_password):
1410         if new_password == '': new_password = None
1411         # this will throw an exception if unicode cannot be converted
1412         decoded = pw_decode(self.seed, old_password)
1413         self.seed = pw_encode( decoded, new_password)
1414         self.storage.put('seed', self.seed, True)
1415         self.use_encryption = (new_password != None)
1416         self.storage.put('use_encryption', self.use_encryption,True)
1417         for k in self.imported_keys.keys():
1418             a = self.imported_keys[k]
1419             b = pw_decode(a, old_password)
1420             c = pw_encode(b, new_password)
1421             self.imported_keys[k] = c
1422         self.storage.put('imported_keys', self.imported_keys, True)
1423
1424         for k, v in self.master_private_keys.items():
1425             b = pw_decode(v, old_password)
1426             c = pw_encode(b, new_password)
1427             self.master_private_keys[k] = c
1428         self.storage.put('master_private_keys', self.master_private_keys, True)
1429
1430
1431     def freeze(self,addr):
1432         if self.is_mine(addr) and addr not in self.frozen_addresses:
1433             self.unprioritize(addr)
1434             self.frozen_addresses.append(addr)
1435             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1436             return True
1437         else:
1438             return False
1439
1440     def unfreeze(self,addr):
1441         if self.is_mine(addr) and addr in self.frozen_addresses:
1442             self.frozen_addresses.remove(addr)
1443             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1444             return True
1445         else:
1446             return False
1447
1448     def prioritize(self,addr):
1449         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1450             self.unfreeze(addr)
1451             self.prioritized_addresses.append(addr)
1452             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1453             return True
1454         else:
1455             return False
1456
1457     def unprioritize(self,addr):
1458         if self.is_mine(addr) and addr in self.prioritized_addresses:
1459             self.prioritized_addresses.remove(addr)
1460             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1461             return True
1462         else:
1463             return False
1464
1465
1466     def set_verifier(self, verifier):
1467         self.verifier = verifier
1468
1469         # review transactions that are in the history
1470         for addr, hist in self.history.items():
1471             if hist == ['*']: continue
1472             for tx_hash, tx_height in hist:
1473                 if tx_height>0:
1474                     # add it in case it was previously unconfirmed
1475                     self.verifier.add(tx_hash, tx_height)
1476
1477
1478         # if we are on a pruning server, remove unverified transactions
1479         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1480         for tx_hash in self.transactions.keys():
1481             if tx_hash not in vr:
1482                 self.transactions.pop(tx_hash)
1483
1484
1485
1486     def check_new_history(self, addr, hist):
1487
1488         # check that all tx in hist are relevant
1489         if hist != ['*']:
1490             for tx_hash, height in hist:
1491                 tx = self.transactions.get(tx_hash)
1492                 if not tx: continue
1493                 if not tx.has_address(addr):
1494                     return False
1495
1496         # check that we are not "orphaning" a transaction
1497         old_hist = self.history.get(addr,[])
1498         if old_hist == ['*']: return True
1499
1500         for tx_hash, height in old_hist:
1501             if tx_hash in map(lambda x:x[0], hist): continue
1502             found = False
1503             for _addr, _hist in self.history.items():
1504                 if _addr == addr: continue
1505                 if _hist == ['*']: continue
1506                 _tx_hist = map(lambda x:x[0], _hist)
1507                 if tx_hash in _tx_hist:
1508                     found = True
1509                     break
1510
1511             if not found:
1512                 tx = self.transactions.get(tx_hash)
1513                 # tx might not be there
1514                 if not tx: continue
1515
1516                 # already verified?
1517                 if self.verifier.get_height(tx_hash):
1518                     continue
1519                 # unconfirmed tx
1520                 print_error("new history is orphaning transaction:", tx_hash)
1521                 # check that all outputs are not mine, request histories
1522                 ext_requests = []
1523                 for _addr, _v in tx.outputs:
1524                     # assert not self.is_mine(_addr)
1525                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1526
1527                 ext_h = self.network.synchronous_get(ext_requests)
1528                 print_error("sync:", ext_requests, ext_h)
1529                 height = None
1530                 for h in ext_h:
1531                     if h == ['*']: continue
1532                     for item in h:
1533                         if item.get('tx_hash') == tx_hash:
1534                             height = item.get('height')
1535                 if height:
1536                     print_error("found height for", tx_hash, height)
1537                     self.verifier.add(tx_hash, height)
1538                 else:
1539                     print_error("removing orphaned tx from history", tx_hash)
1540                     self.transactions.pop(tx_hash)
1541
1542         return True
1543
1544
1545
1546     def check_new_tx(self, tx_hash, tx):
1547         # 1 check that tx is referenced in addr_history.
1548         addresses = []
1549         for addr, hist in self.history.items():
1550             if hist == ['*']:continue
1551             for txh, height in hist:
1552                 if txh == tx_hash:
1553                     addresses.append(addr)
1554
1555         if not addresses:
1556             return False
1557
1558         # 2 check that referencing addresses are in the tx
1559         for addr in addresses:
1560             if not tx.has_address(addr):
1561                 return False
1562
1563         return True
1564
1565
1566     def start_threads(self, network):
1567         from verifier import TxVerifier
1568         self.network = network
1569         if self.network:
1570             self.verifier = TxVerifier(self.network, self.storage)
1571             self.verifier.start()
1572             self.set_verifier(self.verifier)
1573             self.synchronizer = WalletSynchronizer(self, network)
1574             self.synchronizer.start()
1575         else:
1576             self.verifier = None
1577             self.synchronizer =None
1578
1579     def stop_threads(self):
1580         if self.network:
1581             self.verifier.stop()
1582             self.synchronizer.stop()
1583
1584
1585     def restore(self, callback):
1586         from i18n import _
1587         def wait_for_wallet():
1588             self.set_up_to_date(False)
1589             while not self.is_up_to_date():
1590                 msg = "%s\n%s %d\n%s %.1f"%(
1591                     _("Please wait..."),
1592                     _("Addresses generated:"),
1593                     len(self.addresses(True)),
1594                     _("Kilobytes received:"),
1595                     self.network.interface.bytes_received/1024.)
1596
1597                 apply(callback, (msg,))
1598                 time.sleep(0.1)
1599
1600         def wait_for_network():
1601             while not self.network.is_connected():
1602                 msg = "%s \n" % (_("Connecting..."))
1603                 apply(callback, (msg,))
1604                 time.sleep(0.1)
1605
1606         # wait until we are connected, because the user might have selected another server
1607         if self.network:
1608             wait_for_network()
1609
1610         self.create_accounts()
1611
1612         if self.network:
1613             wait_for_wallet()
1614         else:
1615             self.synchronize()
1616
1617         self.fill_addressbook()
1618
1619
1620
1621
1622 class WalletSynchronizer(threading.Thread):
1623
1624
1625     def __init__(self, wallet, network):
1626         threading.Thread.__init__(self)
1627         self.daemon = True
1628         self.wallet = wallet
1629         self.network = network
1630         self.was_updated = True
1631         self.running = False
1632         self.lock = threading.Lock()
1633         self.queue = Queue.Queue()
1634
1635     def stop(self):
1636         with self.lock: self.running = False
1637
1638     def is_running(self):
1639         with self.lock: return self.running
1640
1641
1642     def subscribe_to_addresses(self, addresses):
1643         messages = []
1644         for addr in addresses:
1645             messages.append(('blockchain.address.subscribe', [addr]))
1646         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1647
1648
1649     def run(self):
1650         with self.lock:
1651             self.running = True
1652
1653         while self.is_running():
1654
1655             if not self.network.is_connected():
1656                 print_error("synchronizer: waiting for interface")
1657                 self.network.wait_until_connected()
1658
1659             self.run_interface(self.network.interface)
1660
1661
1662     def run_interface(self, interface):
1663
1664         print_error("synchronizer: connected to", interface.server)
1665
1666         requested_tx = []
1667         missing_tx = []
1668         requested_histories = {}
1669
1670         # request any missing transactions
1671         for history in self.wallet.history.values():
1672             if history == ['*']: continue
1673             for tx_hash, tx_height in history:
1674                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1675                     missing_tx.append( (tx_hash, tx_height) )
1676
1677         if missing_tx:
1678             print_error("missing tx", missing_tx)
1679
1680         # subscriptions
1681         self.subscribe_to_addresses(self.wallet.addresses(True))
1682
1683         while self.is_running():
1684             # 1. create new addresses
1685             new_addresses = self.wallet.synchronize()
1686
1687             # request missing addresses
1688             if new_addresses:
1689                 self.subscribe_to_addresses(new_addresses)
1690
1691             # request missing transactions
1692             for tx_hash, tx_height in missing_tx:
1693                 if (tx_hash, tx_height) not in requested_tx:
1694                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1695                     requested_tx.append( (tx_hash, tx_height) )
1696             missing_tx = []
1697
1698             # detect if situation has changed
1699             if interface.is_up_to_date() and self.queue.empty():
1700                 if not self.wallet.is_up_to_date():
1701                     self.wallet.set_up_to_date(True)
1702                     self.was_updated = True
1703             else:
1704                 if self.wallet.is_up_to_date():
1705                     self.wallet.set_up_to_date(False)
1706                     self.was_updated = True
1707
1708             if self.was_updated:
1709                 self.wallet.network.trigger_callback('updated')
1710                 self.was_updated = False
1711
1712             # 2. get a response
1713             try:
1714                 r = self.queue.get(block=True, timeout=1)
1715             except Queue.Empty:
1716                 continue
1717
1718             if interface != self.network.interface:
1719                 break
1720
1721             if not r:
1722                 continue
1723
1724             # 3. handle response
1725             method = r['method']
1726             params = r['params']
1727             result = r.get('result')
1728             error = r.get('error')
1729             if error:
1730                 print "error", r
1731                 continue
1732
1733             if method == 'blockchain.address.subscribe':
1734                 addr = params[0]
1735                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1736                     if requested_histories.get(addr) is None:
1737                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1738                         requested_histories[addr] = result
1739
1740             elif method == 'blockchain.address.get_history':
1741                 addr = params[0]
1742                 print_error("receiving history", addr, result)
1743                 if result == ['*']:
1744                     assert requested_histories.pop(addr) == '*'
1745                     self.wallet.receive_history_callback(addr, result)
1746                 else:
1747                     hist = []
1748                     # check that txids are unique
1749                     txids = []
1750                     for item in result:
1751                         tx_hash = item['tx_hash']
1752                         if tx_hash not in txids:
1753                             txids.append(tx_hash)
1754                             hist.append( (tx_hash, item['height']) )
1755
1756                     if len(hist) != len(result):
1757                         raise BaseException("error: server sent history with non-unique txid", result)
1758
1759                     # check that the status corresponds to what was announced
1760                     rs = requested_histories.pop(addr)
1761                     if self.wallet.get_status(hist) != rs:
1762                         raise BaseException("error: status mismatch: %s"%addr)
1763
1764                     # store received history
1765                     self.wallet.receive_history_callback(addr, hist)
1766
1767                     # request transactions that we don't have
1768                     for tx_hash, tx_height in hist:
1769                         if self.wallet.transactions.get(tx_hash) is None:
1770                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1771                                 missing_tx.append( (tx_hash, tx_height) )
1772
1773             elif method == 'blockchain.transaction.get':
1774                 tx_hash = params[0]
1775                 tx_height = params[1]
1776                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1777                 tx = Transaction(result)
1778                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1779                 self.was_updated = True
1780                 requested_tx.remove( (tx_hash, tx_height) )
1781                 print_error("received tx:", tx_hash, len(tx.raw))
1782
1783             else:
1784                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1785
1786             if self.was_updated and not requested_tx:
1787                 self.wallet.network.trigger_callback('updated')
1788                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1789
1790                 self.was_updated = False