lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         return self.data.get(key, default)
 129
 130     def put(self, key, value, save = True):
 131
 132         with self.lock:
 133             if value is not None:
 134                 self.data[key] = value
 135             else:
 136                 self.data.pop(key)
 137             if save:
 138                 self.write()
 139
 140     def write(self):
 141         s = repr(self.data)
 142         f = open(self.path,"w")
 143         f.write( s )
 144         f.close()
 145         if 'ANDROID_DATA' not in os.environ:
 146             import stat
 147             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 148
 149
 150 class Wallet:
 151
 152     def __init__(self, storage):
 153
 154         self.storage = storage
 155         self.electrum_version = ELECTRUM_VERSION
 156         self.gap_limit_for_change = 3 # constant
 157
 158         # saved fields
 159         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 160
 161         self.gap_limit             = storage.get('gap_limit', 5)
 162         self.use_change            = storage.get('use_change',True)
 163         self.use_encryption        = storage.get('use_encryption', False)
 164         self.seed                  = storage.get('seed', '')               # encrypted
 165         self.labels                = storage.get('labels', {})
 166         self.frozen_addresses      = storage.get('frozen_addresses',[])
 167         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 168         self.addressbook           = storage.get('contacts', [])
 169
 170         self.imported_keys         = storage.get('imported_keys',{})
 171         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 172
 173         self.fee                   = int(storage.get('fee_per_kb',20000))
 174
 175         self.master_public_keys = storage.get('master_public_keys',{})
 176         self.master_private_keys = storage.get('master_private_keys', {})
 177
 178         self.next_addresses = storage.get('next_addresses',{})
 179
 180         if self.seed_version not in [4, 6]:
 181             msg = "This wallet seed is not supported."
 182             if self.seed_version in [5]:
 183                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 184                 print msg
 185                 sys.exit(1)
 186
 187
 188         self.load_accounts()
 189
 190         self.transactions = {}
 191         tx_list = self.storage.get('transactions',{})
 192         for k,v in tx_list.items():
 193             try:
 194                 tx = Transaction(v)
 195             except Exception:
 196                 print_msg("Warning: Cannot deserialize transactions. skipping")
 197                 continue
 198
 199             self.add_extra_addresses(tx)
 200             self.transactions[k] = tx
 201
 202         for h,tx in self.transactions.items():
 203             if not self.check_new_tx(h, tx):
 204                 print_error("removing unreferenced tx", h)
 205                 self.transactions.pop(h)
 206
 207
 208         # not saved
 209         self.prevout_values = {}     # my own transaction outputs
 210         self.spent_outputs = []
 211
 212         # spv
 213         self.verifier = None
 214
 215         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 216         # interface.is_up_to_date() returns true when all requests have been answered and processed
 217         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 218
 219         self.up_to_date = False
 220         self.lock = threading.Lock()
 221         self.transaction_lock = threading.Lock()
 222         self.tx_event = threading.Event()
 223
 224         for tx_hash, tx in self.transactions.items():
 225             self.update_tx_outputs(tx_hash)
 226
 227
 228     def add_extra_addresses(self, tx):
 229         h = tx.hash()
 230         # find the address corresponding to pay-to-pubkey inputs
 231         tx.add_extra_addresses(self.transactions)
 232         for o in tx.d.get('outputs'):
 233             if o.get('is_pubkey'):
 234                 for tx2 in self.transactions.values():
 235                     tx2.add_extra_addresses({h:tx})
 236
 237
 238
 239
 240     def set_up_to_date(self,b):
 241         with self.lock: self.up_to_date = b
 242
 243
 244     def is_up_to_date(self):
 245         with self.lock: return self.up_to_date
 246
 247
 248     def update(self):
 249         self.up_to_date = False
 250         while not self.is_up_to_date():
 251             time.sleep(0.1)
 252
 253
 254     def import_key(self, sec, password):
 255         # check password
 256         seed = self.get_seed(password)
 257         try:
 258             address = address_from_private_key(sec)
 259         except Exception:
 260             raise Exception('Invalid private key')
 261
 262         if self.is_mine(address):
 263             raise Exception('Address already in wallet')
 264
 265         # store the originally requested keypair into the imported keys table
 266         self.imported_keys[address] = pw_encode(sec, password )
 267         self.storage.put('imported_keys', self.imported_keys, True)
 268         if self.synchronizer:
 269             self.synchronizer.subscribe_to_addresses([address])
 270         return address
 271
 272     def delete_imported_key(self, addr):
 273         if addr in self.imported_keys:
 274             self.imported_keys.pop(addr)
 275             self.storage.put('imported_keys', self.imported_keys, True)
 276
 277
 278     def make_seed(self):
 279         import mnemonic, ecdsa
 280         entropy = ecdsa.util.randrange( pow(2,160) )
 281         nonce = 0
 282         while True:
 283             ss = "%040x"%(entropy+nonce)
 284             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 285             # we keep only 13 words, that's approximately 139 bits of entropy
 286             words = mnemonic.mn_encode(s)[0:13]
 287             seed = ' '.join(words)
 288             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 289                 break  # this removes 12 bits of entropy
 290             nonce += 1
 291
 292         return seed
 293
 294
 295     def init_seed(self, seed):
 296         import mnemonic
 297
 298         if self.seed:
 299             raise Exception("a seed exists")
 300
 301         if not seed:
 302             self.seed = random_seed(128)
 303             self.seed_version = 4
 304             return
 305
 306         #if not seed:
 307         #    self.seed = self.make_seed()
 308         #    self.seed_version = SEED_VERSION
 309         #    return
 310
 311         # find out what kind of wallet we are
 312         try:
 313             seed.decode('hex')
 314             self.seed_version = 4
 315             self.seed = str(seed)
 316             return
 317         except Exception:
 318             pass
 319
 320         words = seed.split()
 321         self.seed_version = 4
 322         self.seed = mnemonic.mn_decode(words)
 323
 324         #try:
 325         #    mnemonic.mn_decode(words)
 326         #    uses_electrum_words = True
 327         #except Exception:
 328         #    uses_electrum_words = False
 329         #
 330         #if uses_electrum_words and len(words) != 13:
 331         #    self.seed_version = 4
 332         #    self.seed = mnemonic.mn_decode(words)
 333         #else:
 334         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 335         #    self.seed_version = SEED_VERSION
 336         #    self.seed = seed
 337
 338
 339     def save_seed(self):
 340         self.storage.put('seed', self.seed, True)
 341         self.storage.put('seed_version', self.seed_version, True)
 342
 343     def create_watching_only_wallet(self, params):
 344         K0, c0 = params
 345         if not K0:
 346             return
 347
 348         if not c0:
 349             self.seed_version = 4
 350             self.storage.put('seed_version', self.seed_version, True)
 351             self.create_old_account(K0)
 352             return
 353
 354         cK0 = ""
 355         self.master_public_keys = {
 356             "m/0'/": (c0, K0, cK0),
 357             }
 358         self.storage.put('master_public_keys', self.master_public_keys, True)
 359         self.storage.put('seed_version', self.seed_version, True)
 360         self.create_account('1','Main account')
 361
 362
 363     def create_accounts(self):
 364         if self.seed_version == 4:
 365             mpk = OldAccount.mpk_from_seed(self.seed)
 366             self.create_old_account(mpk)
 367         else:
 368             # create default account
 369             self.create_master_keys('1')
 370             self.create_account('1','Main account')
 371
 372
 373     def create_master_keys(self, account_type):
 374         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 375         if account_type == '1':
 376             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 377             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 378             self.master_private_keys["m/0'/"] = k0
 379         elif account_type == '2of2':
 380             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 381             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 382             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 383             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 384             self.master_private_keys["m/1'/"] = k1
 385             self.master_private_keys["m/2'/"] = k2
 386         elif account_type == '2of3':
 387             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 388             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 389             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 390             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 391             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 392             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 393             self.master_private_keys["m/3'/"] = k3
 394             self.master_private_keys["m/4'/"] = k4
 395             self.master_private_keys["m/5'/"] = k5
 396
 397         self.storage.put('master_public_keys', self.master_public_keys, True)
 398         self.storage.put('master_private_keys', self.master_private_keys, True)
 399
 400     def has_master_public_keys(self, account_type):
 401         if account_type == '1':
 402             return "m/0'/" in self.master_public_keys
 403         elif account_type == '2of2':
 404             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 405         elif account_type == '2of3':
 406             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 407
 408     def find_root_by_master_key(self, c, K):
 409         for key, v in self.master_public_keys.items():
 410             if key == "m/":continue
 411             cc, KK, _ = v
 412             if (c == cc) and (K == KK):
 413                 return key
 414
 415     def deseed_root(self, seed, password):
 416         # for safety, we ask the user to enter their seed
 417         assert seed == self.get_seed(password)
 418         self.seed = ''
 419         self.storage.put('seed', '', True)
 420
 421
 422     def deseed_branch(self, k):
 423         # check that parent has no seed
 424         assert self.seed == ''
 425         self.master_private_keys.pop(k)
 426         self.storage.put('master_private_keys', self.master_private_keys, True)
 427
 428     def is_watching_only(self):
 429         return (self.seed == '') and (self.master_private_keys == {})
 430
 431
 432
 433     def account_id(self, account_type, i):
 434         if account_type == '1':
 435             return "m/0'/%d"%i
 436         elif account_type == '2of2':
 437             return "m/1'/%d & m/2'/%d"%(i,i)
 438         elif account_type == '2of3':
 439             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 440         else:
 441             raise Exception('unknown account type')
 442
 443
 444     def num_accounts(self, account_type):
 445         keys = self.accounts.keys()
 446         i = 0
 447         while True:
 448             account_id = self.account_id(account_type, i)
 449             if account_id not in keys: break
 450             i += 1
 451         return i
 452
 453
 454     def new_account_address(self, account_type = '1'):
 455         i = self.num_accounts(account_type)
 456         k = self.account_id(account_type,i)
 457
 458         addr = self.next_addresses.get(k)
 459         if not addr:
 460             account_id, account = self.next_account(account_type)
 461             addr = account.first_address()
 462             self.next_addresses[k] = addr
 463             self.storage.put('next_addresses',self.next_addresses)
 464
 465         return k, addr
 466
 467
 468     def next_account(self, account_type = '1'):
 469
 470         i = self.num_accounts(account_type)
 471         account_id = self.account_id(account_type,i)
 472
 473         if account_type is '1':
 474             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 475             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 476             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 477
 478         elif account_type == '2of2':
 479             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 480             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 481             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 482             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 483             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 484
 485         elif account_type == '2of3':
 486             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 487             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 488             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 489             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 490             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 491             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 492             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 493
 494         return account_id, account
 495
 496
 497     def set_label(self, name, text = None):
 498         changed = False
 499         old_text = self.labels.get(name)
 500         if text:
 501             if old_text != text:
 502                 self.labels[name] = text
 503                 changed = True
 504         else:
 505             if old_text:
 506                 self.labels.pop(name)
 507                 changed = True
 508
 509         if changed:
 510             self.storage.put('labels', self.labels, True)
 511
 512         run_hook('set_label', name, text, changed)
 513         return changed
 514
 515
 516
 517     def create_account(self, account_type = '1', name = None):
 518         k, account = self.next_account(account_type)
 519         if k in self.pending_accounts:
 520             self.pending_accounts.pop(k)
 521             self.storage.put('pending_accounts', self.pending_accounts)
 522
 523         self.accounts[k] = account
 524         self.save_accounts()
 525         if name:
 526             self.set_label(k, name)
 527
 528
 529     def create_old_account(self, mpk):
 530         self.storage.put('master_public_key', mpk, True)
 531         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 532         self.save_accounts()
 533
 534
 535     def save_accounts(self):
 536         d = {}
 537         for k, v in self.accounts.items():
 538             d[k] = v.dump()
 539         self.storage.put('accounts', d, True)
 540
 541
 542
 543     def load_accounts(self):
 544         d = self.storage.get('accounts', {})
 545         self.accounts = {}
 546         for k, v in d.items():
 547             if k == 0:
 548                 v['mpk'] = self.storage.get('master_public_key')
 549                 self.accounts[k] = OldAccount(v)
 550             elif '&' in k:
 551                 self.accounts[k] = BIP32_Account_2of2(v)
 552             else:
 553                 self.accounts[k] = BIP32_Account(v)
 554
 555         self.pending_accounts = self.storage.get('pending_accounts',{})
 556
 557
 558     def delete_pending_account(self, k):
 559         self.pending_accounts.pop(k)
 560         self.storage.put('pending_accounts', self.pending_accounts)
 561
 562     def account_is_pending(self, k):
 563         return k in self.pending_accounts
 564
 565     def create_pending_account(self, acct_type, name):
 566         k, addr = self.new_account_address(acct_type)
 567         self.set_label(k, name)
 568         self.pending_accounts[k] = addr
 569         self.storage.put('pending_accounts', self.pending_accounts)
 570
 571     def get_pending_accounts(self):
 572         return self.pending_accounts.items()
 573
 574
 575     def addresses(self, include_change = True, _next=True):
 576         o = self.get_account_addresses(-1, include_change)
 577         for a in self.accounts.keys():
 578             o += self.get_account_addresses(a, include_change)
 579
 580         if _next:
 581             for addr in self.next_addresses.values():
 582                 if addr not in o:
 583                     o += [addr]
 584         return o
 585
 586
 587     def is_mine(self, address):
 588         return address in self.addresses(True)
 589
 590
 591     def is_change(self, address):
 592         if not self.is_mine(address): return False
 593         if address in self.imported_keys.keys(): return False
 594         acct, s = self.get_address_index(address)
 595         if s is None: return False
 596         return s[0] == 1
 597
 598     def get_master_public_key(self):
 599         if self.seed_version == 4:
 600             return self.storage.get("master_public_key")
 601         else:
 602             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 603             return repr((c, K))
 604
 605     def get_master_private_key(self, account, password):
 606         master_k = pw_decode( self.master_private_keys[account], password)
 607         master_c, master_K, master_Kc = self.master_public_keys[account]
 608         try:
 609             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 610             assert K.encode('hex') == master_K
 611         except Exception:
 612             raise Exception("Invalid password")
 613         return master_k
 614
 615
 616     def get_address_index(self, address):
 617         if address in self.imported_keys.keys():
 618             return -1, None
 619
 620         for account in self.accounts.keys():
 621             for for_change in [0,1]:
 622                 addresses = self.accounts[account].get_addresses(for_change)
 623                 for addr in addresses:
 624                     if address == addr:
 625                         return account, (for_change, addresses.index(addr))
 626
 627         for k,v in self.next_addresses.items():
 628             if v == address:
 629                 return k, (0,0)
 630
 631         raise Exception("Address not found", address)
 632
 633
 634     def get_roots(self, account):
 635         roots = []
 636         for a in account.split('&'):
 637             s = a.strip()
 638             m = re.match("(m/\d+'/)(\d+)", s)
 639             roots.append( m.group(1) )
 640         return roots
 641
 642     def is_seeded(self, account):
 643         if type(account) is int:
 644             return self.seed is not None
 645
 646         for root in self.get_roots(account):
 647             if root not in self.master_private_keys.keys():
 648                 return False
 649         return True
 650
 651     def rebase_sequence(self, account, sequence):
 652         c, i = sequence
 653         dd = []
 654         for a in account.split('&'):
 655             s = a.strip()
 656             m = re.match("(m/\d+'/)(\d+)", s)
 657             root = m.group(1)
 658             num = int(m.group(2))
 659             dd.append( (root, [num,c,i] ) )
 660         return dd
 661
 662
 663     def get_keyID(self, account, sequence):
 664         if account == 0:
 665             a, b = sequence
 666             mpk = self.storage.get('master_public_key')
 667             return 'old(%s,%d,%d)'%(mpk,a,b)
 668
 669         rs = self.rebase_sequence(account, sequence)
 670         dd = []
 671         for root, public_sequence in rs:
 672             c, K, _ = self.master_public_keys[root]
 673             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 674             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 675         return '&'.join(dd)
 676
 677
 678
 679     def get_seed(self, password):
 680         s = pw_decode(self.seed, password)
 681         if self.seed_version == 4:
 682             seed = s
 683             self.accounts[0].check_seed(seed)
 684         else:
 685             seed = mnemonic_hash(s)
 686         return seed
 687
 688
 689     def get_mnemonic(self, password):
 690         import mnemonic
 691         s = pw_decode(self.seed, password)
 692         if self.seed_version == 4:
 693             return ' '.join(mnemonic.mn_encode(s))
 694         else:
 695             return s
 696
 697
 698
 699     def get_private_key(self, address, password):
 700         # first check the provided password
 701         seed = self.get_seed(password)
 702
 703         out = []
 704         if address in self.imported_keys.keys():
 705             out.append( pw_decode( self.imported_keys[address], password ) )
 706         else:
 707             account, sequence = self.get_address_index(address)
 708             if account == 0:
 709                 pk = self.accounts[account].get_private_key(seed, sequence)
 710                 out.append(pk)
 711                 return out
 712
 713             # assert address == self.accounts[account].get_address(*sequence)
 714             rs = self.rebase_sequence( account, sequence)
 715             for root, public_sequence in rs:
 716
 717                 if root not in self.master_private_keys.keys(): continue
 718                 master_k = self.get_master_private_key(root, password)
 719                 master_c, _, _ = self.master_public_keys[root]
 720                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 721                 out.append(pk)
 722
 723         return out
 724
 725
 726     def add_keypairs_from_wallet(self, tx, keypairs, password):
 727         for txin in tx.inputs:
 728             address = txin['address']
 729             private_keys = self.get_private_key(address, password)
 730             for sec in private_keys:
 731                 pubkey = public_key_from_private_key(sec)
 732                 keypairs[ pubkey ] = sec
 733                 if address in self.imported_keys.keys():
 734                     txin['redeemPubkey'] = pubkey
 735
 736
 737     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 738         for txin in tx.inputs:
 739             keyid = txin.get('KeyID')
 740             if keyid:
 741
 742                 if self.seed_version==4:
 743                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 744                     if not m: continue
 745                     mpk = m.group(1)
 746                     if mpk != self.storage.get('master_public_key'): continue
 747                     index = int(m.group(2))
 748                     num = int(m.group(3))
 749                     account = self.accounts[0]
 750                     addr = account.get_address(index, num)
 751                     txin['address'] = addr # fixme: side effect
 752                     pk = self.get_private_key(addr, password)
 753                     for sec in pk:
 754                         pubkey = public_key_from_private_key(sec)
 755                         keypairs[pubkey] = sec
 756                     continue
 757
 758
 759                 roots = []
 760                 for s in keyid.split('&'):
 761                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 762                     if not m: continue
 763                     c = m.group(1)
 764                     K = m.group(2)
 765                     sequence = m.group(3)
 766                     root = self.find_root_by_master_key(c,K)
 767                     if not root: continue
 768                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 769                     root = root + '%d'%sequence[0]
 770                     sequence = sequence[1:]
 771                     roots.append((root,sequence))
 772
 773                 account_id = " & ".join( map(lambda x:x[0], roots) )
 774                 account = self.accounts.get(account_id)
 775                 if not account: continue
 776                 addr = account.get_address(*sequence)
 777                 txin['address'] = addr # fixme: side effect
 778                 pk = self.get_private_key(addr, password)
 779                 for sec in pk:
 780                     pubkey = public_key_from_private_key(sec)
 781                     keypairs[pubkey] = sec
 782
 783
 784
 785     def signrawtransaction(self, tx, input_info, private_keys, password):
 786
 787         # check that the password is correct
 788         seed = self.get_seed(password)
 789
 790         # add input info
 791         tx.add_input_info(input_info)
 792
 793         # add redeem script for coins that are in the wallet
 794         # FIXME: add redeemPubkey too!
 795         unspent_coins = self.get_unspent_coins()
 796         for txin in tx.inputs:
 797             for item in unspent_coins:
 798                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 799                     print_error( "tx input is in unspent coins" )
 800                     txin['scriptPubKey'] = item['scriptPubKey']
 801                     account, sequence = self.get_address_index(item['address'])
 802                     if account != -1:
 803                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 804                         print_error("added redeemScript", txin['redeemScript'])
 805                     break
 806
 807
 808         # build a list of public/private keys
 809         keypairs = {}
 810
 811         # add private keys from parameter
 812         for sec in private_keys:
 813             pubkey = public_key_from_private_key(sec)
 814             keypairs[ pubkey ] = sec
 815
 816         # add private_keys from KeyID
 817         self.add_keypairs_from_KeyID(tx, keypairs, password)
 818
 819         # add private keys from wallet
 820         self.add_keypairs_from_wallet(tx, keypairs, password)
 821         self.sign_transaction(tx, keypairs)
 822
 823
 824     def sign_message(self, address, message, password):
 825         keys = self.get_private_key(address, password)
 826         assert len(keys) == 1
 827         sec = keys[0]
 828         key = regenerate_key(sec)
 829         compressed = is_compressed(sec)
 830         return key.sign_message(message, compressed, address)
 831
 832
 833     def change_gap_limit(self, value):
 834         if value >= self.gap_limit:
 835             self.gap_limit = value
 836             self.storage.put('gap_limit', self.gap_limit, True)
 837             #self.interface.poke('synchronizer')
 838             return True
 839
 840         elif value >= self.min_acceptable_gap():
 841             for key, account in self.accounts.items():
 842                 addresses = account[0]
 843                 k = self.num_unused_trailing_addresses(addresses)
 844                 n = len(addresses) - k + value
 845                 addresses = addresses[0:n]
 846                 self.accounts[key][0] = addresses
 847
 848             self.gap_limit = value
 849             self.storage.put('gap_limit', self.gap_limit, True)
 850             self.save_accounts()
 851             return True
 852         else:
 853             return False
 854
 855     def num_unused_trailing_addresses(self, addresses):
 856         k = 0
 857         for a in addresses[::-1]:
 858             if self.history.get(a):break
 859             k = k + 1
 860         return k
 861
 862     def min_acceptable_gap(self):
 863         # fixme: this assumes wallet is synchronized
 864         n = 0
 865         nmax = 0
 866
 867         for account in self.accounts.values():
 868             addresses = account.get_addresses(0)
 869             k = self.num_unused_trailing_addresses(addresses)
 870             for a in addresses[0:-k]:
 871                 if self.history.get(a):
 872                     n = 0
 873                 else:
 874                     n += 1
 875                     if n > nmax: nmax = n
 876         return nmax + 1
 877
 878
 879     def address_is_old(self, address):
 880         age = -1
 881         h = self.history.get(address, [])
 882         if h == ['*']:
 883             return True
 884         for tx_hash, tx_height in h:
 885             if tx_height == 0:
 886                 tx_age = 0
 887             else:
 888                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 889             if tx_age > age:
 890                 age = tx_age
 891         return age > 2
 892
 893
 894     def synchronize_sequence(self, account, for_change):
 895         limit = self.gap_limit_for_change if for_change else self.gap_limit
 896         new_addresses = []
 897         while True:
 898             addresses = account.get_addresses(for_change)
 899             if len(addresses) < limit:
 900                 address = account.create_new_address(for_change)
 901                 self.history[address] = []
 902                 new_addresses.append( address )
 903                 continue
 904
 905             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 906                 break
 907             else:
 908                 address = account.create_new_address(for_change)
 909                 self.history[address] = []
 910                 new_addresses.append( address )
 911
 912         return new_addresses
 913
 914
 915
 916     def create_pending_accounts(self):
 917         for account_type in ['1','2of2','2of3']:
 918             if not self.has_master_public_keys(account_type):
 919                 continue
 920             k, a = self.new_account_address(account_type)
 921             if self.address_is_old(a):
 922                 print_error( "creating account", a )
 923                 self.create_account(account_type)
 924                 self.next_addresses.pop(k)
 925
 926
 927     def synchronize_account(self, account):
 928         new = []
 929         new += self.synchronize_sequence(account, 0)
 930         new += self.synchronize_sequence(account, 1)
 931         return new
 932
 933
 934     def synchronize(self):
 935         if self.master_public_keys:
 936             self.create_pending_accounts()
 937         new = []
 938         for account in self.accounts.values():
 939             new += self.synchronize_account(account)
 940         if new:
 941             self.save_accounts()
 942             self.storage.put('addr_history', self.history, True)
 943         return new
 944
 945
 946     def is_found(self):
 947         return self.history.values() != [[]] * len(self.history)
 948
 949
 950     def add_contact(self, address, label=None):
 951         self.addressbook.append(address)
 952         self.storage.put('contacts', self.addressbook, True)
 953         if label:
 954             self.set_label(address, label)
 955
 956
 957     def delete_contact(self, addr):
 958         if addr in self.addressbook:
 959             self.addressbook.remove(addr)
 960             self.storage.put('addressbook', self.addressbook, True)
 961
 962
 963     def fill_addressbook(self):
 964         for tx_hash, tx in self.transactions.items():
 965             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 966             if is_send:
 967                 for addr, v in tx.outputs:
 968                     if not self.is_mine(addr) and addr not in self.addressbook:
 969                         self.addressbook.append(addr)
 970         # redo labels
 971         # self.update_tx_labels()
 972
 973     def get_num_tx(self, address):
 974         n = 0
 975         for tx in self.transactions.values():
 976             if address in map(lambda x:x[0], tx.outputs): n += 1
 977         return n
 978
 979
 980     def get_address_flags(self, addr):
 981         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 982         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 983         return flags
 984
 985
 986     def get_tx_value(self, tx, account=None):
 987         domain = self.get_account_addresses(account)
 988         return tx.get_value(domain, self.prevout_values)
 989
 990
 991     def update_tx_outputs(self, tx_hash):
 992         tx = self.transactions.get(tx_hash)
 993
 994         for i, (addr, value) in enumerate(tx.outputs):
 995             key = tx_hash+ ':%d'%i
 996             self.prevout_values[key] = value
 997
 998         for item in tx.inputs:
 999             if self.is_mine(item.get('address')):
1000                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1001                 self.spent_outputs.append(key)
1002
1003
1004     def get_addr_balance(self, address):
1005         assert self.is_mine(address)
1006         h = self.history.get(address,[])
1007         if h == ['*']: return 0,0
1008         c = u = 0
1009         received_coins = []   # list of coins received at address
1010
1011         for tx_hash, tx_height in h:
1012             tx = self.transactions.get(tx_hash)
1013             if not tx: continue
1014
1015             for i, (addr, value) in enumerate(tx.outputs):
1016                 if addr == address:
1017                     key = tx_hash + ':%d'%i
1018                     received_coins.append(key)
1019
1020         for tx_hash, tx_height in h:
1021             tx = self.transactions.get(tx_hash)
1022             if not tx: continue
1023             v = 0
1024
1025             for item in tx.inputs:
1026                 addr = item.get('address')
1027                 if addr == address:
1028                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1029                     value = self.prevout_values.get( key )
1030                     if key in received_coins:
1031                         v -= value
1032
1033             for i, (addr, value) in enumerate(tx.outputs):
1034                 key = tx_hash + ':%d'%i
1035                 if addr == address:
1036                     v += value
1037
1038             if tx_height:
1039                 c += v
1040             else:
1041                 u += v
1042         return c, u
1043
1044
1045     def get_account_name(self, k):
1046         if k == 0:
1047             if self.seed_version == 4:
1048                 name = 'Main account'
1049             else:
1050                 name = 'Old account'
1051         else:
1052             default = "Unnamed account"
1053             m = re.match("m/0'/(\d+)", k)
1054             if m:
1055                 num = m.group(1)
1056                 if num == '0':
1057                     default = "Main account"
1058                 else:
1059                     default = "Account %s"%num
1060
1061             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1062             if m:
1063                 num = m.group(1)
1064                 default = "2of2 account %s"%num
1065             name = self.labels.get(k, default)
1066
1067         return name
1068
1069     def get_account_names(self):
1070         accounts = {}
1071         for k, account in self.accounts.items():
1072             accounts[k] = self.get_account_name(k)
1073         if self.imported_keys:
1074             accounts[-1] = 'Imported keys'
1075         return accounts
1076
1077     def get_account_addresses(self, a, include_change=True):
1078         if a is None:
1079             o = self.addresses(True)
1080         elif a == -1:
1081             o = self.imported_keys.keys()
1082         else:
1083             ac = self.accounts[a]
1084             o = ac.get_addresses(0)
1085             if include_change: o += ac.get_addresses(1)
1086         return o
1087
1088     def get_imported_balance(self):
1089         return self.get_balance(self.imported_keys.keys())
1090
1091     def get_account_balance(self, account):
1092         return self.get_balance(self.get_account_addresses(account))
1093
1094     def get_frozen_balance(self):
1095         return self.get_balance(self.frozen_addresses)
1096
1097     def get_balance(self, domain=None):
1098         if domain is None: domain = self.addresses(True)
1099         cc = uu = 0
1100         for addr in domain:
1101             c, u = self.get_addr_balance(addr)
1102             cc += c
1103             uu += u
1104         return cc, uu
1105
1106
1107     def get_unspent_coins(self, domain=None):
1108         coins = []
1109         if domain is None: domain = self.addresses(True)
1110         for addr in domain:
1111             h = self.history.get(addr, [])
1112             if h == ['*']: continue
1113             for tx_hash, tx_height in h:
1114                 tx = self.transactions.get(tx_hash)
1115                 if tx is None: raise Exception("Wallet not synchronized")
1116                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1117                 for o in tx.d.get('outputs'):
1118                     output = o.copy()
1119                     if output.get('address') != addr: continue
1120                     key = tx_hash + ":%d" % output.get('prevout_n')
1121                     if key in self.spent_outputs: continue
1122                     output['prevout_hash'] = tx_hash
1123                     output['height'] = tx_height
1124                     output['coinbase'] = is_coinbase
1125                     coins.append((tx_height, output))
1126
1127         # sort by age
1128         if coins:
1129             coins = sorted(coins)
1130             if coins[-1][0] != 0:
1131                 while coins[0][0] == 0:
1132                     coins = coins[1:] + [ coins[0] ]
1133         return [x[1] for x in coins]
1134
1135
1136
1137     def choose_tx_inputs_from_account( self, amount, fixed_fee, account ):
1138         domain = self.get_account_addresses(account) if account else None
1139         return self.choose_tx_inputs( amount, fixed_fee, domain )
1140
1141
1142     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1143         """ todo: minimize tx size """
1144         total = 0
1145         fee = self.fee if fixed_fee is None else fixed_fee
1146         if domain is None:
1147             domain = self.addresses(True)
1148
1149         for i in self.frozen_addresses:
1150             if i in domain: domain.remove(i)
1151
1152         prioritized = []
1153         for i in self.prioritized_addresses:
1154             if i in domain:
1155                 domain.remove(i)
1156                 prioritized.append(i)
1157
1158         coins = self.get_unspent_coins(domain)
1159         prioritized_coins = self.get_unspent_coins(prioritized)
1160
1161         inputs = []
1162         coins = prioritized_coins + coins
1163
1164         for item in coins:
1165             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1166                 continue
1167             addr = item.get('address')
1168             v = item.get('value')
1169             total += v
1170             inputs.append(item)
1171             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1172             if total >= amount + fee: break
1173         else:
1174             inputs = []
1175
1176         return inputs, total, fee
1177
1178
1179     def set_fee(self, fee):
1180         if self.fee != fee:
1181             self.fee = fee
1182             self.storage.put('fee_per_kb', self.fee, True)
1183
1184     def estimated_fee(self, inputs):
1185         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1186         fee = self.fee * int(round(estimated_size/1024.))
1187         if fee == 0: fee = self.fee
1188         return fee
1189
1190
1191     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1192         "add change to a transaction"
1193         change_amount = total - ( amount + fee )
1194         if change_amount > DUST_THRESHOLD:
1195             if not change_addr:
1196
1197                 # send change to one of the accounts involved in the tx
1198                 address = inputs[0].get('address')
1199                 account, _ = self.get_address_index(address)
1200
1201                 if not self.use_change or account == -1:
1202                     change_addr = inputs[-1]['address']
1203                 else:
1204                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1205
1206             # Insert the change output at a random position in the outputs
1207             posn = random.randint(0, len(outputs))
1208             outputs[posn:posn] = [( change_addr,  change_amount)]
1209         return outputs
1210
1211
1212     def get_history(self, address):
1213         with self.lock:
1214             return self.history.get(address)
1215
1216
1217     def get_status(self, h):
1218         if not h: return None
1219         if h == ['*']: return '*'
1220         status = ''
1221         for tx_hash, height in h:
1222             status += tx_hash + ':%d:' % height
1223         return hashlib.sha256( status ).digest().encode('hex')
1224
1225
1226     def receive_tx_callback(self, tx_hash, tx, tx_height):
1227
1228         with self.transaction_lock:
1229             self.add_extra_addresses(tx)
1230             if not self.check_new_tx(tx_hash, tx):
1231                 # may happen due to pruning
1232                 print_error("received transaction that is no longer referenced in history", tx_hash)
1233                 return
1234             self.transactions[tx_hash] = tx
1235             self.network.interface.pending_transactions_for_notifications.append(tx)
1236             self.save_transactions()
1237             if self.verifier and tx_height>0:
1238                 self.verifier.add(tx_hash, tx_height)
1239             self.update_tx_outputs(tx_hash)
1240
1241
1242     def save_transactions(self):
1243         tx = {}
1244         for k,v in self.transactions.items():
1245             tx[k] = str(v)
1246         self.storage.put('transactions', tx, True)
1247
1248     def receive_history_callback(self, addr, hist):
1249
1250         if not self.check_new_history(addr, hist):
1251             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1252
1253         with self.lock:
1254             self.history[addr] = hist
1255             self.storage.put('addr_history', self.history, True)
1256
1257         if hist != ['*']:
1258             for tx_hash, tx_height in hist:
1259                 if tx_height>0:
1260                     # add it in case it was previously unconfirmed
1261                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1262
1263
1264     def get_tx_history(self, account=None):
1265         if not self.verifier:
1266             return []
1267
1268         with self.transaction_lock:
1269             history = self.transactions.items()
1270             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1271             result = []
1272
1273             balance = 0
1274             for tx_hash, tx in history:
1275                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1276                 if v is not None: balance += v
1277
1278             c, u = self.get_account_balance(account)
1279
1280             if balance != c+u:
1281                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1282
1283             balance = c + u - balance
1284             for tx_hash, tx in history:
1285                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1286                 if not is_relevant:
1287                     continue
1288                 if value is not None:
1289                     balance += value
1290
1291                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1292                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1293
1294         return result
1295
1296
1297     def get_label(self, tx_hash):
1298         label = self.labels.get(tx_hash)
1299         is_default = (label == '') or (label is None)
1300         if is_default: label = self.get_default_label(tx_hash)
1301         return label, is_default
1302
1303
1304     def get_default_label(self, tx_hash):
1305         tx = self.transactions.get(tx_hash)
1306         default_label = ''
1307         if tx:
1308             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1309             if is_mine:
1310                 for o in tx.outputs:
1311                     o_addr, _ = o
1312                     if not self.is_mine(o_addr):
1313                         try:
1314                             default_label = self.labels[o_addr]
1315                         except KeyError:
1316                             default_label = o_addr
1317                         break
1318                 else:
1319                     default_label = '(internal)'
1320             else:
1321                 for o in tx.outputs:
1322                     o_addr, _ = o
1323                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1324                         break
1325                 else:
1326                     for o in tx.outputs:
1327                         o_addr, _ = o
1328                         if self.is_mine(o_addr):
1329                             break
1330                     else:
1331                         o_addr = None
1332
1333                 if o_addr:
1334                     dest_label = self.labels.get(o_addr)
1335                     try:
1336                         default_label = self.labels[o_addr]
1337                     except KeyError:
1338                         default_label = o_addr
1339
1340         return default_label
1341
1342
1343     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1344         for address, x in outputs:
1345             assert is_valid(address)
1346         amount = sum( map(lambda x:x[1], outputs) )
1347         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1348         if not inputs:
1349             raise ValueError("Not enough funds")
1350         self.add_input_info(inputs)
1351         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1352         return Transaction.from_io(inputs, outputs)
1353
1354
1355     def mktx_from_account(self, outputs, password, fee=None, account=None):
1356         domain = self.get_account_addresses(account) if account else None
1357         return self.mktx(outputs, password, fee, change_addr=None, domain=domain)
1358
1359
1360     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1361         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1362         keypairs = {}
1363         self.add_keypairs_from_wallet(tx, keypairs, password)
1364         if keypairs:
1365             self.sign_transaction(tx, keypairs)
1366         return tx
1367
1368
1369     def add_input_info(self, inputs):
1370         for txin in inputs:
1371             address = txin['address']
1372             if address in self.imported_keys.keys():
1373                 continue
1374             account, sequence = self.get_address_index(address)
1375             txin['KeyID'] = self.get_keyID(account, sequence)
1376             redeemScript = self.accounts[account].redeem_script(sequence)
1377             if redeemScript:
1378                 txin['redeemScript'] = redeemScript
1379             else:
1380                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1381
1382
1383     def sign_transaction(self, tx, keypairs):
1384         tx.sign(keypairs)
1385         run_hook('sign_transaction', tx)
1386
1387
1388     def sendtx(self, tx):
1389         # synchronous
1390         h = self.send_tx(tx)
1391         self.tx_event.wait()
1392         return self.receive_tx(h)
1393
1394     def send_tx(self, tx):
1395         # asynchronous
1396         self.tx_event.clear()
1397         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1398         return tx.hash()
1399
1400     def on_broadcast(self, i, r):
1401         self.tx_result = r.get('result')
1402         self.tx_event.set()
1403
1404     def receive_tx(self,tx_hash):
1405         out = self.tx_result
1406         if out != tx_hash:
1407             return False, "error: " + out
1408         return True, out
1409
1410
1411
1412     def update_password(self, old_password, new_password):
1413         if new_password == '': new_password = None
1414         decoded = self.get_seed(old_password)
1415         self.seed = pw_encode( decoded, new_password)
1416         self.storage.put('seed', self.seed, True)
1417         self.use_encryption = (new_password != None)
1418         self.storage.put('use_encryption', self.use_encryption,True)
1419         for k in self.imported_keys.keys():
1420             a = self.imported_keys[k]
1421             b = pw_decode(a, old_password)
1422             c = pw_encode(b, new_password)
1423             self.imported_keys[k] = c
1424         self.storage.put('imported_keys', self.imported_keys, True)
1425
1426         for k, v in self.master_private_keys.items():
1427             b = pw_decode(v, old_password)
1428             c = pw_encode(b, new_password)
1429             self.master_private_keys[k] = c
1430         self.storage.put('master_private_keys', self.master_private_keys, True)
1431
1432
1433     def freeze(self,addr):
1434         if self.is_mine(addr) and addr not in self.frozen_addresses:
1435             self.unprioritize(addr)
1436             self.frozen_addresses.append(addr)
1437             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1438             return True
1439         else:
1440             return False
1441
1442     def unfreeze(self,addr):
1443         if self.is_mine(addr) and addr in self.frozen_addresses:
1444             self.frozen_addresses.remove(addr)
1445             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1446             return True
1447         else:
1448             return False
1449
1450     def prioritize(self,addr):
1451         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1452             self.unfreeze(addr)
1453             self.prioritized_addresses.append(addr)
1454             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1455             return True
1456         else:
1457             return False
1458
1459     def unprioritize(self,addr):
1460         if self.is_mine(addr) and addr in self.prioritized_addresses:
1461             self.prioritized_addresses.remove(addr)
1462             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1463             return True
1464         else:
1465             return False
1466
1467
1468     def set_verifier(self, verifier):
1469         self.verifier = verifier
1470
1471         # review transactions that are in the history
1472         for addr, hist in self.history.items():
1473             if hist == ['*']: continue
1474             for tx_hash, tx_height in hist:
1475                 if tx_height>0:
1476                     # add it in case it was previously unconfirmed
1477                     self.verifier.add(tx_hash, tx_height)
1478
1479
1480         # if we are on a pruning server, remove unverified transactions
1481         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1482         for tx_hash in self.transactions.keys():
1483             if tx_hash not in vr:
1484                 self.transactions.pop(tx_hash)
1485
1486
1487
1488     def check_new_history(self, addr, hist):
1489
1490         # check that all tx in hist are relevant
1491         if hist != ['*']:
1492             for tx_hash, height in hist:
1493                 tx = self.transactions.get(tx_hash)
1494                 if not tx: continue
1495                 if not tx.has_address(addr):
1496                     return False
1497
1498         # check that we are not "orphaning" a transaction
1499         old_hist = self.history.get(addr,[])
1500         if old_hist == ['*']: return True
1501
1502         for tx_hash, height in old_hist:
1503             if tx_hash in map(lambda x:x[0], hist): continue
1504             found = False
1505             for _addr, _hist in self.history.items():
1506                 if _addr == addr: continue
1507                 if _hist == ['*']: continue
1508                 _tx_hist = map(lambda x:x[0], _hist)
1509                 if tx_hash in _tx_hist:
1510                     found = True
1511                     break
1512
1513             if not found:
1514                 tx = self.transactions.get(tx_hash)
1515                 # tx might not be there
1516                 if not tx: continue
1517
1518                 # already verified?
1519                 if self.verifier.get_height(tx_hash):
1520                     continue
1521                 # unconfirmed tx
1522                 print_error("new history is orphaning transaction:", tx_hash)
1523                 # check that all outputs are not mine, request histories
1524                 ext_requests = []
1525                 for _addr, _v in tx.outputs:
1526                     # assert not self.is_mine(_addr)
1527                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1528
1529                 ext_h = self.network.synchronous_get(ext_requests)
1530                 print_error("sync:", ext_requests, ext_h)
1531                 height = None
1532                 for h in ext_h:
1533                     if h == ['*']: continue
1534                     for item in h:
1535                         if item.get('tx_hash') == tx_hash:
1536                             height = item.get('height')
1537                 if height:
1538                     print_error("found height for", tx_hash, height)
1539                     self.verifier.add(tx_hash, height)
1540                 else:
1541                     print_error("removing orphaned tx from history", tx_hash)
1542                     self.transactions.pop(tx_hash)
1543
1544         return True
1545
1546
1547
1548     def check_new_tx(self, tx_hash, tx):
1549         # 1 check that tx is referenced in addr_history.
1550         addresses = []
1551         for addr, hist in self.history.items():
1552             if hist == ['*']:continue
1553             for txh, height in hist:
1554                 if txh == tx_hash:
1555                     addresses.append(addr)
1556
1557         if not addresses:
1558             return False
1559
1560         # 2 check that referencing addresses are in the tx
1561         for addr in addresses:
1562             if not tx.has_address(addr):
1563                 return False
1564
1565         return True
1566
1567
1568     def start_threads(self, network):
1569         from verifier import TxVerifier
1570         self.network = network
1571         if self.network:
1572             self.verifier = TxVerifier(self.network, self.storage)
1573             self.verifier.start()
1574             self.set_verifier(self.verifier)
1575             self.synchronizer = WalletSynchronizer(self, network)
1576             self.synchronizer.start()
1577         else:
1578             self.verifier = None
1579             self.synchronizer =None
1580
1581     def stop_threads(self):
1582         if self.network:
1583             self.verifier.stop()
1584             self.synchronizer.stop()
1585
1586
1587     def restore(self, callback):
1588         from i18n import _
1589         def wait_for_wallet():
1590             self.set_up_to_date(False)
1591             while not self.is_up_to_date():
1592                 msg = "%s\n%s %d\n%s %.1f"%(
1593                     _("Please wait..."),
1594                     _("Addresses generated:"),
1595                     len(self.addresses(True)),
1596                     _("Kilobytes received:"),
1597                     self.network.interface.bytes_received/1024.)
1598
1599                 apply(callback, (msg,))
1600                 time.sleep(0.1)
1601
1602         def wait_for_network():
1603             while not self.network.is_connected():
1604                 msg = "%s \n" % (_("Connecting..."))
1605                 apply(callback, (msg,))
1606                 time.sleep(0.1)
1607
1608         # wait until we are connected, because the user might have selected another server
1609         if self.network:
1610             wait_for_network()
1611
1612         self.create_accounts()
1613
1614         if self.network:
1615             wait_for_wallet()
1616         else:
1617             self.synchronize()
1618
1619         self.fill_addressbook()
1620
1621
1622
1623
1624 class WalletSynchronizer(threading.Thread):
1625
1626
1627     def __init__(self, wallet, network):
1628         threading.Thread.__init__(self)
1629         self.daemon = True
1630         self.wallet = wallet
1631         self.network = network
1632         self.was_updated = True
1633         self.running = False
1634         self.lock = threading.Lock()
1635         self.queue = Queue.Queue()
1636
1637     def stop(self):
1638         with self.lock: self.running = False
1639
1640     def is_running(self):
1641         with self.lock: return self.running
1642
1643
1644     def subscribe_to_addresses(self, addresses):
1645         messages = []
1646         for addr in addresses:
1647             messages.append(('blockchain.address.subscribe', [addr]))
1648         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1649
1650
1651     def run(self):
1652         with self.lock:
1653             self.running = True
1654
1655         while self.is_running():
1656
1657             if not self.network.is_connected():
1658                 print_error("synchronizer: waiting for interface")
1659                 self.network.wait_until_connected()
1660
1661             self.run_interface(self.network.interface)
1662
1663
1664     def run_interface(self, interface):
1665
1666         print_error("synchronizer: connected to", interface.server)
1667
1668         requested_tx = []
1669         missing_tx = []
1670         requested_histories = {}
1671
1672         # request any missing transactions
1673         for history in self.wallet.history.values():
1674             if history == ['*']: continue
1675             for tx_hash, tx_height in history:
1676                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1677                     missing_tx.append( (tx_hash, tx_height) )
1678
1679         if missing_tx:
1680             print_error("missing tx", missing_tx)
1681
1682         # subscriptions
1683         self.subscribe_to_addresses(self.wallet.addresses(True))
1684
1685         while self.is_running():
1686             # 1. create new addresses
1687             new_addresses = self.wallet.synchronize()
1688
1689             # request missing addresses
1690             if new_addresses:
1691                 self.subscribe_to_addresses(new_addresses)
1692
1693             # request missing transactions
1694             for tx_hash, tx_height in missing_tx:
1695                 if (tx_hash, tx_height) not in requested_tx:
1696                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1697                     requested_tx.append( (tx_hash, tx_height) )
1698             missing_tx = []
1699
1700             # detect if situation has changed
1701             if interface.is_up_to_date() and self.queue.empty():
1702                 if not self.wallet.is_up_to_date():
1703                     self.wallet.set_up_to_date(True)
1704                     self.was_updated = True
1705             else:
1706                 if self.wallet.is_up_to_date():
1707                     self.wallet.set_up_to_date(False)
1708                     self.was_updated = True
1709
1710             if self.was_updated:
1711                 self.wallet.network.trigger_callback('updated')
1712                 self.was_updated = False
1713
1714             # 2. get a response
1715             try:
1716                 r = self.queue.get(block=True, timeout=1)
1717             except Queue.Empty:
1718                 continue
1719
1720             if interface != self.network.interface:
1721                 break
1722
1723             if not r:
1724                 continue
1725
1726             # 3. handle response
1727             method = r['method']
1728             params = r['params']
1729             result = r.get('result')
1730             error = r.get('error')
1731             if error:
1732                 print "error", r
1733                 continue
1734
1735             if method == 'blockchain.address.subscribe':
1736                 addr = params[0]
1737                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1738                     if requested_histories.get(addr) is None:
1739                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1740                         requested_histories[addr] = result
1741
1742             elif method == 'blockchain.address.get_history':
1743                 addr = params[0]
1744                 print_error("receiving history", addr, result)
1745                 if result == ['*']:
1746                     assert requested_histories.pop(addr) == '*'
1747                     self.wallet.receive_history_callback(addr, result)
1748                 else:
1749                     hist = []
1750                     # check that txids are unique
1751                     txids = []
1752                     for item in result:
1753                         tx_hash = item['tx_hash']
1754                         if tx_hash not in txids:
1755                             txids.append(tx_hash)
1756                             hist.append( (tx_hash, item['height']) )
1757
1758                     if len(hist) != len(result):
1759                         raise Exception("error: server sent history with non-unique txid", result)
1760
1761                     # check that the status corresponds to what was announced
1762                     rs = requested_histories.pop(addr)
1763                     if self.wallet.get_status(hist) != rs:
1764                         raise Exception("error: status mismatch: %s"%addr)
1765
1766                     # store received history
1767                     self.wallet.receive_history_callback(addr, hist)
1768
1769                     # request transactions that we don't have
1770                     for tx_hash, tx_height in hist:
1771                         if self.wallet.transactions.get(tx_hash) is None:
1772                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1773                                 missing_tx.append( (tx_hash, tx_height) )
1774
1775             elif method == 'blockchain.transaction.get':
1776                 tx_hash = params[0]
1777                 tx_height = params[1]
1778                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1779                 tx = Transaction(result)
1780                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1781                 self.was_updated = True
1782                 requested_tx.remove( (tx_hash, tx_height) )
1783                 print_error("received tx:", tx_hash, len(tx.raw))
1784
1785             else:
1786                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1787
1788             if self.was_updated and not requested_tx:
1789                 self.wallet.network.trigger_callback('updated')
1790                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1791
1792                 self.was_updated = False