lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36 from transaction import Transaction
  37 from plugins import run_hook
  38
  39 COINBASE_MATURITY = 100
  40 DUST_THRESHOLD = 5430
  41
  42 # AES encryption
  43 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  44 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  45
  46 def pw_encode(s, password):
  47     if password:
  48         secret = Hash(password)
  49         return EncodeAES(secret, s)
  50     else:
  51         return s
  52
  53 def pw_decode(s, password):
  54     if password is not None:
  55         secret = Hash(password)
  56         try:
  57             d = DecodeAES(secret, s)
  58         except Exception:
  59             raise Exception('Invalid password')
  60         return d
  61     else:
  62         return s
  63
  64
  65
  66
  67
  68 from version import *
  69
  70
  71 class WalletStorage:
  72
  73     def __init__(self, config):
  74         self.lock = threading.Lock()
  75         self.data = {}
  76         self.file_exists = False
  77         self.path = self.init_path(config)
  78         print_error( "wallet path", self.path )
  79         if self.path:
  80             self.read(self.path)
  81
  82
  83     def init_path(self, config):
  84         """Set the path of the wallet."""
  85
  86         # command line -w option
  87         path = config.get('wallet_path')
  88         if path:
  89             return path
  90
  91         # path in config file
  92         path = config.get('default_wallet_path')
  93         if path:
  94             return path
  95
  96         # default path
  97         dirpath = os.path.join(config.path, "wallets")
  98         if not os.path.exists(dirpath):
  99             os.mkdir(dirpath)
 100
 101         new_path = os.path.join(config.path, "wallets", "default_wallet")
 102
 103         # default path in pre 1.9 versions
 104         old_path = os.path.join(config.path, "electrum.dat")
 105         if os.path.exists(old_path) and not os.path.exists(new_path):
 106             os.rename(old_path, new_path)
 107
 108         return new_path
 109
 110
 111     def read(self, path):
 112         """Read the contents of the wallet file."""
 113         try:
 114             with open(self.path, "r") as f:
 115                 data = f.read()
 116         except IOError:
 117             return
 118         try:
 119             d = ast.literal_eval( data )  #parse raw data from reading wallet file
 120         except Exception:
 121             raise IOError("Cannot read wallet file.")
 122
 123         self.data = d
 124         self.file_exists = True
 125
 126
 127     def get(self, key, default=None):
 128         v = self.data.get(key)
 129         if v is None:
 130             v = default
 131         return v
 132
 133     def put(self, key, value, save = True):
 134
 135         with self.lock:
 136             if value is not None:
 137                 self.data[key] = value
 138             else:
 139                 self.data.pop(key)
 140             if save:
 141                 self.write()
 142
 143     def write(self):
 144         s = repr(self.data)
 145         f = open(self.path,"w")
 146         f.write( s )
 147         f.close()
 148         if 'ANDROID_DATA' not in os.environ:
 149             import stat
 150             os.chmod(self.path,stat.S_IREAD | stat.S_IWRITE)
 151
 152
 153 class Wallet:
 154
 155     def __init__(self, storage):
 156
 157         self.storage = storage
 158         self.electrum_version = ELECTRUM_VERSION
 159         self.gap_limit_for_change = 3 # constant
 160
 161         # saved fields
 162         self.seed_version          = storage.get('seed_version', SEED_VERSION)
 163
 164         self.gap_limit             = storage.get('gap_limit', 5)
 165         self.use_change            = storage.get('use_change',True)
 166         self.use_encryption        = storage.get('use_encryption', False)
 167         self.seed                  = storage.get('seed', '')               # encrypted
 168         self.labels                = storage.get('labels', {})
 169         self.frozen_addresses      = storage.get('frozen_addresses',[])
 170         self.prioritized_addresses = storage.get('prioritized_addresses',[])
 171         self.addressbook           = storage.get('contacts', [])
 172
 173         self.imported_keys         = storage.get('imported_keys',{})
 174         self.history               = storage.get('addr_history',{})        # address -> list(txid, height)
 175
 176         self.fee                   = int(storage.get('fee_per_kb',20000))
 177
 178         self.master_public_keys = storage.get('master_public_keys',{})
 179         self.master_private_keys = storage.get('master_private_keys', {})
 180
 181         self.next_addresses = storage.get('next_addresses',{})
 182
 183         if self.seed_version not in [4, 6]:
 184             msg = "This wallet seed is not supported."
 185             if self.seed_version in [5]:
 186                 msg += "\nTo open this wallet, try 'git checkout seed_v%d'"%self.seed_version
 187                 print msg
 188                 sys.exit(1)
 189
 190
 191         self.load_accounts()
 192
 193         self.transactions = {}
 194         tx_list = self.storage.get('transactions',{})
 195         for k,v in tx_list.items():
 196             try:
 197                 tx = Transaction(v)
 198             except Exception:
 199                 print_msg("Warning: Cannot deserialize transactions. skipping")
 200                 continue
 201
 202             self.add_extra_addresses(tx)
 203             self.transactions[k] = tx
 204
 205         for h,tx in self.transactions.items():
 206             if not self.check_new_tx(h, tx):
 207                 print_error("removing unreferenced tx", h)
 208                 self.transactions.pop(h)
 209
 210
 211         # not saved
 212         self.prevout_values = {}     # my own transaction outputs
 213         self.spent_outputs = []
 214
 215         # spv
 216         self.verifier = None
 217
 218         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 219         # interface.is_up_to_date() returns true when all requests have been answered and processed
 220         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 221
 222         self.up_to_date = False
 223         self.lock = threading.Lock()
 224         self.transaction_lock = threading.Lock()
 225         self.tx_event = threading.Event()
 226
 227         for tx_hash, tx in self.transactions.items():
 228             self.update_tx_outputs(tx_hash)
 229
 230
 231     def add_extra_addresses(self, tx):
 232         h = tx.hash()
 233         # find the address corresponding to pay-to-pubkey inputs
 234         tx.add_extra_addresses(self.transactions)
 235         for o in tx.d.get('outputs'):
 236             if o.get('is_pubkey'):
 237                 for tx2 in self.transactions.values():
 238                     tx2.add_extra_addresses({h:tx})
 239
 240
 241
 242
 243     def set_up_to_date(self,b):
 244         with self.lock: self.up_to_date = b
 245
 246
 247     def is_up_to_date(self):
 248         with self.lock: return self.up_to_date
 249
 250
 251     def update(self):
 252         self.up_to_date = False
 253         while not self.is_up_to_date():
 254             time.sleep(0.1)
 255
 256
 257     def import_key(self, sec, password):
 258         # check password
 259         seed = self.get_seed(password)
 260         try:
 261             address = address_from_private_key(sec)
 262         except Exception:
 263             raise Exception('Invalid private key')
 264
 265         if self.is_mine(address):
 266             raise Exception('Address already in wallet')
 267
 268         # store the originally requested keypair into the imported keys table
 269         self.imported_keys[address] = pw_encode(sec, password )
 270         self.storage.put('imported_keys', self.imported_keys, True)
 271         if self.synchronizer:
 272             self.synchronizer.subscribe_to_addresses([address])
 273         return address
 274
 275     def delete_imported_key(self, addr):
 276         if addr in self.imported_keys:
 277             self.imported_keys.pop(addr)
 278             self.storage.put('imported_keys', self.imported_keys, True)
 279
 280
 281     def make_seed(self):
 282         import mnemonic, ecdsa
 283         entropy = ecdsa.util.randrange( pow(2,160) )
 284         nonce = 0
 285         while True:
 286             ss = "%040x"%(entropy+nonce)
 287             s = hashlib.sha256(ss.decode('hex')).digest().encode('hex')
 288             # we keep only 13 words, that's approximately 139 bits of entropy
 289             words = mnemonic.mn_encode(s)[0:13]
 290             seed = ' '.join(words)
 291             if mnemonic_hash(seed).startswith(SEED_PREFIX):
 292                 break  # this removes 12 bits of entropy
 293             nonce += 1
 294
 295         return seed
 296
 297
 298     def init_seed(self, seed):
 299         import mnemonic
 300
 301         if self.seed:
 302             raise Exception("a seed exists")
 303
 304         if not seed:
 305             self.seed = random_seed(128)
 306             self.seed_version = 4
 307             return
 308
 309         #if not seed:
 310         #    self.seed = self.make_seed()
 311         #    self.seed_version = SEED_VERSION
 312         #    return
 313
 314         # find out what kind of wallet we are
 315         try:
 316             seed.strip().decode('hex')
 317             self.seed_version = 4
 318             self.seed = str(seed)
 319             return
 320         except Exception:
 321             pass
 322
 323         words = seed.split()
 324         self.seed_version = 4
 325         self.seed = mnemonic.mn_decode(words)
 326
 327         #try:
 328         #    mnemonic.mn_decode(words)
 329         #    uses_electrum_words = True
 330         #except Exception:
 331         #    uses_electrum_words = False
 332         #
 333         #if uses_electrum_words and len(words) != 13:
 334         #    self.seed_version = 4
 335         #    self.seed = mnemonic.mn_decode(words)
 336         #else:
 337         #    assert mnemonic_hash(seed).startswith(SEED_PREFIX)
 338         #    self.seed_version = SEED_VERSION
 339         #    self.seed = seed
 340
 341
 342     def save_seed(self):
 343         self.storage.put('seed', self.seed, True)
 344         self.storage.put('seed_version', self.seed_version, True)
 345         self.create_accounts()
 346
 347
 348     def create_watching_only_wallet(self, params):
 349         K0, c0 = params
 350         if not K0:
 351             return
 352
 353         if not c0:
 354             self.seed_version = 4
 355             self.storage.put('seed_version', self.seed_version, True)
 356             self.create_old_account(K0)
 357             return
 358
 359         cK0 = ""
 360         self.master_public_keys = {
 361             "m/0'/": (c0, K0, cK0),
 362             }
 363         self.storage.put('master_public_keys', self.master_public_keys, True)
 364         self.storage.put('seed_version', self.seed_version, True)
 365         self.create_account('1','Main account')
 366
 367
 368     def create_accounts(self):
 369         if self.seed_version == 4:
 370             mpk = OldAccount.mpk_from_seed(self.seed)
 371             self.create_old_account(mpk)
 372         else:
 373             # create default account
 374             self.create_master_keys('1')
 375             self.create_account('1','Main account')
 376
 377
 378     def create_master_keys(self, account_type):
 379         master_k, master_c, master_K, master_cK = bip32_init(self.get_seed(None))
 380         if account_type == '1':
 381             k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 382             self.master_public_keys["m/0'/"] = (c0, K0, cK0)
 383             self.master_private_keys["m/0'/"] = k0
 384         elif account_type == '2of2':
 385             k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 386             k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 387             self.master_public_keys["m/1'/"] = (c1, K1, cK1)
 388             self.master_public_keys["m/2'/"] = (c2, K2, cK2)
 389             self.master_private_keys["m/1'/"] = k1
 390             self.master_private_keys["m/2'/"] = k2
 391         elif account_type == '2of3':
 392             k3, c3, K3, cK3 = bip32_private_derivation(master_k, master_c, "m/", "m/3'/")
 393             k4, c4, K4, cK4 = bip32_private_derivation(master_k, master_c, "m/", "m/4'/")
 394             k5, c5, K5, cK5 = bip32_private_derivation(master_k, master_c, "m/", "m/5'/")
 395             self.master_public_keys["m/3'/"] = (c3, K3, cK3)
 396             self.master_public_keys["m/4'/"] = (c4, K4, cK4)
 397             self.master_public_keys["m/5'/"] = (c5, K5, cK5)
 398             self.master_private_keys["m/3'/"] = k3
 399             self.master_private_keys["m/4'/"] = k4
 400             self.master_private_keys["m/5'/"] = k5
 401
 402         self.storage.put('master_public_keys', self.master_public_keys, True)
 403         self.storage.put('master_private_keys', self.master_private_keys, True)
 404
 405     def has_master_public_keys(self, account_type):
 406         if account_type == '1':
 407             return "m/0'/" in self.master_public_keys
 408         elif account_type == '2of2':
 409             return set(["m/1'/", "m/2'/"]) <= set(self.master_public_keys.keys())
 410         elif account_type == '2of3':
 411             return set(["m/3'/", "m/4'/", "m/5'/"]) <= set(self.master_public_keys.keys())
 412
 413     def find_root_by_master_key(self, c, K):
 414         for key, v in self.master_public_keys.items():
 415             if key == "m/":continue
 416             cc, KK, _ = v
 417             if (c == cc) and (K == KK):
 418                 return key
 419
 420     def deseed_root(self, seed, password):
 421         # for safety, we ask the user to enter their seed
 422         assert seed == self.get_seed(password)
 423         self.seed = ''
 424         self.storage.put('seed', '', True)
 425
 426
 427     def deseed_branch(self, k):
 428         # check that parent has no seed
 429         assert self.seed == ''
 430         self.master_private_keys.pop(k)
 431         self.storage.put('master_private_keys', self.master_private_keys, True)
 432
 433     def is_watching_only(self):
 434         return (self.seed == '') and (self.master_private_keys == {})
 435
 436
 437
 438     def account_id(self, account_type, i):
 439         if account_type == '1':
 440             return "m/0'/%d"%i
 441         elif account_type == '2of2':
 442             return "m/1'/%d & m/2'/%d"%(i,i)
 443         elif account_type == '2of3':
 444             return "m/3'/%d & m/4'/%d & m/5'/%d"%(i,i,i)
 445         else:
 446             raise Exception('unknown account type')
 447
 448
 449     def num_accounts(self, account_type):
 450         keys = self.accounts.keys()
 451         i = 0
 452         while True:
 453             account_id = self.account_id(account_type, i)
 454             if account_id not in keys: break
 455             i += 1
 456         return i
 457
 458
 459     def new_account_address(self, account_type = '1'):
 460         i = self.num_accounts(account_type)
 461         k = self.account_id(account_type,i)
 462
 463         addr = self.next_addresses.get(k)
 464         if not addr:
 465             account_id, account = self.next_account(account_type)
 466             addr = account.first_address()
 467             self.next_addresses[k] = addr
 468             self.storage.put('next_addresses',self.next_addresses)
 469
 470         return k, addr
 471
 472
 473     def next_account(self, account_type = '1'):
 474
 475         i = self.num_accounts(account_type)
 476         account_id = self.account_id(account_type,i)
 477
 478         if account_type is '1':
 479             master_c0, master_K0, _ = self.master_public_keys["m/0'/"]
 480             c0, K0, cK0 = bip32_public_derivation(master_c0.decode('hex'), master_K0.decode('hex'), "m/0'/", "m/0'/%d"%i)
 481             account = BIP32_Account({ 'c':c0, 'K':K0, 'cK':cK0 })
 482
 483         elif account_type == '2of2':
 484             master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 485             c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 486             master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 487             c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 488             account = BIP32_Account_2of2({ 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 489
 490         elif account_type == '2of3':
 491             master_c3, master_K3, _ = self.master_public_keys["m/3'/"]
 492             c3, K3, cK3 = bip32_public_derivation(master_c3.decode('hex'), master_K3.decode('hex'), "m/3'/", "m/3'/%d"%i)
 493             master_c4, master_K4, _ = self.master_public_keys["m/4'/"]
 494             c4, K4, cK4 = bip32_public_derivation(master_c4.decode('hex'), master_K4.decode('hex'), "m/4'/", "m/4'/%d"%i)
 495             master_c5, master_K5, _ = self.master_public_keys["m/5'/"]
 496             c5, K5, cK5 = bip32_public_derivation(master_c5.decode('hex'), master_K5.decode('hex'), "m/5'/", "m/5'/%d"%i)
 497             account = BIP32_Account_2of3({ 'c':c3, 'K':K3, 'cK':cK3, 'c2':c4, 'K2':K4, 'cK2':cK4, 'c3':c5, 'K3':K5, 'cK3':cK5 })
 498
 499         return account_id, account
 500
 501
 502     def set_label(self, name, text = None):
 503         changed = False
 504         old_text = self.labels.get(name)
 505         if text:
 506             if old_text != text:
 507                 self.labels[name] = text
 508                 changed = True
 509         else:
 510             if old_text:
 511                 self.labels.pop(name)
 512                 changed = True
 513
 514         if changed:
 515             self.storage.put('labels', self.labels, True)
 516
 517         run_hook('set_label', name, text, changed)
 518         return changed
 519
 520
 521
 522     def create_account(self, account_type = '1', name = None):
 523         k, account = self.next_account(account_type)
 524         if k in self.pending_accounts:
 525             self.pending_accounts.pop(k)
 526             self.storage.put('pending_accounts', self.pending_accounts)
 527
 528         self.accounts[k] = account
 529         self.save_accounts()
 530         if name:
 531             self.set_label(k, name)
 532
 533
 534     def create_old_account(self, mpk):
 535         self.storage.put('master_public_key', mpk, True)
 536         self.accounts[0] = OldAccount({'mpk':mpk, 0:[], 1:[]})
 537         self.save_accounts()
 538
 539
 540     def save_accounts(self):
 541         d = {}
 542         for k, v in self.accounts.items():
 543             d[k] = v.dump()
 544         self.storage.put('accounts', d, True)
 545
 546
 547
 548     def load_accounts(self):
 549         d = self.storage.get('accounts', {})
 550         self.accounts = {}
 551         for k, v in d.items():
 552             if k == 0:
 553                 v['mpk'] = self.storage.get('master_public_key')
 554                 self.accounts[k] = OldAccount(v)
 555             elif '&' in k:
 556                 self.accounts[k] = BIP32_Account_2of2(v)
 557             else:
 558                 self.accounts[k] = BIP32_Account(v)
 559
 560         self.pending_accounts = self.storage.get('pending_accounts',{})
 561
 562
 563     def delete_pending_account(self, k):
 564         self.pending_accounts.pop(k)
 565         self.storage.put('pending_accounts', self.pending_accounts)
 566
 567     def account_is_pending(self, k):
 568         return k in self.pending_accounts
 569
 570     def create_pending_account(self, acct_type, name):
 571         k, addr = self.new_account_address(acct_type)
 572         self.set_label(k, name)
 573         self.pending_accounts[k] = addr
 574         self.storage.put('pending_accounts', self.pending_accounts)
 575
 576     def get_pending_accounts(self):
 577         return self.pending_accounts.items()
 578
 579
 580     def addresses(self, include_change = True, _next=True):
 581         o = self.get_account_addresses(-1, include_change)
 582         for a in self.accounts.keys():
 583             o += self.get_account_addresses(a, include_change)
 584
 585         if _next:
 586             for addr in self.next_addresses.values():
 587                 if addr not in o:
 588                     o += [addr]
 589         return o
 590
 591
 592     def is_mine(self, address):
 593         return address in self.addresses(True)
 594
 595
 596     def is_change(self, address):
 597         if not self.is_mine(address): return False
 598         if address in self.imported_keys.keys(): return False
 599         acct, s = self.get_address_index(address)
 600         if s is None: return False
 601         return s[0] == 1
 602
 603     def get_master_public_key(self):
 604         if self.seed_version == 4:
 605             return self.storage.get("master_public_key")
 606         else:
 607             c, K, cK = self.storage.get("master_public_keys")["m/0'/"]
 608             return repr((c, K))
 609
 610     def get_master_private_key(self, account, password):
 611         master_k = pw_decode( self.master_private_keys[account], password)
 612         master_c, master_K, master_Kc = self.master_public_keys[account]
 613         try:
 614             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 615             assert K.encode('hex') == master_K
 616         except Exception:
 617             raise Exception("Invalid password")
 618         return master_k
 619
 620
 621     def get_address_index(self, address):
 622         if address in self.imported_keys.keys():
 623             return -1, None
 624
 625         for account in self.accounts.keys():
 626             for for_change in [0,1]:
 627                 addresses = self.accounts[account].get_addresses(for_change)
 628                 for addr in addresses:
 629                     if address == addr:
 630                         return account, (for_change, addresses.index(addr))
 631
 632         for k,v in self.next_addresses.items():
 633             if v == address:
 634                 return k, (0,0)
 635
 636         raise Exception("Address not found", address)
 637
 638
 639     def get_roots(self, account):
 640         roots = []
 641         for a in account.split('&'):
 642             s = a.strip()
 643             m = re.match("(m/\d+'/)(\d+)", s)
 644             roots.append( m.group(1) )
 645         return roots
 646
 647     def is_seeded(self, account):
 648         if type(account) is int:
 649             return self.seed is not None
 650
 651         for root in self.get_roots(account):
 652             if root not in self.master_private_keys.keys():
 653                 return False
 654         return True
 655
 656     def rebase_sequence(self, account, sequence):
 657         c, i = sequence
 658         dd = []
 659         for a in account.split('&'):
 660             s = a.strip()
 661             m = re.match("(m/\d+'/)(\d+)", s)
 662             root = m.group(1)
 663             num = int(m.group(2))
 664             dd.append( (root, [num,c,i] ) )
 665         return dd
 666
 667
 668     def get_keyID(self, account, sequence):
 669         if account == 0:
 670             a, b = sequence
 671             mpk = self.storage.get('master_public_key')
 672             return 'old(%s,%d,%d)'%(mpk,a,b)
 673
 674         rs = self.rebase_sequence(account, sequence)
 675         dd = []
 676         for root, public_sequence in rs:
 677             c, K, _ = self.master_public_keys[root]
 678             s = '/' + '/'.join( map(lambda x:str(x), public_sequence) )
 679             dd.append( 'bip32(%s,%s,%s)'%(c,K, s) )
 680         return '&'.join(dd)
 681
 682
 683
 684     def get_seed(self, password):
 685         s = pw_decode(self.seed, password)
 686         if self.seed_version == 4:
 687             seed = s
 688             self.accounts[0].check_seed(seed)
 689         else:
 690             seed = mnemonic_hash(s)
 691         return seed
 692
 693
 694     def get_mnemonic(self, password):
 695         import mnemonic
 696         s = pw_decode(self.seed, password)
 697         if self.seed_version == 4:
 698             return ' '.join(mnemonic.mn_encode(s))
 699         else:
 700             return s
 701
 702
 703
 704     def get_private_key(self, address, password):
 705         if self.is_watching_only():
 706             return []
 707
 708         # first check the provided password
 709         seed = self.get_seed(password)
 710
 711         out = []
 712         if address in self.imported_keys.keys():
 713             out.append( pw_decode( self.imported_keys[address], password ) )
 714         else:
 715             account, sequence = self.get_address_index(address)
 716             if account == 0:
 717                 pk = self.accounts[account].get_private_key(seed, sequence)
 718                 out.append(pk)
 719                 return out
 720
 721             # assert address == self.accounts[account].get_address(*sequence)
 722             rs = self.rebase_sequence( account, sequence)
 723             for root, public_sequence in rs:
 724
 725                 if root not in self.master_private_keys.keys(): continue
 726                 master_k = self.get_master_private_key(root, password)
 727                 master_c, _, _ = self.master_public_keys[root]
 728                 pk = bip32_private_key( public_sequence, master_k.decode('hex'), master_c.decode('hex'))
 729                 out.append(pk)
 730
 731         return out
 732
 733
 734     def add_keypairs_from_wallet(self, tx, keypairs, password):
 735         for txin in tx.inputs:
 736             address = txin['address']
 737             private_keys = self.get_private_key(address, password)
 738             for sec in private_keys:
 739                 pubkey = public_key_from_private_key(sec)
 740                 keypairs[ pubkey ] = sec
 741                 if address in self.imported_keys.keys():
 742                     txin['redeemPubkey'] = pubkey
 743
 744
 745     def add_keypairs_from_KeyID(self, tx, keypairs, password):
 746         for txin in tx.inputs:
 747             keyid = txin.get('KeyID')
 748             if keyid:
 749
 750                 if self.seed_version==4:
 751                     m = re.match("old\(([0-9a-f]+),(\d+),(\d+)", keyid)
 752                     if not m: continue
 753                     mpk = m.group(1)
 754                     if mpk != self.storage.get('master_public_key'): continue
 755                     index = int(m.group(2))
 756                     num = int(m.group(3))
 757                     account = self.accounts[0]
 758                     addr = account.get_address(index, num)
 759                     txin['address'] = addr # fixme: side effect
 760                     pk = self.get_private_key(addr, password)
 761                     for sec in pk:
 762                         pubkey = public_key_from_private_key(sec)
 763                         keypairs[pubkey] = sec
 764                     continue
 765
 766
 767                 roots = []
 768                 for s in keyid.split('&'):
 769                     m = re.match("bip32\(([0-9a-f]+),([0-9a-f]+),(/\d+/\d+/\d+)", s)
 770                     if not m: continue
 771                     c = m.group(1)
 772                     K = m.group(2)
 773                     sequence = m.group(3)
 774                     root = self.find_root_by_master_key(c,K)
 775                     if not root: continue
 776                     sequence = map(lambda x:int(x), sequence.strip('/').split('/'))
 777                     root = root + '%d'%sequence[0]
 778                     sequence = sequence[1:]
 779                     roots.append((root,sequence))
 780
 781                 account_id = " & ".join( map(lambda x:x[0], roots) )
 782                 account = self.accounts.get(account_id)
 783                 if not account: continue
 784                 addr = account.get_address(*sequence)
 785                 txin['address'] = addr # fixme: side effect
 786                 pk = self.get_private_key(addr, password)
 787                 for sec in pk:
 788                     pubkey = public_key_from_private_key(sec)
 789                     keypairs[pubkey] = sec
 790
 791
 792
 793     def signrawtransaction(self, tx, input_info, private_keys, password):
 794
 795         # check that the password is correct
 796         seed = self.get_seed(password)
 797
 798         # add input info
 799         tx.add_input_info(input_info)
 800
 801         # add redeem script for coins that are in the wallet
 802         # FIXME: add redeemPubkey too!
 803         unspent_coins = self.get_unspent_coins()
 804         for txin in tx.inputs:
 805             for item in unspent_coins:
 806                 if txin['prevout_hash'] == item['prevout_hash'] and txin['prevout_n'] == item['prevout_n']:
 807                     print_error( "tx input is in unspent coins" )
 808                     txin['scriptPubKey'] = item['scriptPubKey']
 809                     account, sequence = self.get_address_index(item['address'])
 810                     if account != -1:
 811                         txin['redeemScript'] = self.accounts[account].redeem_script(sequence)
 812                         print_error("added redeemScript", txin['redeemScript'])
 813                     break
 814
 815
 816         # build a list of public/private keys
 817         keypairs = {}
 818
 819         # add private keys from parameter
 820         for sec in private_keys:
 821             pubkey = public_key_from_private_key(sec)
 822             keypairs[ pubkey ] = sec
 823
 824         # add private_keys from KeyID
 825         self.add_keypairs_from_KeyID(tx, keypairs, password)
 826
 827         # add private keys from wallet
 828         self.add_keypairs_from_wallet(tx, keypairs, password)
 829         self.sign_transaction(tx, keypairs)
 830
 831
 832     def sign_message(self, address, message, password):
 833         keys = self.get_private_key(address, password)
 834         assert len(keys) == 1
 835         sec = keys[0]
 836         key = regenerate_key(sec)
 837         compressed = is_compressed(sec)
 838         return key.sign_message(message, compressed, address)
 839
 840
 841     def change_gap_limit(self, value):
 842         if value >= self.gap_limit:
 843             self.gap_limit = value
 844             self.storage.put('gap_limit', self.gap_limit, True)
 845             #self.interface.poke('synchronizer')
 846             return True
 847
 848         elif value >= self.min_acceptable_gap():
 849             for key, account in self.accounts.items():
 850                 addresses = account[0]
 851                 k = self.num_unused_trailing_addresses(addresses)
 852                 n = len(addresses) - k + value
 853                 addresses = addresses[0:n]
 854                 self.accounts[key][0] = addresses
 855
 856             self.gap_limit = value
 857             self.storage.put('gap_limit', self.gap_limit, True)
 858             self.save_accounts()
 859             return True
 860         else:
 861             return False
 862
 863     def num_unused_trailing_addresses(self, addresses):
 864         k = 0
 865         for a in addresses[::-1]:
 866             if self.history.get(a):break
 867             k = k + 1
 868         return k
 869
 870     def min_acceptable_gap(self):
 871         # fixme: this assumes wallet is synchronized
 872         n = 0
 873         nmax = 0
 874
 875         for account in self.accounts.values():
 876             addresses = account.get_addresses(0)
 877             k = self.num_unused_trailing_addresses(addresses)
 878             for a in addresses[0:-k]:
 879                 if self.history.get(a):
 880                     n = 0
 881                 else:
 882                     n += 1
 883                     if n > nmax: nmax = n
 884         return nmax + 1
 885
 886
 887     def address_is_old(self, address):
 888         age = -1
 889         h = self.history.get(address, [])
 890         if h == ['*']:
 891             return True
 892         for tx_hash, tx_height in h:
 893             if tx_height == 0:
 894                 tx_age = 0
 895             else:
 896                 tx_age = self.verifier.blockchain.height() - tx_height + 1
 897             if tx_age > age:
 898                 age = tx_age
 899         return age > 2
 900
 901
 902     def synchronize_sequence(self, account, for_change):
 903         limit = self.gap_limit_for_change if for_change else self.gap_limit
 904         new_addresses = []
 905         while True:
 906             addresses = account.get_addresses(for_change)
 907             if len(addresses) < limit:
 908                 address = account.create_new_address(for_change)
 909                 self.history[address] = []
 910                 new_addresses.append( address )
 911                 continue
 912
 913             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 914                 break
 915             else:
 916                 address = account.create_new_address(for_change)
 917                 self.history[address] = []
 918                 new_addresses.append( address )
 919
 920         return new_addresses
 921
 922
 923
 924     def create_pending_accounts(self):
 925         for account_type in ['1','2of2','2of3']:
 926             if not self.has_master_public_keys(account_type):
 927                 continue
 928             k, a = self.new_account_address(account_type)
 929             if self.address_is_old(a):
 930                 print_error( "creating account", a )
 931                 self.create_account(account_type)
 932                 self.next_addresses.pop(k)
 933
 934
 935     def synchronize_account(self, account):
 936         new = []
 937         new += self.synchronize_sequence(account, 0)
 938         new += self.synchronize_sequence(account, 1)
 939         return new
 940
 941
 942     def synchronize(self):
 943         if self.master_public_keys:
 944             self.create_pending_accounts()
 945         new = []
 946         for account in self.accounts.values():
 947             new += self.synchronize_account(account)
 948         if new:
 949             self.save_accounts()
 950             self.storage.put('addr_history', self.history, True)
 951         return new
 952
 953
 954     def is_found(self):
 955         return self.history.values() != [[]] * len(self.history)
 956
 957
 958     def add_contact(self, address, label=None):
 959         self.addressbook.append(address)
 960         self.storage.put('contacts', self.addressbook, True)
 961         if label:
 962             self.set_label(address, label)
 963
 964
 965     def delete_contact(self, addr):
 966         if addr in self.addressbook:
 967             self.addressbook.remove(addr)
 968             self.storage.put('addressbook', self.addressbook, True)
 969
 970
 971     def fill_addressbook(self):
 972         for tx_hash, tx in self.transactions.items():
 973             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 974             if is_send:
 975                 for addr, v in tx.outputs:
 976                     if not self.is_mine(addr) and addr not in self.addressbook:
 977                         self.addressbook.append(addr)
 978         # redo labels
 979         # self.update_tx_labels()
 980
 981     def get_num_tx(self, address):
 982         n = 0
 983         for tx in self.transactions.values():
 984             if address in map(lambda x:x[0], tx.outputs): n += 1
 985         return n
 986
 987
 988     def get_address_flags(self, addr):
 989         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 990         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 991         return flags
 992
 993
 994     def get_tx_value(self, tx, account=None):
 995         domain = self.get_account_addresses(account)
 996         return tx.get_value(domain, self.prevout_values)
 997
 998
 999     def update_tx_outputs(self, tx_hash):
1000         tx = self.transactions.get(tx_hash)
1001
1002         for i, (addr, value) in enumerate(tx.outputs):
1003             key = tx_hash+ ':%d'%i
1004             self.prevout_values[key] = value
1005
1006         for item in tx.inputs:
1007             if self.is_mine(item.get('address')):
1008                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
1009                 self.spent_outputs.append(key)
1010
1011
1012     def get_addr_balance(self, address):
1013         assert self.is_mine(address)
1014         h = self.history.get(address,[])
1015         if h == ['*']: return 0,0
1016         c = u = 0
1017         received_coins = []   # list of coins received at address
1018
1019         for tx_hash, tx_height in h:
1020             tx = self.transactions.get(tx_hash)
1021             if not tx: continue
1022
1023             for i, (addr, value) in enumerate(tx.outputs):
1024                 if addr == address:
1025                     key = tx_hash + ':%d'%i
1026                     received_coins.append(key)
1027
1028         for tx_hash, tx_height in h:
1029             tx = self.transactions.get(tx_hash)
1030             if not tx: continue
1031             v = 0
1032
1033             for item in tx.inputs:
1034                 addr = item.get('address')
1035                 if addr == address:
1036                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
1037                     value = self.prevout_values.get( key )
1038                     if key in received_coins:
1039                         v -= value
1040
1041             for i, (addr, value) in enumerate(tx.outputs):
1042                 key = tx_hash + ':%d'%i
1043                 if addr == address:
1044                     v += value
1045
1046             if tx_height:
1047                 c += v
1048             else:
1049                 u += v
1050         return c, u
1051
1052
1053     def get_account_name(self, k):
1054         if k == 0:
1055             if self.seed_version == 4:
1056                 name = 'Main account'
1057             else:
1058                 name = 'Old account'
1059         else:
1060             default = "Unnamed account"
1061             m = re.match("m/0'/(\d+)", k)
1062             if m:
1063                 num = m.group(1)
1064                 if num == '0':
1065                     default = "Main account"
1066                 else:
1067                     default = "Account %s"%num
1068
1069             m = re.match("m/1'/(\d+) & m/2'/(\d+)", k)
1070             if m:
1071                 num = m.group(1)
1072                 default = "2of2 account %s"%num
1073             name = self.labels.get(k, default)
1074
1075         return name
1076
1077     def get_account_names(self):
1078         accounts = {}
1079         for k, account in self.accounts.items():
1080             accounts[k] = self.get_account_name(k)
1081         if self.imported_keys:
1082             accounts[-1] = 'Imported keys'
1083         return accounts
1084
1085     def get_account_addresses(self, a, include_change=True):
1086         if a is None:
1087             o = self.addresses(True)
1088         elif a == -1:
1089             o = self.imported_keys.keys()
1090         else:
1091             ac = self.accounts[a]
1092             o = ac.get_addresses(0)
1093             if include_change: o += ac.get_addresses(1)
1094         return o
1095
1096     def get_imported_balance(self):
1097         return self.get_balance(self.imported_keys.keys())
1098
1099     def get_account_balance(self, account):
1100         return self.get_balance(self.get_account_addresses(account))
1101
1102     def get_frozen_balance(self):
1103         return self.get_balance(self.frozen_addresses)
1104
1105     def get_balance(self, domain=None):
1106         if domain is None: domain = self.addresses(True)
1107         cc = uu = 0
1108         for addr in domain:
1109             c, u = self.get_addr_balance(addr)
1110             cc += c
1111             uu += u
1112         return cc, uu
1113
1114
1115     def get_unspent_coins(self, domain=None):
1116         coins = []
1117         if domain is None: domain = self.addresses(True)
1118         for addr in domain:
1119             h = self.history.get(addr, [])
1120             if h == ['*']: continue
1121             for tx_hash, tx_height in h:
1122                 tx = self.transactions.get(tx_hash)
1123                 if tx is None: raise Exception("Wallet not synchronized")
1124                 is_coinbase = tx.inputs[0].get('prevout_hash') == '0'*64
1125                 for o in tx.d.get('outputs'):
1126                     output = o.copy()
1127                     if output.get('address') != addr: continue
1128                     key = tx_hash + ":%d" % output.get('prevout_n')
1129                     if key in self.spent_outputs: continue
1130                     output['prevout_hash'] = tx_hash
1131                     output['height'] = tx_height
1132                     output['coinbase'] = is_coinbase
1133                     coins.append((tx_height, output))
1134
1135         # sort by age
1136         if coins:
1137             coins = sorted(coins)
1138             if coins[-1][0] != 0:
1139                 while coins[0][0] == 0:
1140                     coins = coins[1:] + [ coins[0] ]
1141         return [x[1] for x in coins]
1142
1143
1144     def choose_tx_inputs( self, amount, fixed_fee, domain = None ):
1145         """ todo: minimize tx size """
1146         total = 0
1147         fee = self.fee if fixed_fee is None else fixed_fee
1148         if domain is None:
1149             domain = self.addresses(True)
1150
1151         for i in self.frozen_addresses:
1152             if i in domain: domain.remove(i)
1153
1154         prioritized = []
1155         for i in self.prioritized_addresses:
1156             if i in domain:
1157                 domain.remove(i)
1158                 prioritized.append(i)
1159
1160         coins = self.get_unspent_coins(domain)
1161         prioritized_coins = self.get_unspent_coins(prioritized)
1162
1163         inputs = []
1164         coins = prioritized_coins + coins
1165
1166         for item in coins:
1167             if item.get('coinbase') and item.get('height') + COINBASE_MATURITY > self.network.blockchain.height:
1168                 continue
1169             addr = item.get('address')
1170             v = item.get('value')
1171             total += v
1172             inputs.append(item)
1173             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
1174             if total >= amount + fee: break
1175         else:
1176             inputs = []
1177
1178         return inputs, total, fee
1179
1180
1181     def set_fee(self, fee):
1182         if self.fee != fee:
1183             self.fee = fee
1184             self.storage.put('fee_per_kb', self.fee, True)
1185
1186     def estimated_fee(self, inputs):
1187         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
1188         fee = self.fee * int(round(estimated_size/1024.))
1189         if fee == 0: fee = self.fee
1190         return fee
1191
1192
1193     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None):
1194         "add change to a transaction"
1195         change_amount = total - ( amount + fee )
1196         if change_amount > DUST_THRESHOLD:
1197             if not change_addr:
1198
1199                 # send change to one of the accounts involved in the tx
1200                 address = inputs[0].get('address')
1201                 account, _ = self.get_address_index(address)
1202
1203                 if not self.use_change or account == -1:
1204                     change_addr = inputs[-1]['address']
1205                 else:
1206                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
1207
1208             # Insert the change output at a random position in the outputs
1209             posn = random.randint(0, len(outputs))
1210             outputs[posn:posn] = [( change_addr,  change_amount)]
1211         return outputs
1212
1213
1214     def get_history(self, address):
1215         with self.lock:
1216             return self.history.get(address)
1217
1218
1219     def get_status(self, h):
1220         if not h: return None
1221         if h == ['*']: return '*'
1222         status = ''
1223         for tx_hash, height in h:
1224             status += tx_hash + ':%d:' % height
1225         return hashlib.sha256( status ).digest().encode('hex')
1226
1227
1228     def receive_tx_callback(self, tx_hash, tx, tx_height):
1229
1230         with self.transaction_lock:
1231             self.add_extra_addresses(tx)
1232             if not self.check_new_tx(tx_hash, tx):
1233                 # may happen due to pruning
1234                 print_error("received transaction that is no longer referenced in history", tx_hash)
1235                 return
1236             self.transactions[tx_hash] = tx
1237             self.network.interface.pending_transactions_for_notifications.append(tx)
1238             self.save_transactions()
1239             if self.verifier and tx_height>0:
1240                 self.verifier.add(tx_hash, tx_height)
1241             self.update_tx_outputs(tx_hash)
1242
1243
1244     def save_transactions(self):
1245         tx = {}
1246         for k,v in self.transactions.items():
1247             tx[k] = str(v)
1248         self.storage.put('transactions', tx, True)
1249
1250     def receive_history_callback(self, addr, hist):
1251
1252         if not self.check_new_history(addr, hist):
1253             raise Exception("error: received history for %s is not consistent with known transactions"%addr)
1254
1255         with self.lock:
1256             self.history[addr] = hist
1257             self.storage.put('addr_history', self.history, True)
1258
1259         if hist != ['*']:
1260             for tx_hash, tx_height in hist:
1261                 if tx_height>0:
1262                     # add it in case it was previously unconfirmed
1263                     if self.verifier: self.verifier.add(tx_hash, tx_height)
1264
1265
1266     def get_tx_history(self, account=None):
1267         if not self.verifier:
1268             return []
1269
1270         with self.transaction_lock:
1271             history = self.transactions.items()
1272             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
1273             result = []
1274
1275             balance = 0
1276             for tx_hash, tx in history:
1277                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
1278                 if v is not None: balance += v
1279
1280             c, u = self.get_account_balance(account)
1281
1282             if balance != c+u:
1283                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
1284
1285             balance = c + u - balance
1286             for tx_hash, tx in history:
1287                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
1288                 if not is_relevant:
1289                     continue
1290                 if value is not None:
1291                     balance += value
1292
1293                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
1294                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
1295
1296         return result
1297
1298
1299     def get_label(self, tx_hash):
1300         label = self.labels.get(tx_hash)
1301         is_default = (label == '') or (label is None)
1302         if is_default: label = self.get_default_label(tx_hash)
1303         return label, is_default
1304
1305
1306     def get_default_label(self, tx_hash):
1307         tx = self.transactions.get(tx_hash)
1308         default_label = ''
1309         if tx:
1310             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
1311             if is_mine:
1312                 for o in tx.outputs:
1313                     o_addr, _ = o
1314                     if not self.is_mine(o_addr):
1315                         try:
1316                             default_label = self.labels[o_addr]
1317                         except KeyError:
1318                             default_label = o_addr
1319                         break
1320                 else:
1321                     default_label = '(internal)'
1322             else:
1323                 for o in tx.outputs:
1324                     o_addr, _ = o
1325                     if self.is_mine(o_addr) and not self.is_change(o_addr):
1326                         break
1327                 else:
1328                     for o in tx.outputs:
1329                         o_addr, _ = o
1330                         if self.is_mine(o_addr):
1331                             break
1332                     else:
1333                         o_addr = None
1334
1335                 if o_addr:
1336                     dest_label = self.labels.get(o_addr)
1337                     try:
1338                         default_label = self.labels[o_addr]
1339                     except KeyError:
1340                         default_label = o_addr
1341
1342         return default_label
1343
1344
1345     def make_unsigned_transaction(self, outputs, fee=None, change_addr=None, domain=None ):
1346         for address, x in outputs:
1347             assert is_valid(address)
1348         amount = sum( map(lambda x:x[1], outputs) )
1349         inputs, total, fee = self.choose_tx_inputs( amount, fee, domain )
1350         if not inputs:
1351             raise ValueError("Not enough funds")
1352         self.add_input_info(inputs)
1353         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr)
1354         return Transaction.from_io(inputs, outputs)
1355
1356
1357     def mktx(self, outputs, password, fee=None, change_addr=None, domain= None ):
1358         tx = self.make_unsigned_transaction(outputs, fee, change_addr, domain)
1359         keypairs = {}
1360         self.add_keypairs_from_wallet(tx, keypairs, password)
1361         if keypairs:
1362             self.sign_transaction(tx, keypairs)
1363         return tx
1364
1365
1366     def add_input_info(self, inputs):
1367         for txin in inputs:
1368             address = txin['address']
1369             if address in self.imported_keys.keys():
1370                 continue
1371             account, sequence = self.get_address_index(address)
1372             txin['KeyID'] = self.get_keyID(account, sequence)
1373             redeemScript = self.accounts[account].redeem_script(sequence)
1374             if redeemScript:
1375                 txin['redeemScript'] = redeemScript
1376             else:
1377                 txin['redeemPubkey'] = self.accounts[account].get_pubkey(*sequence)
1378
1379
1380     def sign_transaction(self, tx, keypairs):
1381         tx.sign(keypairs)
1382         run_hook('sign_transaction', tx)
1383
1384
1385     def sendtx(self, tx):
1386         # synchronous
1387         h = self.send_tx(tx)
1388         self.tx_event.wait()
1389         return self.receive_tx(h)
1390
1391     def send_tx(self, tx):
1392         # asynchronous
1393         self.tx_event.clear()
1394         self.network.interface.send([('blockchain.transaction.broadcast', [str(tx)])], self.on_broadcast)
1395         return tx.hash()
1396
1397     def on_broadcast(self, i, r):
1398         self.tx_result = r.get('result')
1399         self.tx_event.set()
1400
1401     def receive_tx(self,tx_hash):
1402         out = self.tx_result
1403         if out != tx_hash:
1404             return False, "error: " + out
1405         return True, out
1406
1407
1408
1409     def update_password(self, old_password, new_password):
1410         if new_password == '': new_password = None
1411         decoded = self.get_seed(old_password)
1412         self.seed = pw_encode( decoded, new_password)
1413         self.storage.put('seed', self.seed, True)
1414         self.use_encryption = (new_password != None)
1415         self.storage.put('use_encryption', self.use_encryption,True)
1416         for k in self.imported_keys.keys():
1417             a = self.imported_keys[k]
1418             b = pw_decode(a, old_password)
1419             c = pw_encode(b, new_password)
1420             self.imported_keys[k] = c
1421         self.storage.put('imported_keys', self.imported_keys, True)
1422
1423         for k, v in self.master_private_keys.items():
1424             b = pw_decode(v, old_password)
1425             c = pw_encode(b, new_password)
1426             self.master_private_keys[k] = c
1427         self.storage.put('master_private_keys', self.master_private_keys, True)
1428
1429
1430     def freeze(self,addr):
1431         if self.is_mine(addr) and addr not in self.frozen_addresses:
1432             self.unprioritize(addr)
1433             self.frozen_addresses.append(addr)
1434             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1435             return True
1436         else:
1437             return False
1438
1439     def unfreeze(self,addr):
1440         if self.is_mine(addr) and addr in self.frozen_addresses:
1441             self.frozen_addresses.remove(addr)
1442             self.storage.put('frozen_addresses', self.frozen_addresses, True)
1443             return True
1444         else:
1445             return False
1446
1447     def prioritize(self,addr):
1448         if self.is_mine(addr) and addr not in self.prioritized_addresses:
1449             self.unfreeze(addr)
1450             self.prioritized_addresses.append(addr)
1451             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1452             return True
1453         else:
1454             return False
1455
1456     def unprioritize(self,addr):
1457         if self.is_mine(addr) and addr in self.prioritized_addresses:
1458             self.prioritized_addresses.remove(addr)
1459             self.storage.put('prioritized_addresses', self.prioritized_addresses, True)
1460             return True
1461         else:
1462             return False
1463
1464
1465     def set_verifier(self, verifier):
1466         self.verifier = verifier
1467
1468         # review transactions that are in the history
1469         for addr, hist in self.history.items():
1470             if hist == ['*']: continue
1471             for tx_hash, tx_height in hist:
1472                 if tx_height>0:
1473                     # add it in case it was previously unconfirmed
1474                     self.verifier.add(tx_hash, tx_height)
1475
1476
1477         # if we are on a pruning server, remove unverified transactions
1478         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1479         for tx_hash in self.transactions.keys():
1480             if tx_hash not in vr:
1481                 self.transactions.pop(tx_hash)
1482
1483
1484
1485     def check_new_history(self, addr, hist):
1486
1487         # check that all tx in hist are relevant
1488         if hist != ['*']:
1489             for tx_hash, height in hist:
1490                 tx = self.transactions.get(tx_hash)
1491                 if not tx: continue
1492                 if not tx.has_address(addr):
1493                     return False
1494
1495         # check that we are not "orphaning" a transaction
1496         old_hist = self.history.get(addr,[])
1497         if old_hist == ['*']: return True
1498
1499         for tx_hash, height in old_hist:
1500             if tx_hash in map(lambda x:x[0], hist): continue
1501             found = False
1502             for _addr, _hist in self.history.items():
1503                 if _addr == addr: continue
1504                 if _hist == ['*']: continue
1505                 _tx_hist = map(lambda x:x[0], _hist)
1506                 if tx_hash in _tx_hist:
1507                     found = True
1508                     break
1509
1510             if not found:
1511                 tx = self.transactions.get(tx_hash)
1512                 # tx might not be there
1513                 if not tx: continue
1514
1515                 # already verified?
1516                 if self.verifier.get_height(tx_hash):
1517                     continue
1518                 # unconfirmed tx
1519                 print_error("new history is orphaning transaction:", tx_hash)
1520                 # check that all outputs are not mine, request histories
1521                 ext_requests = []
1522                 for _addr, _v in tx.outputs:
1523                     # assert not self.is_mine(_addr)
1524                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1525
1526                 ext_h = self.network.synchronous_get(ext_requests)
1527                 print_error("sync:", ext_requests, ext_h)
1528                 height = None
1529                 for h in ext_h:
1530                     if h == ['*']: continue
1531                     for item in h:
1532                         if item.get('tx_hash') == tx_hash:
1533                             height = item.get('height')
1534                 if height:
1535                     print_error("found height for", tx_hash, height)
1536                     self.verifier.add(tx_hash, height)
1537                 else:
1538                     print_error("removing orphaned tx from history", tx_hash)
1539                     self.transactions.pop(tx_hash)
1540
1541         return True
1542
1543
1544
1545     def check_new_tx(self, tx_hash, tx):
1546         # 1 check that tx is referenced in addr_history.
1547         addresses = []
1548         for addr, hist in self.history.items():
1549             if hist == ['*']:continue
1550             for txh, height in hist:
1551                 if txh == tx_hash:
1552                     addresses.append(addr)
1553
1554         if not addresses:
1555             return False
1556
1557         # 2 check that referencing addresses are in the tx
1558         for addr in addresses:
1559             if not tx.has_address(addr):
1560                 return False
1561
1562         return True
1563
1564
1565     def start_threads(self, network):
1566         from verifier import TxVerifier
1567         self.network = network
1568         if self.network:
1569             self.verifier = TxVerifier(self.network, self.storage)
1570             self.verifier.start()
1571             self.set_verifier(self.verifier)
1572             self.synchronizer = WalletSynchronizer(self, network)
1573             self.synchronizer.start()
1574         else:
1575             self.verifier = None
1576             self.synchronizer =None
1577
1578     def stop_threads(self):
1579         if self.network:
1580             self.verifier.stop()
1581             self.synchronizer.stop()
1582
1583
1584     def restore(self, callback):
1585         from i18n import _
1586         def wait_for_wallet():
1587             self.set_up_to_date(False)
1588             while not self.is_up_to_date():
1589                 msg = "%s\n%s %d\n%s %.1f"%(
1590                     _("Please wait..."),
1591                     _("Addresses generated:"),
1592                     len(self.addresses(True)),
1593                     _("Kilobytes received:"),
1594                     self.network.interface.bytes_received/1024.)
1595
1596                 apply(callback, (msg,))
1597                 time.sleep(0.1)
1598
1599         def wait_for_network():
1600             while not self.network.is_connected():
1601                 msg = "%s \n" % (_("Connecting..."))
1602                 apply(callback, (msg,))
1603                 time.sleep(0.1)
1604
1605         # wait until we are connected, because the user might have selected another server
1606         if self.network:
1607             wait_for_network()
1608             wait_for_wallet()
1609         else:
1610             self.synchronize()
1611
1612         self.fill_addressbook()
1613
1614
1615
1616
1617 class WalletSynchronizer(threading.Thread):
1618
1619
1620     def __init__(self, wallet, network):
1621         threading.Thread.__init__(self)
1622         self.daemon = True
1623         self.wallet = wallet
1624         self.network = network
1625         self.was_updated = True
1626         self.running = False
1627         self.lock = threading.Lock()
1628         self.queue = Queue.Queue()
1629
1630     def stop(self):
1631         with self.lock: self.running = False
1632
1633     def is_running(self):
1634         with self.lock: return self.running
1635
1636
1637     def subscribe_to_addresses(self, addresses):
1638         messages = []
1639         for addr in addresses:
1640             messages.append(('blockchain.address.subscribe', [addr]))
1641         self.network.subscribe( messages, lambda i,r: self.queue.put(r))
1642
1643
1644     def run(self):
1645         with self.lock:
1646             self.running = True
1647
1648         while self.is_running():
1649
1650             if not self.network.is_connected():
1651                 print_error("synchronizer: waiting for interface")
1652                 self.network.wait_until_connected()
1653
1654             self.run_interface(self.network.interface)
1655
1656
1657     def run_interface(self, interface):
1658
1659         print_error("synchronizer: connected to", interface.server)
1660
1661         requested_tx = []
1662         missing_tx = []
1663         requested_histories = {}
1664
1665         # request any missing transactions
1666         for history in self.wallet.history.values():
1667             if history == ['*']: continue
1668             for tx_hash, tx_height in history:
1669                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1670                     missing_tx.append( (tx_hash, tx_height) )
1671
1672         if missing_tx:
1673             print_error("missing tx", missing_tx)
1674
1675         # subscriptions
1676         self.subscribe_to_addresses(self.wallet.addresses(True))
1677
1678         while self.is_running():
1679             # 1. create new addresses
1680             new_addresses = self.wallet.synchronize()
1681
1682             # request missing addresses
1683             if new_addresses:
1684                 self.subscribe_to_addresses(new_addresses)
1685
1686             # request missing transactions
1687             for tx_hash, tx_height in missing_tx:
1688                 if (tx_hash, tx_height) not in requested_tx:
1689                     interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], lambda i,r: self.queue.put(r))
1690                     requested_tx.append( (tx_hash, tx_height) )
1691             missing_tx = []
1692
1693             # detect if situation has changed
1694             if interface.is_up_to_date() and self.queue.empty():
1695                 if not self.wallet.is_up_to_date():
1696                     self.wallet.set_up_to_date(True)
1697                     self.was_updated = True
1698             else:
1699                 if self.wallet.is_up_to_date():
1700                     self.wallet.set_up_to_date(False)
1701                     self.was_updated = True
1702
1703             if self.was_updated:
1704                 self.wallet.network.trigger_callback('updated')
1705                 self.was_updated = False
1706
1707             # 2. get a response
1708             try:
1709                 r = self.queue.get(block=True, timeout=1)
1710             except Queue.Empty:
1711                 continue
1712
1713             if interface != self.network.interface:
1714                 break
1715
1716             if not r:
1717                 continue
1718
1719             # 3. handle response
1720             method = r['method']
1721             params = r['params']
1722             result = r.get('result')
1723             error = r.get('error')
1724             if error:
1725                 print "error", r
1726                 continue
1727
1728             if method == 'blockchain.address.subscribe':
1729                 addr = params[0]
1730                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1731                     if requested_histories.get(addr) is None:
1732                         interface.send([('blockchain.address.get_history', [addr])], lambda i,r:self.queue.put(r))
1733                         requested_histories[addr] = result
1734
1735             elif method == 'blockchain.address.get_history':
1736                 addr = params[0]
1737                 print_error("receiving history", addr, result)
1738                 if result == ['*']:
1739                     assert requested_histories.pop(addr) == '*'
1740                     self.wallet.receive_history_callback(addr, result)
1741                 else:
1742                     hist = []
1743                     # check that txids are unique
1744                     txids = []
1745                     for item in result:
1746                         tx_hash = item['tx_hash']
1747                         if tx_hash not in txids:
1748                             txids.append(tx_hash)
1749                             hist.append( (tx_hash, item['height']) )
1750
1751                     if len(hist) != len(result):
1752                         raise Exception("error: server sent history with non-unique txid", result)
1753
1754                     # check that the status corresponds to what was announced
1755                     rs = requested_histories.pop(addr)
1756                     if self.wallet.get_status(hist) != rs:
1757                         raise Exception("error: status mismatch: %s"%addr)
1758
1759                     # store received history
1760                     self.wallet.receive_history_callback(addr, hist)
1761
1762                     # request transactions that we don't have
1763                     for tx_hash, tx_height in hist:
1764                         if self.wallet.transactions.get(tx_hash) is None:
1765                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1766                                 missing_tx.append( (tx_hash, tx_height) )
1767
1768             elif method == 'blockchain.transaction.get':
1769                 tx_hash = params[0]
1770                 tx_height = params[1]
1771                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1772                 tx = Transaction(result)
1773                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1774                 self.was_updated = True
1775                 requested_tx.remove( (tx_hash, tx_height) )
1776                 print_error("received tx:", tx_hash, len(tx.raw))
1777
1778             else:
1779                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1780
1781             if self.was_updated and not requested_tx:
1782                 self.wallet.network.trigger_callback('updated')
1783                 self.wallet.network.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1784
1785                 self.was_updated = False