lib/wallet.py

   1 #!/usr/bin/env python
   2 #
   3 # Electrum - lightweight Bitcoin client
   4 # Copyright (C) 2011 thomasv@gitorious
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18
  19 import sys
  20 import base64
  21 import os
  22 import re
  23 import hashlib
  24 import copy
  25 import operator
  26 import ast
  27 import threading
  28 import random
  29 import aes
  30 import Queue
  31 import time
  32
  33 from util import print_msg, print_error, user_dir, format_satoshis
  34 from bitcoin import *
  35 from account import *
  36
  37 # AES encryption
  38 EncodeAES = lambda secret, s: base64.b64encode(aes.encryptData(secret,s))
  39 DecodeAES = lambda secret, e: aes.decryptData(secret, base64.b64decode(e))
  40
  41 def pw_encode(s, password):
  42     if password:
  43         secret = Hash(password)
  44         return EncodeAES(secret, s)
  45     else:
  46         return s
  47
  48 def pw_decode(s, password):
  49     if password is not None:
  50         secret = Hash(password)
  51         try:
  52             d = DecodeAES(secret, s)
  53         except:
  54             raise BaseException('Invalid password')
  55         return d
  56     else:
  57         return s
  58
  59
  60
  61
  62
  63 from version import ELECTRUM_VERSION, SEED_VERSION
  64
  65
  66 class Wallet:
  67     def __init__(self, config={}):
  68
  69         self.config = config
  70         self.electrum_version = ELECTRUM_VERSION
  71         self.gap_limit_for_change = 3 # constant
  72
  73         # saved fields
  74         self.seed_version          = config.get('seed_version', SEED_VERSION)
  75         self.gap_limit             = config.get('gap_limit', 5)
  76         self.use_change            = config.get('use_change',True)
  77         self.fee                   = int(config.get('fee_per_kb',50000))
  78         self.num_zeros             = int(config.get('num_zeros',0))
  79         self.use_encryption        = config.get('use_encryption', False)
  80         self.seed                  = config.get('seed', '')               # encrypted
  81         self.labels                = config.get('labels', {})
  82         self.frozen_addresses      = config.get('frozen_addresses',[])
  83         self.prioritized_addresses = config.get('prioritized_addresses',[])
  84         self.addressbook           = config.get('contacts', [])
  85
  86         self.imported_keys         = config.get('imported_keys',{})
  87         self.history               = config.get('addr_history',{})        # address -> list(txid, height)
  88
  89
  90         self.master_public_keys = config.get('master_public_keys',{})
  91         self.master_private_keys = config.get('master_private_keys', {})
  92
  93         self.load_accounts(config)
  94
  95         self.transactions = {}
  96         tx = config.get('transactions',{})
  97         try:
  98             for k,v in tx.items(): self.transactions[k] = Transaction(v)
  99         except:
 100             print_msg("Warning: Cannot deserialize transactions. skipping")
 101
 102         # not saved
 103         self.prevout_values = {}     # my own transaction outputs
 104         self.spent_outputs = []
 105
 106         # spv
 107         self.verifier = None
 108
 109         # there is a difference between wallet.up_to_date and interface.is_up_to_date()
 110         # interface.is_up_to_date() returns true when all requests have been answered and processed
 111         # wallet.up_to_date is true when the wallet is synchronized (stronger requirement)
 112
 113         self.up_to_date = False
 114         self.lock = threading.Lock()
 115         self.transaction_lock = threading.Lock()
 116         self.tx_event = threading.Event()
 117
 118         if self.seed_version != SEED_VERSION:
 119             raise ValueError("This wallet seed is deprecated. Please run upgrade.py for a diagnostic.")
 120
 121         for tx_hash, tx in self.transactions.items():
 122             if self.check_new_tx(tx_hash, tx):
 123                 self.update_tx_outputs(tx_hash)
 124             else:
 125                 print_error("unreferenced tx", tx_hash)
 126                 self.transactions.pop(tx_hash)
 127
 128
 129     def set_up_to_date(self,b):
 130         with self.lock: self.up_to_date = b
 131
 132     def is_up_to_date(self):
 133         with self.lock: return self.up_to_date
 134
 135     def update(self):
 136         self.up_to_date = False
 137         self.interface.poke('synchronizer')
 138         while not self.is_up_to_date(): time.sleep(0.1)
 139
 140     def import_key(self, sec, password):
 141         # check password
 142         seed = self.decode_seed(password)
 143         try:
 144             address = address_from_private_key(sec)
 145         except:
 146             raise BaseException('Invalid private key')
 147
 148         if self.is_mine(address):
 149             raise BaseException('Address already in wallet')
 150
 151         # store the originally requested keypair into the imported keys table
 152         self.imported_keys[address] = pw_encode(sec, password )
 153         self.config.set_key('imported_keys', self.imported_keys, True)
 154         return address
 155
 156     def delete_imported_key(self, addr):
 157         if addr in self.imported_keys:
 158             self.imported_keys.pop(addr)
 159             self.config.set_key('imported_keys', self.imported_keys, True)
 160
 161
 162     def init_seed(self, seed):
 163         if self.seed: raise BaseException("a seed exists")
 164         if not seed:
 165             seed = random_seed(128)
 166         self.seed = seed
 167
 168
 169     def save_seed(self):
 170         self.config.set_key('seed', self.seed, True)
 171         self.config.set_key('seed_version', self.seed_version, True)
 172
 173         master_k, master_c, master_K, master_cK = bip32_init(self.seed)
 174
 175         k0, c0, K0, cK0 = bip32_private_derivation(master_k, master_c, "m/", "m/0'/")
 176         k1, c1, K1, cK1 = bip32_private_derivation(master_k, master_c, "m/", "m/1'/")
 177         k2, c2, K2, cK2 = bip32_private_derivation(master_k, master_c, "m/", "m/2'/")
 178
 179         self.master_public_keys = {
 180             "m/0'/": (c0, K0, cK0),
 181             "m/1'/": (c1, K1, cK1),
 182             "m/2'/": (c2, K2, cK2)
 183             }
 184
 185         self.master_private_keys = {
 186             "m/0'/": k0,
 187             "m/1'/": k1
 188             }
 189         # send k2 to service
 190
 191         self.config.set_key('master_public_keys', self.master_public_keys, True)
 192         self.config.set_key('master_private_keys', self.master_private_keys, True)
 193
 194         # create default account
 195         self.create_new_account('Main account')
 196
 197
 198     def create_new_account(self, name, password):
 199         keys = self.accounts.keys()
 200         i = 0
 201
 202         while True:
 203             derivation = "m/0'/%d'"%i
 204             if derivation not in keys: break
 205             i += 1
 206
 207         start = "m/0'/"
 208         master_k = self.get_master_private_key(start, password )
 209         master_c, master_K, master_cK = self.master_public_keys[start]
 210         k, c, K, cK = bip32_private_derivation(master_k, master_c, start, derivation)
 211
 212         self.accounts[derivation] = BIP32_Account({ 'name':name, 'c':c, 'K':K, 'cK':cK })
 213         self.save_accounts()
 214
 215     def create_p2sh_account(self, name):
 216         keys = self.accounts.keys()
 217         i = 0
 218         while True:
 219             account_id = "m/1'/%d & m/2'/%d"%(i,i)
 220             if account_id not in keys: break
 221             i += 1
 222
 223         master_c1, master_K1, _ = self.master_public_keys["m/1'/"]
 224         c1, K1, cK1 = bip32_public_derivation(master_c1.decode('hex'), master_K1.decode('hex'), "m/1'/", "m/1'/%d"%i)
 225
 226         master_c2, master_K2, _ = self.master_public_keys["m/2'/"]
 227         c2, K2, cK2 = bip32_public_derivation(master_c2.decode('hex'), master_K2.decode('hex'), "m/2'/", "m/2'/%d"%i)
 228
 229         self.accounts[account_id] = BIP32_Account_2of2({ 'name':name, 'c':c1, 'K':K1, 'cK':cK1, 'c2':c2, 'K2':K2, 'cK2':cK2 })
 230         self.save_accounts()
 231
 232
 233     def save_accounts(self):
 234         d = {}
 235         for k, v in self.accounts.items():
 236             d[k] = v.dump()
 237         self.config.set_key('accounts', d, True)
 238
 239
 240     def load_accounts(self, config):
 241         d = config.get('accounts', {})
 242         self.accounts = {}
 243         for k, v in d.items():
 244             if '&' in k:
 245                 self.accounts[k] = BIP32_Account_2of2(v)
 246             else:
 247                 self.accounts[k] = BIP32_Account(v)
 248
 249
 250
 251
 252     def addresses(self, include_change = True):
 253         o = self.get_account_addresses(-1, include_change)
 254         for a in self.accounts.keys():
 255             o += self.get_account_addresses(a, include_change)
 256         return o
 257
 258
 259     def is_mine(self, address):
 260         return address in self.addresses(True)
 261
 262     def is_change(self, address):
 263         if not self.is_mine(address): return False
 264         if address in self.imported_keys.keys(): return False
 265         acct, s = self.get_address_index(address)
 266         return s[0] == 1
 267
 268     def get_master_public_key(self):
 269         raise
 270         return self.config.get("master_public_key")
 271
 272     def get_master_private_key(self, account, password):
 273         master_k = pw_decode( self.master_private_keys[account], password)
 274         master_c, master_K, master_Kc = self.master_public_keys[account]
 275         try:
 276             K, Kc = get_pubkeys_from_secret(master_k.decode('hex'))
 277             assert K.encode('hex') == master_K
 278         except:
 279             raise BaseException("Invalid password")
 280         return master_k
 281
 282
 283     def get_address_index(self, address):
 284         if address in self.imported_keys.keys():
 285             return -1, None
 286         for account in self.accounts.keys():
 287             for for_change in [0,1]:
 288                 addresses = self.accounts[account].get_addresses(for_change)
 289                 for addr in addresses:
 290                     if address == addr:
 291                         return account, (for_change, addresses.index(addr))
 292         raise BaseException("not found")
 293
 294
 295     def get_public_key(self, address):
 296         account, sequence = self.get_address_index(address)
 297         return self.accounts[account].get_pubkey( sequence )
 298
 299
 300     def decode_seed(self, password):
 301         seed = pw_decode(self.seed, password)
 302         #todo:  #self.sequences[0].check_seed(seed)
 303         return seed
 304
 305
 306     def get_private_key(self, address, password):
 307         if address in self.imported_keys.keys():
 308             return pw_decode( self.imported_keys[address], password )
 309         else:
 310             account, sequence = self.get_address_index(address)
 311             m = re.match("m/0'/(\d+)'", account)
 312             if m:
 313                 num = int(m.group(1))
 314                 master_k = self.get_master_private_key("m/0'/", password)
 315                 master_c, _, _ = self.master_public_keys["m/0'/"]
 316                 master_k, master_c = CKD(master_k, master_c, num + BIP32_PRIME)
 317                 return self.accounts[account].get_private_key(sequence, master_k)
 318
 319             m2 = re.match("m/1'/(\d+) & m/2'/(\d+)", account)
 320             if m2:
 321                 num = int(m2.group(1))
 322                 master_k = self.get_master_private_key("m/1'/", password)
 323                 master_c, master_K, _ = self.master_public_keys["m/1'/"]
 324                 master_k, master_c = CKD(master_k.decode('hex'), master_c.decode('hex'), num)
 325                 return self.accounts[account].get_private_key(sequence, master_k)
 326         return
 327
 328
 329     def get_private_keys(self, addresses, password):
 330         if not self.seed: return {}
 331         # decode seed in any case, in order to test the password
 332         seed = self.decode_seed(password)
 333         out = {}
 334         for address in addresses:
 335             pk = self.get_private_key(address, password)
 336             if pk: out[address] = pk
 337
 338         return out
 339
 340
 341     def signrawtransaction(self, tx, input_info, private_keys, password):
 342         unspent_coins = self.get_unspent_coins()
 343         seed = self.decode_seed(password)
 344
 345         # convert private_keys to dict
 346         pk = {}
 347         for sec in private_keys:
 348             address = address_from_private_key(sec)
 349             pk[address] = sec
 350         private_keys = pk
 351
 352         for txin in tx.inputs:
 353             # convert to own format
 354             txin['tx_hash'] = txin['prevout_hash']
 355             txin['index'] = txin['prevout_n']
 356
 357             for item in input_info:
 358                 if item.get('txid') == txin['tx_hash'] and item.get('vout') == txin['index']:
 359                     txin['raw_output_script'] = item['scriptPubKey']
 360                     txin['redeemScript'] = item.get('redeemScript')
 361                     txin['KeyID'] = item.get('KeyID')
 362                     break
 363             else:
 364                 for item in unspent_coins:
 365                     if txin['tx_hash'] == item['tx_hash'] and txin['index'] == item['index']:
 366                         txin['raw_output_script'] = item['raw_output_script']
 367                         break
 368                 else:
 369                     # if neither, we might want to get it from the server..
 370                     raise
 371
 372             # find the address:
 373             if txin.get('KeyID'):
 374                 account, name, sequence = txin.get('KeyID')
 375                 if name != 'Electrum': continue
 376                 sec = self.accounts[account].get_private_key(sequence, seed)
 377                 addr = self.accounts[account].get_address(sequence)
 378                 txin['address'] = addr
 379                 private_keys[addr] = sec
 380
 381             elif txin.get("redeemScript"):
 382                 txin['address'] = hash_160_to_bc_address(hash_160(txin.get("redeemScript").decode('hex')), 5)
 383
 384             elif txin.get("raw_output_script"):
 385                 import deserialize
 386                 addr = deserialize.get_address_from_output_script(txin.get("raw_output_script").decode('hex'))
 387                 sec = self.get_private_key(addr, password)
 388                 if sec:
 389                     private_keys[addr] = sec
 390                     txin['address'] = addr
 391
 392         tx.sign( private_keys )
 393
 394     def sign_message(self, address, message, password):
 395         sec = self.get_private_key(address, password)
 396         key = regenerate_key(sec)
 397         compressed = is_compressed(sec)
 398         return key.sign_message(message, compressed, address)
 399
 400     def verify_message(self, address, signature, message):
 401         try:
 402             EC_KEY.verify_message(address, signature, message)
 403             return True
 404         except BaseException as e:
 405             print_error("Verification error: {0}".format(e))
 406             return False
 407
 408
 409     def change_gap_limit(self, value):
 410         if value >= self.gap_limit:
 411             self.gap_limit = value
 412             self.config.set_key('gap_limit', self.gap_limit, True)
 413             self.interface.poke('synchronizer')
 414             return True
 415
 416         elif value >= self.min_acceptable_gap():
 417             for key, account in self.accounts.items():
 418                 addresses = account[0]
 419                 k = self.num_unused_trailing_addresses(addresses)
 420                 n = len(addresses) - k + value
 421                 addresses = addresses[0:n]
 422                 self.accounts[key][0] = addresses
 423
 424             self.gap_limit = value
 425             self.config.set_key('gap_limit', self.gap_limit, True)
 426             self.save_accounts()
 427             return True
 428         else:
 429             return False
 430
 431     def num_unused_trailing_addresses(self, addresses):
 432         k = 0
 433         for a in addresses[::-1]:
 434             if self.history.get(a):break
 435             k = k + 1
 436         return k
 437
 438     def min_acceptable_gap(self):
 439         # fixme: this assumes wallet is synchronized
 440         n = 0
 441         nmax = 0
 442
 443         for account in self.accounts.values():
 444             addresses = account.get_addresses(0)
 445             k = self.num_unused_trailing_addresses(addresses)
 446             for a in addresses[0:-k]:
 447                 if self.history.get(a):
 448                     n = 0
 449                 else:
 450                     n += 1
 451                     if n > nmax: nmax = n
 452         return nmax + 1
 453
 454
 455     def address_is_old(self, address):
 456         age = -1
 457         h = self.history.get(address, [])
 458         if h == ['*']:
 459             return True
 460         for tx_hash, tx_height in h:
 461             if tx_height == 0:
 462                 tx_age = 0
 463             else:
 464                 tx_age = self.verifier.height - tx_height + 1
 465             if tx_age > age:
 466                 age = tx_age
 467         return age > 2
 468
 469
 470     def synchronize_sequence(self, account, for_change):
 471         limit = self.gap_limit_for_change if for_change else self.gap_limit
 472         new_addresses = []
 473         while True:
 474             addresses = account.get_addresses(for_change)
 475             if len(addresses) < limit:
 476                 address = account.create_new_address(for_change)
 477                 self.history[address] = []
 478                 new_addresses.append( address )
 479                 continue
 480
 481             if map( lambda a: self.address_is_old(a), addresses[-limit:] ) == limit*[False]:
 482                 break
 483             else:
 484                 address = account.create_new_address(for_change)
 485                 self.history[address] = []
 486                 new_addresses.append( address )
 487
 488         return new_addresses
 489
 490
 491     def synchronize_account(self, account):
 492         new = []
 493         new += self.synchronize_sequence(account, 0)
 494         new += self.synchronize_sequence(account, 1)
 495         return new
 496
 497     def synchronize(self):
 498         new = []
 499         for account in self.accounts.values():
 500             new += self.synchronize_account(account)
 501         if new:
 502             self.save_accounts()
 503             self.config.set_key('addr_history', self.history, True)
 504         return new
 505
 506
 507     def is_found(self):
 508         return self.history.values() != [[]] * len(self.history)
 509
 510
 511     def add_contact(self, address, label=None):
 512         self.addressbook.append(address)
 513         self.config.set_key('contacts', self.addressbook, True)
 514         if label:
 515             self.labels[address] = label
 516             self.config.set_key('labels', self.labels)
 517
 518     def delete_contact(self, addr):
 519         if addr in self.addressbook:
 520             self.addressbook.remove(addr)
 521             self.config.set_key('addressbook', self.addressbook, True)
 522
 523
 524     def fill_addressbook(self):
 525         for tx_hash, tx in self.transactions.items():
 526             is_relevant, is_send, _, _ = self.get_tx_value(tx)
 527             if is_send:
 528                 for addr, v in tx.outputs:
 529                     if not self.is_mine(addr) and addr not in self.addressbook:
 530                         self.addressbook.append(addr)
 531         # redo labels
 532         # self.update_tx_labels()
 533
 534     def get_num_tx(self, address):
 535         n = 0
 536         for tx in self.transactions.values():
 537             if address in map(lambda x:x[0], tx.outputs): n += 1
 538         return n
 539
 540
 541     def get_address_flags(self, addr):
 542         flags = "C" if self.is_change(addr) else "I" if addr in self.imported_keys.keys() else "-"
 543         flags += "F" if addr in self.frozen_addresses else "P" if addr in self.prioritized_addresses else "-"
 544         return flags
 545
 546
 547     def get_tx_value(self, tx, account=None):
 548         domain = self.get_account_addresses(account)
 549         return tx.get_value(domain, self.prevout_values)
 550
 551
 552     def update_tx_outputs(self, tx_hash):
 553         tx = self.transactions.get(tx_hash)
 554
 555         for i, (addr, value) in enumerate(tx.outputs):
 556             key = tx_hash+ ':%d'%i
 557             self.prevout_values[key] = value
 558
 559         for item in tx.inputs:
 560             if self.is_mine(item.get('address')):
 561                 key = item['prevout_hash'] + ':%d'%item['prevout_n']
 562                 self.spent_outputs.append(key)
 563
 564
 565     def get_addr_balance(self, address):
 566         assert self.is_mine(address)
 567         h = self.history.get(address,[])
 568         if h == ['*']: return 0,0
 569         c = u = 0
 570         received_coins = []   # list of coins received at address
 571
 572         for tx_hash, tx_height in h:
 573             tx = self.transactions.get(tx_hash)
 574             if not tx: continue
 575
 576             for i, (addr, value) in enumerate(tx.outputs):
 577                 if addr == address:
 578                     key = tx_hash + ':%d'%i
 579                     received_coins.append(key)
 580
 581         for tx_hash, tx_height in h:
 582             tx = self.transactions.get(tx_hash)
 583             if not tx: continue
 584             v = 0
 585
 586             for item in tx.inputs:
 587                 addr = item.get('address')
 588                 if addr == address:
 589                     key = item['prevout_hash']  + ':%d'%item['prevout_n']
 590                     value = self.prevout_values.get( key )
 591                     if key in received_coins:
 592                         v -= value
 593
 594             for i, (addr, value) in enumerate(tx.outputs):
 595                 key = tx_hash + ':%d'%i
 596                 if addr == address:
 597                     v += value
 598
 599             if tx_height:
 600                 c += v
 601             else:
 602                 u += v
 603         return c, u
 604
 605
 606     def get_accounts(self):
 607         accounts = {}
 608         for k, account in self.accounts.items():
 609             accounts[k] = account.name
 610         if self.imported_keys:
 611             accounts[-1] = 'Imported keys'
 612         return accounts
 613
 614     def get_account_addresses(self, a, include_change=True):
 615         if a is None:
 616             o = self.addresses(True)
 617         elif a == -1:
 618             o = self.imported_keys.keys()
 619         else:
 620             ac = self.accounts[a]
 621             o = ac.get_addresses(0)
 622             if include_change: o += ac.get_addresses(1)
 623         return o
 624
 625     def get_imported_balance(self):
 626         cc = uu = 0
 627         for addr in self.imported_keys.keys():
 628             c, u = self.get_addr_balance(addr)
 629             cc += c
 630             uu += u
 631         return cc, uu
 632
 633     def get_account_balance(self, account):
 634         if account is None:
 635             return self.get_balance()
 636         elif account == -1:
 637             return self.get_imported_balance()
 638
 639         conf = unconf = 0
 640         for addr in self.get_account_addresses(account):
 641             c, u = self.get_addr_balance(addr)
 642             conf += c
 643             unconf += u
 644         return conf, unconf
 645
 646     def get_frozen_balance(self):
 647         conf = unconf = 0
 648         for addr in self.frozen_addresses:
 649             c, u = self.get_addr_balance(addr)
 650             conf += c
 651             unconf += u
 652         return conf, unconf
 653
 654
 655     def get_balance(self):
 656         cc = uu = 0
 657         for a in self.accounts.keys():
 658             c, u = self.get_account_balance(a)
 659             cc += c
 660             uu += u
 661         c, u = self.get_imported_balance()
 662         cc += c
 663         uu += u
 664         return cc, uu
 665
 666
 667     def get_unspent_coins(self, domain=None):
 668         coins = []
 669         if domain is None: domain = self.addresses(True)
 670         for addr in domain:
 671             h = self.history.get(addr, [])
 672             if h == ['*']: continue
 673             for tx_hash, tx_height in h:
 674                 tx = self.transactions.get(tx_hash)
 675                 if tx is None: raise BaseException("Wallet not synchronized")
 676                 for output in tx.d.get('outputs'):
 677                     if output.get('address') != addr: continue
 678                     key = tx_hash + ":%d" % output.get('index')
 679                     if key in self.spent_outputs: continue
 680                     output['tx_hash'] = tx_hash
 681                     coins.append(output)
 682         return coins
 683
 684
 685
 686     def choose_tx_inputs( self, amount, fixed_fee, account = None ):
 687         """ todo: minimize tx size """
 688         total = 0
 689         fee = self.fee if fixed_fee is None else fixed_fee
 690         domain = self.get_account_addresses(account)
 691         coins = []
 692         prioritized_coins = []
 693         for i in self.frozen_addresses:
 694             if i in domain: domain.remove(i)
 695
 696         for i in self.prioritized_addresses:
 697             if i in domain: domain.remove(i)
 698
 699         coins = self.get_unspent_coins(domain)
 700         prioritized_coins = self.get_unspent_coins(self.prioritized_addresses)
 701
 702         inputs = []
 703         coins = prioritized_coins + coins
 704
 705         for item in coins:
 706             addr = item.get('address')
 707             v = item.get('value')
 708             total += v
 709             inputs.append( item )
 710             fee = self.estimated_fee(inputs) if fixed_fee is None else fixed_fee
 711             if total >= amount + fee: break
 712         else:
 713             inputs = []
 714
 715         return inputs, total, fee
 716
 717
 718     def estimated_fee(self, inputs):
 719         estimated_size =  len(inputs) * 180 + 80     # this assumes non-compressed keys
 720         fee = self.fee * int(round(estimated_size/1024.))
 721         if fee == 0: fee = self.fee
 722         return fee
 723
 724
 725     def add_tx_change( self, inputs, outputs, amount, fee, total, change_addr=None, account=0 ):
 726         "add change to a transaction"
 727         change_amount = total - ( amount + fee )
 728         if change_amount != 0:
 729             if not change_addr:
 730                 if account is None:
 731                     # send change to one of the accounts involved in the tx
 732                     address = inputs[0].get('address')
 733                     account, _ = self.get_address_index(address)
 734
 735                 if not self.use_change or account == -1:
 736                     change_addr = inputs[-1]['address']
 737                 else:
 738                     change_addr = self.accounts[account].get_addresses(1)[-self.gap_limit_for_change]
 739
 740             # Insert the change output at a random position in the outputs
 741             posn = random.randint(0, len(outputs))
 742             outputs[posn:posn] = [( change_addr,  change_amount)]
 743         return outputs
 744
 745
 746     def get_history(self, address):
 747         with self.lock:
 748             return self.history.get(address)
 749
 750
 751     def get_status(self, h):
 752         if not h: return None
 753         if h == ['*']: return '*'
 754         status = ''
 755         for tx_hash, height in h:
 756             status += tx_hash + ':%d:' % height
 757         return hashlib.sha256( status ).digest().encode('hex')
 758
 759
 760     def receive_tx_callback(self, tx_hash, tx, tx_height):
 761         if not self.check_new_tx(tx_hash, tx):
 762             # may happen due to pruning
 763             print_error("received transaction that is no longer referenced in history", tx_hash)
 764             return
 765
 766         with self.transaction_lock:
 767             self.transactions[tx_hash] = tx
 768
 769             self.interface.pending_transactions_for_notifications.append(tx)
 770
 771             self.save_transactions()
 772             if self.verifier and tx_height>0:
 773                 self.verifier.add(tx_hash, tx_height)
 774             self.update_tx_outputs(tx_hash)
 775
 776
 777     def save_transactions(self):
 778         tx = {}
 779         for k,v in self.transactions.items():
 780             tx[k] = str(v)
 781         self.config.set_key('transactions', tx, True)
 782
 783     def receive_history_callback(self, addr, hist):
 784
 785         if not self.check_new_history(addr, hist):
 786             raise BaseException("error: received history for %s is not consistent with known transactions"%addr)
 787
 788         with self.lock:
 789             self.history[addr] = hist
 790             self.config.set_key('addr_history', self.history, True)
 791
 792         if hist != ['*']:
 793             for tx_hash, tx_height in hist:
 794                 if tx_height>0:
 795                     # add it in case it was previously unconfirmed
 796                     if self.verifier: self.verifier.add(tx_hash, tx_height)
 797
 798
 799     def get_tx_history(self, account=None):
 800         with self.transaction_lock:
 801             history = self.transactions.items()
 802             history.sort(key = lambda x: self.verifier.get_txpos(x[0]))
 803             result = []
 804
 805             balance = 0
 806             for tx_hash, tx in history:
 807                 is_relevant, is_mine, v, fee = self.get_tx_value(tx, account)
 808                 if v is not None: balance += v
 809
 810             c, u = self.get_account_balance(account)
 811
 812             if balance != c+u:
 813                 result.append( ('', 1000, 0, c+u-balance, None, c+u-balance, None ) )
 814
 815             balance = c + u - balance
 816             for tx_hash, tx in history:
 817                 is_relevant, is_mine, value, fee = self.get_tx_value(tx, account)
 818                 if not is_relevant:
 819                     continue
 820                 if value is not None:
 821                     balance += value
 822
 823                 conf, timestamp = self.verifier.get_confirmations(tx_hash) if self.verifier else (None, None)
 824                 result.append( (tx_hash, conf, is_mine, value, fee, balance, timestamp) )
 825
 826         return result
 827
 828
 829     def get_label(self, tx_hash):
 830         label = self.labels.get(tx_hash)
 831         is_default = (label == '') or (label is None)
 832         if is_default: label = self.get_default_label(tx_hash)
 833         return label, is_default
 834
 835
 836     def get_default_label(self, tx_hash):
 837         tx = self.transactions.get(tx_hash)
 838         default_label = ''
 839         if tx:
 840             is_relevant, is_mine, _, _ = self.get_tx_value(tx)
 841             if is_mine:
 842                 for o in tx.outputs:
 843                     o_addr, _ = o
 844                     if not self.is_mine(o_addr):
 845                         try:
 846                             default_label = self.labels[o_addr]
 847                         except KeyError:
 848                             default_label = o_addr
 849                         break
 850                 else:
 851                     default_label = '(internal)'
 852             else:
 853                 for o in tx.outputs:
 854                     o_addr, _ = o
 855                     if self.is_mine(o_addr) and not self.is_change(o_addr):
 856                         break
 857                 else:
 858                     for o in tx.outputs:
 859                         o_addr, _ = o
 860                         if self.is_mine(o_addr):
 861                             break
 862                     else:
 863                         o_addr = None
 864
 865                 if o_addr:
 866                     dest_label = self.labels.get(o_addr)
 867                     try:
 868                         default_label = self.labels[o_addr]
 869                     except KeyError:
 870                         default_label = o_addr
 871
 872         return default_label
 873
 874
 875     def mktx(self, outputs, password, fee=None, change_addr=None, account=None ):
 876         """
 877         create a transaction
 878         account parameter:
 879            None means use all accounts
 880            -1 means imported keys
 881            0, 1, etc are seed accounts
 882         """
 883
 884         for address, x in outputs:
 885             assert is_valid(address)
 886
 887         amount = sum( map(lambda x:x[1], outputs) )
 888
 889         inputs, total, fee = self.choose_tx_inputs( amount, fee, account )
 890         if not inputs:
 891             raise ValueError("Not enough funds")
 892
 893         outputs = self.add_tx_change(inputs, outputs, amount, fee, total, change_addr, account)
 894
 895         tx = Transaction.from_io(inputs, outputs)
 896
 897         pk_addresses = []
 898         for i in range(len(tx.inputs)):
 899             txin = tx.inputs[i]
 900             address = txin['address']
 901             if address in self.imported_keys.keys():
 902                 pk_addresses.append(address)
 903                 continue
 904             account, sequence = self.get_address_index(address)
 905
 906             txin['KeyID'] = (account, 'BIP32', sequence) # used by the server to find the key
 907
 908             _, redeemScript = self.accounts[account].get_input_info(sequence)
 909
 910             if redeemScript: txin['redeemScript'] = redeemScript
 911             pk_addresses.append(address)
 912
 913         print "pk_addresses", pk_addresses
 914
 915         # get all private keys at once.
 916         if self.seed:
 917             private_keys = self.get_private_keys(pk_addresses, password)
 918             print "private keys", private_keys
 919             tx.sign(private_keys)
 920
 921         for address, x in outputs:
 922             if address not in self.addressbook and not self.is_mine(address):
 923                 self.addressbook.append(address)
 924
 925         return tx
 926
 927
 928
 929     def sendtx(self, tx):
 930         # synchronous
 931         h = self.send_tx(tx)
 932         self.tx_event.wait()
 933         return self.receive_tx(h)
 934
 935     def send_tx(self, tx):
 936         # asynchronous
 937         self.tx_event.clear()
 938         self.interface.send([('blockchain.transaction.broadcast', [str(tx)])], 'synchronizer')
 939         return tx.hash()
 940
 941     def receive_tx(self,tx_hash):
 942         out = self.tx_result
 943         if out != tx_hash:
 944             return False, "error: " + out
 945         return True, out
 946
 947
 948
 949     def update_password(self, seed, old_password, new_password):
 950         if new_password == '': new_password = None
 951         # this will throw an exception if unicode cannot be converted
 952         self.seed = pw_encode( seed, new_password)
 953         self.config.set_key('seed', self.seed, True)
 954         self.use_encryption = (new_password != None)
 955         self.config.set_key('use_encryption', self.use_encryption,True)
 956         for k in self.imported_keys.keys():
 957             a = self.imported_keys[k]
 958             b = pw_decode(a, old_password)
 959             c = pw_encode(b, new_password)
 960             self.imported_keys[k] = c
 961         self.config.set_key('imported_keys', self.imported_keys, True)
 962
 963         for k, v in self.master_private_keys.items():
 964             b = pw_decode(v, old_password)
 965             c = pw_encode(b, new_password)
 966             self.master_private_keys[k] = c
 967         self.config.set_key('master_private_keys', self.master_private_keys, True)
 968
 969
 970     def freeze(self,addr):
 971         if self.is_mine(addr) and addr not in self.frozen_addresses:
 972             self.unprioritize(addr)
 973             self.frozen_addresses.append(addr)
 974             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 975             return True
 976         else:
 977             return False
 978
 979     def unfreeze(self,addr):
 980         if self.is_mine(addr) and addr in self.frozen_addresses:
 981             self.frozen_addresses.remove(addr)
 982             self.config.set_key('frozen_addresses', self.frozen_addresses, True)
 983             return True
 984         else:
 985             return False
 986
 987     def prioritize(self,addr):
 988         if self.is_mine(addr) and addr not in self.prioritized_addresses:
 989             self.unfreeze(addr)
 990             self.prioritized_addresses.append(addr)
 991             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
 992             return True
 993         else:
 994             return False
 995
 996     def unprioritize(self,addr):
 997         if self.is_mine(addr) and addr in self.prioritized_addresses:
 998             self.prioritized_addresses.remove(addr)
 999             self.config.set_key('prioritized_addresses', self.prioritized_addresses, True)
1000             return True
1001         else:
1002             return False
1003
1004     def set_fee(self, fee):
1005         if self.fee != fee:
1006             self.fee = fee
1007             self.config.set_key('fee_per_kb', self.fee, True)
1008
1009
1010     def save(self):
1011         print_error("Warning: wallet.save() is deprecated")
1012         tx = {}
1013         for k,v in self.transactions.items():
1014             tx[k] = str(v)
1015
1016         s = {
1017             'use_change': self.use_change,
1018             'fee_per_kb': self.fee,
1019             'addr_history': self.history,
1020             'labels': self.labels,
1021             'contacts': self.addressbook,
1022             'num_zeros': self.num_zeros,
1023             'frozen_addresses': self.frozen_addresses,
1024             'prioritized_addresses': self.prioritized_addresses,
1025             'gap_limit': self.gap_limit,
1026             'transactions': tx,
1027         }
1028         for k, v in s.items():
1029             self.config.set_key(k,v)
1030         self.config.save()
1031
1032     def set_verifier(self, verifier):
1033         self.verifier = verifier
1034
1035         # review transactions that are in the history
1036         for addr, hist in self.history.items():
1037             if hist == ['*']: continue
1038             for tx_hash, tx_height in hist:
1039                 if tx_height>0:
1040                     # add it in case it was previously unconfirmed
1041                     self.verifier.add(tx_hash, tx_height)
1042
1043
1044         # if we are on a pruning server, remove unverified transactions
1045         vr = self.verifier.transactions.keys() + self.verifier.verified_tx.keys()
1046         for tx_hash in self.transactions.keys():
1047             if tx_hash not in vr:
1048                 self.transactions.pop(tx_hash)
1049
1050
1051
1052     def check_new_history(self, addr, hist):
1053
1054         # check that all tx in hist are relevant
1055         if hist != ['*']:
1056             for tx_hash, height in hist:
1057                 tx = self.transactions.get(tx_hash)
1058                 if not tx: continue
1059                 if not tx.has_address(addr):
1060                     return False
1061
1062         # check that we are not "orphaning" a transaction
1063         old_hist = self.history.get(addr,[])
1064         if old_hist == ['*']: return True
1065
1066         for tx_hash, height in old_hist:
1067             if tx_hash in map(lambda x:x[0], hist): continue
1068             found = False
1069             for _addr, _hist in self.history.items():
1070                 if _addr == addr: continue
1071                 if _hist == ['*']: continue
1072                 _tx_hist = map(lambda x:x[0], _hist)
1073                 if tx_hash in _tx_hist:
1074                     found = True
1075                     break
1076
1077             if not found:
1078                 tx = self.transactions.get(tx_hash)
1079                 # tx might not be there
1080                 if not tx: continue
1081
1082                 # already verified?
1083                 if self.verifier.get_height(tx_hash):
1084                     continue
1085                 # unconfirmed tx
1086                 print_error("new history is orphaning transaction:", tx_hash)
1087                 # check that all outputs are not mine, request histories
1088                 ext_requests = []
1089                 for _addr, _v in tx.outputs:
1090                     # assert not self.is_mine(_addr)
1091                     ext_requests.append( ('blockchain.address.get_history', [_addr]) )
1092
1093                 ext_h = self.interface.synchronous_get(ext_requests)
1094                 print_error("sync:", ext_requests, ext_h)
1095                 height = None
1096                 for h in ext_h:
1097                     if h == ['*']: continue
1098                     for item in h:
1099                         if item.get('tx_hash') == tx_hash:
1100                             height = item.get('height')
1101                 if height:
1102                     print_error("found height for", tx_hash, height)
1103                     self.verifier.add(tx_hash, height)
1104                 else:
1105                     print_error("removing orphaned tx from history", tx_hash)
1106                     self.transactions.pop(tx_hash)
1107
1108         return True
1109
1110
1111
1112     def check_new_tx(self, tx_hash, tx):
1113         # 1 check that tx is referenced in addr_history.
1114         addresses = []
1115         for addr, hist in self.history.items():
1116             if hist == ['*']:continue
1117             for txh, height in hist:
1118                 if txh == tx_hash:
1119                     addresses.append(addr)
1120
1121         if not addresses:
1122             return False
1123
1124         # 2 check that referencing addresses are in the tx
1125         for addr in addresses:
1126             if not tx.has_address(addr):
1127                 return False
1128
1129         return True
1130
1131
1132
1133
1134 class WalletSynchronizer(threading.Thread):
1135
1136
1137     def __init__(self, wallet, config):
1138         threading.Thread.__init__(self)
1139         self.daemon = True
1140         self.wallet = wallet
1141         wallet.synchronizer = self
1142         self.interface = self.wallet.interface
1143         self.interface.register_channel('synchronizer')
1144         self.wallet.interface.register_callback('connected', lambda: self.wallet.set_up_to_date(False))
1145         self.was_updated = True
1146         self.running = False
1147         self.lock = threading.Lock()
1148
1149     def stop(self):
1150         with self.lock: self.running = False
1151         self.interface.poke('synchronizer')
1152
1153     def is_running(self):
1154         with self.lock: return self.running
1155
1156
1157     def subscribe_to_addresses(self, addresses):
1158         messages = []
1159         for addr in addresses:
1160             messages.append(('blockchain.address.subscribe', [addr]))
1161         self.interface.send( messages, 'synchronizer')
1162
1163
1164     def run(self):
1165         with self.lock: self.running = True
1166
1167         requested_tx = []
1168         missing_tx = []
1169         requested_histories = {}
1170
1171         # request any missing transactions
1172         for history in self.wallet.history.values():
1173             if history == ['*']: continue
1174             for tx_hash, tx_height in history:
1175                 if self.wallet.transactions.get(tx_hash) is None and (tx_hash, tx_height) not in missing_tx:
1176                     missing_tx.append( (tx_hash, tx_height) )
1177         print_error("missing tx", missing_tx)
1178
1179         # wait until we are connected, in case the user is not connected
1180         while not self.interface.is_connected:
1181             time.sleep(1)
1182
1183         # subscriptions
1184         self.subscribe_to_addresses(self.wallet.addresses(True))
1185
1186         while self.is_running():
1187             # 1. create new addresses
1188             new_addresses = self.wallet.synchronize()
1189
1190             # request missing addresses
1191             if new_addresses:
1192                 self.subscribe_to_addresses(new_addresses)
1193
1194             # request missing transactions
1195             for tx_hash, tx_height in missing_tx:
1196                 if (tx_hash, tx_height) not in requested_tx:
1197                     self.interface.send([ ('blockchain.transaction.get',[tx_hash, tx_height]) ], 'synchronizer')
1198                     requested_tx.append( (tx_hash, tx_height) )
1199             missing_tx = []
1200
1201             # detect if situation has changed
1202             if not self.interface.is_up_to_date('synchronizer'):
1203                 if self.wallet.is_up_to_date():
1204                     self.wallet.set_up_to_date(False)
1205                     self.was_updated = True
1206             else:
1207                 if not self.wallet.is_up_to_date():
1208                     self.wallet.set_up_to_date(True)
1209                     self.was_updated = True
1210
1211             if self.was_updated:
1212                 self.interface.trigger_callback('updated')
1213                 self.was_updated = False
1214
1215             # 2. get a response
1216             r = self.interface.get_response('synchronizer')
1217
1218             # poke sends None. (needed during stop)
1219             if not r: continue
1220
1221             # 3. handle response
1222             method = r['method']
1223             params = r['params']
1224             result = r.get('result')
1225             error = r.get('error')
1226             if error:
1227                 print "error", r
1228                 continue
1229
1230             if method == 'blockchain.address.subscribe':
1231                 addr = params[0]
1232                 if self.wallet.get_status(self.wallet.get_history(addr)) != result:
1233                     if requested_histories.get(addr) is None:
1234                         self.interface.send([('blockchain.address.get_history', [addr])], 'synchronizer')
1235                         requested_histories[addr] = result
1236
1237             elif method == 'blockchain.address.get_history':
1238                 addr = params[0]
1239                 print_error("receiving history", addr, result)
1240                 if result == ['*']:
1241                     assert requested_histories.pop(addr) == '*'
1242                     self.wallet.receive_history_callback(addr, result)
1243                 else:
1244                     hist = []
1245                     # check that txids are unique
1246                     txids = []
1247                     for item in result:
1248                         tx_hash = item['tx_hash']
1249                         if tx_hash not in txids:
1250                             txids.append(tx_hash)
1251                             hist.append( (tx_hash, item['height']) )
1252
1253                     if len(hist) != len(result):
1254                         raise BaseException("error: server sent history with non-unique txid", result)
1255
1256                     # check that the status corresponds to what was announced
1257                     rs = requested_histories.pop(addr)
1258                     if self.wallet.get_status(hist) != rs:
1259                         raise BaseException("error: status mismatch: %s"%addr)
1260
1261                     # store received history
1262                     self.wallet.receive_history_callback(addr, hist)
1263
1264                     # request transactions that we don't have
1265                     for tx_hash, tx_height in hist:
1266                         if self.wallet.transactions.get(tx_hash) is None:
1267                             if (tx_hash, tx_height) not in requested_tx and (tx_hash, tx_height) not in missing_tx:
1268                                 missing_tx.append( (tx_hash, tx_height) )
1269
1270             elif method == 'blockchain.transaction.get':
1271                 tx_hash = params[0]
1272                 tx_height = params[1]
1273                 assert tx_hash == hash_encode(Hash(result.decode('hex')))
1274                 tx = Transaction(result)
1275                 self.wallet.receive_tx_callback(tx_hash, tx, tx_height)
1276                 self.was_updated = True
1277                 requested_tx.remove( (tx_hash, tx_height) )
1278                 print_error("received tx:", tx_hash, len(tx.raw))
1279
1280             elif method == 'blockchain.transaction.broadcast':
1281                 self.wallet.tx_result = result
1282                 self.wallet.tx_event.set()
1283
1284             else:
1285                 print_error("Error: Unknown message:" + method + ", " + repr(params) + ", " + repr(result) )
1286
1287             if self.was_updated and not requested_tx:
1288                 self.interface.trigger_callback('updated')
1289                 self.interface.trigger_callback("new_transaction") # Updated gets called too many times from other places as well; if we use that signal we get the notification three times
1290
1291
1292                 self.was_updated = False