2 from json import dumps, loads
4 from Queue import Queue
12 from Abe import DataStore, readconf, BCDataStream, deserialize
13 from Abe.util import hash_to_address, decode_check_address
15 from processor import Processor, print_log
19 class AbeStore(Datastore.Datastore):
21 def __init__(self, config):
22 conf = DataStore.CONFIG_DEFAULTS
23 args, argv = readconf.parse_argv([], conf)
24 args.dbtype = config.get('database', 'type')
25 if args.dbtype == 'sqlite3':
26 args.connect_args = {'database': config.get('database', 'database')}
27 elif args.dbtype == 'MySQLdb':
28 args.connect_args = {'db': config.get('database', 'database'), 'user': config.get('database', 'username'), 'passwd': config.get('database', 'password')}
29 elif args.dbtype == 'psycopg2':
30 args.connect_args = {'database': config.get('database', 'database')}
32 coin = config.get('server', 'coin')
34 if coin == 'litecoin':
35 print_log('Litecoin settings:')
36 datadir = config.get('server', 'datadir')
37 print_log(' datadir = ' + datadir)
38 args.datadir = [{"dirname": datadir, "chain": "Litecoin", "code3": "LTC", "address_version": "\u0030"}]
39 print_log(' addrtype = 48')
42 Datastore.Datastore.__init__(self, args)
44 # Use 1 (Bitcoin) if chain_id is not sent
45 self.chain_id = self.datadirs[0]["chain_id"] or 1
46 print_log('Coin chain_id = %d' % self.chain_id)
48 self.sql_limit = int(config.get('database', 'limit'))
51 self.bitcoind_url = 'http://%s:%s@%s:%s/' % (config.get('bitcoind', 'user'), config.get('bitcoind', 'password'), config.get('bitcoind', 'host'), config.get('bitcoind', 'port'))
55 self.address_queue = Queue()
57 self.lock = threading.Lock() # for the database
58 self.cache_lock = threading.Lock() # for the cache
60 self.known_mempool_hashes = []
62 def import_tx(self, tx, is_coinbase):
63 tx_id = super(AbeStore, self).import_tx(tx, is_coinbase)
64 self.last_tx_id = tx_id
67 def import_block(self, b, chain_ids=frozenset()):
68 #print_log("import block")
69 block_id = super(AbeStore, self).import_block(b, chain_ids)
70 for pos in xrange(len(b['transactions'])):
71 tx = b['transactions'][pos]
73 tx['hash'] = Hash(tx['tx'])
74 tx_id = self.tx_find_id_and_value(tx)
76 self.update_tx_cache(tx_id)
78 print_log("error: import_block: no tx_id")
81 def update_tx_cache(self, txid):
82 inrows = self.get_tx_inputs(txid, False)
84 _hash = self.binout(row[6])
86 #print_log("WARNING: missing tx_in for tx", txid)
89 address = hash_to_address(chr(self.addrtype), _hash)
91 if address in self.tx_cache:
92 print_log("cache: invalidating", address)
93 self.tx_cache.pop(address)
95 self.address_queue.put(address)
97 outrows = self.get_tx_outputs(txid, False)
99 _hash = self.binout(row[6])
101 #print_log("WARNING: missing tx_out for tx", txid)
104 address = hash_to_address(chr(self.addrtype), _hash)
105 with self.cache_lock:
106 if address in self.tx_cache:
107 print_log("cache: invalidating", address)
108 self.tx_cache.pop(address)
110 self.address_queue.put(address)
112 def safe_sql(self, sql, params=(), lock=True):
117 ret = self.selectall(sql, params)
120 traceback.print_exc(file=sys.stdout)
126 raise Exception('sql error')
130 def get_tx_outputs(self, tx_id, lock=True):
131 return self.safe_sql("""SELECT
133 txout.txout_scriptPubKey,
140 LEFT JOIN txin ON (txin.txout_id = txout.txout_id)
141 LEFT JOIN pubkey ON (pubkey.pubkey_id = txout.pubkey_id)
142 LEFT JOIN tx nexttx ON (txin.tx_id = nexttx.tx_id)
143 WHERE txout.tx_id = %d
144 ORDER BY txout.txout_pos
145 """ % (tx_id), (), lock)
147 def get_tx_inputs(self, tx_id, lock=True):
148 return self.safe_sql(""" SELECT
152 COALESCE(prevtx.tx_hash, u.txout_tx_hash),
154 COALESCE(txout.txout_pos, u.txout_pos),
157 LEFT JOIN txout ON (txout.txout_id = txin.txout_id)
158 LEFT JOIN pubkey ON (pubkey.pubkey_id = txout.pubkey_id)
159 LEFT JOIN tx prevtx ON (txout.tx_id = prevtx.tx_id)
160 LEFT JOIN unlinked_txin u ON (u.txin_id = txin.txin_id)
161 WHERE txin.tx_id = %d
162 ORDER BY txin.txin_pos
163 """ % (tx_id,), (), lock)
165 def get_address_out_rows(self, dbhash):
166 out = self.safe_sql(""" SELECT
176 FROM chain_candidate cc
177 JOIN block b ON (b.block_id = cc.block_id)
178 JOIN block_tx ON (block_tx.block_id = b.block_id)
179 JOIN tx ON (tx.tx_id = block_tx.tx_id)
180 JOIN txin ON (txin.tx_id = tx.tx_id)
181 JOIN txout prevout ON (txin.txout_id = prevout.txout_id)
182 JOIN pubkey ON (pubkey.pubkey_id = prevout.pubkey_id)
183 WHERE pubkey.pubkey_hash = ?
185 AND cc.in_longest = 1
186 LIMIT ? """, (dbhash, self.chain_id, self.sql_limit))
188 if len(out) == self.sql_limit:
189 raise Exception('limit reached')
192 def get_address_out_rows_memorypool(self, dbhash):
193 out = self.safe_sql(""" SELECT
200 JOIN txin ON (txin.tx_id = tx.tx_id)
201 JOIN txout prevout ON (txin.txout_id = prevout.txout_id)
202 JOIN pubkey ON (pubkey.pubkey_id = prevout.pubkey_id)
203 WHERE pubkey.pubkey_hash = ?
204 LIMIT ? """, (dbhash, self.sql_limit))
206 if len(out) == self.sql_limit:
207 raise Exception('limit reached')
210 def get_address_in_rows(self, dbhash):
211 out = self.safe_sql(""" SELECT
221 FROM chain_candidate cc
222 JOIN block b ON (b.block_id = cc.block_id)
223 JOIN block_tx ON (block_tx.block_id = b.block_id)
224 JOIN tx ON (tx.tx_id = block_tx.tx_id)
225 JOIN txout ON (txout.tx_id = tx.tx_id)
226 JOIN pubkey ON (pubkey.pubkey_id = txout.pubkey_id)
227 WHERE pubkey.pubkey_hash = ?
229 AND cc.in_longest = 1
230 LIMIT ? """, (dbhash, self.chain_id, self.sql_limit))
232 if len(out) == self.sql_limit:
233 raise Exception('limit reached')
236 def get_address_in_rows_memorypool(self, dbhash):
237 out = self.safe_sql(""" SELECT
244 JOIN txout ON (txout.tx_id = tx.tx_id)
245 JOIN pubkey ON (pubkey.pubkey_id = txout.pubkey_id)
246 WHERE pubkey.pubkey_hash = ?
247 LIMIT ? """, (dbhash, self.sql_limit))
249 if len(out) == self.sql_limit:
250 raise Exception('limit reached')
253 def get_history(self, addr, cache_only=False):
254 # todo: make this more efficient. it iterates over txpoints multiple times
255 with self.cache_lock:
256 cached_version = self.tx_cache.get(addr)
257 if cached_version is not None:
258 return cached_version
263 version, binaddr = decode_check_address(addr)
267 dbhash = self.binin(binaddr)
269 rows += self.get_address_out_rows(dbhash)
270 rows += self.get_address_in_rows(dbhash)
277 nTime, chain_id, height, is_in, blk_hash, tx_hash, tx_id, pos, value = row
279 print_log("cannot unpack row", row)
281 tx_hash = self.hashout_hex(tx_hash)
284 "timestamp": int(nTime),
285 "height": int(height),
286 "is_input": int(is_in),
287 "block_hash": self.hashout_hex(blk_hash),
293 known_tx.append(self.hashout_hex(tx_hash))
295 # todo: sort them really...
296 txpoints = sorted(txpoints, key=operator.itemgetter("timestamp"))
300 rows += self.get_address_in_rows_memorypool(dbhash)
301 rows += self.get_address_out_rows_memorypool(dbhash)
302 address_has_mempool = False
305 is_in, tx_hash, tx_id, pos, value = row
306 tx_hash = self.hashout_hex(tx_hash)
307 if tx_hash in known_tx:
310 # discard transactions that are too old
311 if self.last_tx_id - tx_id > 50000:
312 print_log("discarding tx id", tx_id)
315 # this means that pending transactions were added to the db, even if they are not returned by getmemorypool
316 address_has_mempool = True
318 #print_log("mempool", tx_hash)
322 "is_input": int(is_in),
323 "block_hash": 'mempool',
330 for txpoint in txpoints:
331 tx_id = txpoint['tx_id']
334 inrows = self.get_tx_inputs(tx_id)
336 _hash = self.binout(row[6])
338 #print_log("WARNING: missing tx_in for tx", tx_id, addr)
340 address = hash_to_address(chr(self.addrtype), _hash)
341 txinputs.append(address)
342 txpoint['inputs'] = txinputs
344 outrows = self.get_tx_outputs(tx_id)
346 _hash = self.binout(row[6])
348 #print_log("WARNING: missing tx_out for tx", tx_id, addr)
350 address = hash_to_address(chr(self.addrtype), _hash)
351 txoutputs.append(address)
352 txpoint['outputs'] = txoutputs
354 # for all unspent inputs, I want their scriptpubkey. (actually I could deduce it from the address)
355 if not txpoint['is_input']:
356 # detect if already redeemed...
362 #row = self.get_tx_output(tx_id,dbhash)
363 # pos, script, value, o_hash, o_id, o_pos, binaddr = row
364 # if not redeemed, we add the script
367 txpoint['raw_output_script'] = row[1]
371 txpoints = map(lambda x: {'tx_hash': x['tx_hash'], 'height': x['height']}, txpoints)
373 for item in txpoints:
378 ## do not cache mempool results because statuses are ambiguous
379 #if not address_has_mempool:
380 with self.cache_lock:
381 self.tx_cache[addr] = out
385 def get_status(self, addr, cache_only=False):
387 tx_points = self.get_history(addr, cache_only)
388 if cache_only and tx_points == -1:
395 status += tx.get('tx_hash') + ':%d:' % tx.get('height')
396 return hashlib.sha256(status).digest().encode('hex')
398 def get_block_header(self, block_height):
399 out = self.safe_sql("""
403 block_hashMerkleRoot,
411 WHERE block_height = %d AND in_longest = 1""" % block_height)
414 raise Exception("block not found")
416 (block_hash, block_version, hashMerkleRoot, nTime, nBits, nNonce, height, prev_block_hash, block_id) \
417 = (self.hashout_hex(row[0]), int(row[1]), self.hashout_hex(row[2]), int(row[3]), int(row[4]), int(row[5]), int(row[6]), self.hashout_hex(row[7]), int(row[8]))
420 "block_height": block_height,
421 "version": block_version,
422 "prev_block_hash": prev_block_hash,
423 "merkle_root": hashMerkleRoot,
429 def get_chunk(self, index):
430 with self.cache_lock:
431 msg = self.chunk_cache.get(index)
439 block_hashMerkleRoot,
447 WHERE block_height >= %d AND block_height< %d AND in_longest = 1 ORDER BY block_height""" % (index * 2016, (index+1) * 2016)
449 out = self.safe_sql(sql)
452 (block_hash, block_version, hashMerkleRoot, nTime, nBits, nNonce, height, prev_block_hash, block_height) \
453 = (self.hashout_hex(row[0]), int(row[1]), self.hashout_hex(row[2]), int(row[3]), int(row[4]), int(row[5]), int(row[6]), self.hashout_hex(row[7]), int(row[8]))
455 "block_height": block_height,
456 "version": block_version,
457 "prev_block_hash": prev_block_hash,
458 "merkle_root": hashMerkleRoot,
464 if h.get('block_height') == 0:
465 h['prev_block_hash'] = "0" * 64
466 msg += header_to_string(h)
468 #print_log("hash", encode(Hash(msg.decode('hex'))))
469 #if h.get('block_height')==1:break
471 with self.cache_lock:
472 self.chunk_cache[index] = msg
473 print_log("get_chunk", index, len(msg))
476 def get_raw_tx(self, tx_hash):
477 postdata = dumps({"method": 'getrawtransaction', 'params': [tx_hash, 0], 'id': 'jsonrpc'})
478 respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
480 if r['error'] is not None:
481 raise Exception(r['error'])
483 return r.get('result')
485 def get_tx_merkle(self, tx_hash):
486 out = self.safe_sql("""
487 SELECT block_tx.block_id FROM tx
488 JOIN block_tx on tx.tx_id = block_tx.tx_id
489 JOIN chain_summary on chain_summary.block_id = block_tx.block_id
490 WHERE tx_hash='%s' AND in_longest = 1""" % tx_hash)
493 raise Exception("not in a block")
494 block_id = int(out[0][0])
497 out = self.safe_sql("SELECT block_height FROM chain_summary WHERE block_id = %d AND in_longest = 1" % block_id)
500 raise Exception("block not found")
501 block_height = int(out[0][0])
506 # list all tx in block
507 for row in self.safe_sql("""
508 SELECT DISTINCT tx_id, tx_pos, tx_hash
511 ORDER BY tx_pos""", (block_id,)):
512 _id, _pos, _hash = row
518 # TODO: do not compute this on client request, better store the hash tree of each block in a database...
520 merkle = map(decode, merkle)
521 target_hash = decode(tx_hash)
524 while len(merkle) != 1:
526 merkle.append(merkle[-1])
529 new_hash = Hash(merkle[0] + merkle[1])
530 if merkle[0] == target_hash:
531 s.append(encode(merkle[1]))
532 target_hash = new_hash
533 elif merkle[1] == target_hash:
534 s.append(encode(merkle[0]))
535 target_hash = new_hash
541 return {"block_height": block_height, "merkle": s, "pos": tx_pos}
543 def memorypool_update(store):
544 ds = BCDataStream.BCDataStream()
545 postdata = dumps({"method": 'getrawmempool', 'params': [], 'id': 'jsonrpc'})
546 respdata = urllib.urlopen(store.bitcoind_url, postdata).read()
549 if r['error'] is not None:
550 print_log(r['error'])
553 mempool_hashes = r.get('result')
556 for tx_hash in mempool_hashes:
558 if tx_hash in store.known_mempool_hashes:
560 store.known_mempool_hashes.append(tx_hash)
563 postdata = dumps({"method": 'getrawtransaction', 'params': [tx_hash], 'id': 'jsonrpc'})
564 respdata = urllib.urlopen(store.bitcoind_url, postdata).read()
566 if r['error'] is not None:
568 hextx = r.get('result')
570 ds.write(hextx.decode('hex'))
571 tx = deserialize.parse_Transaction(ds)
572 tx['hash'] = Hash(tx['tx'])
574 if store.tx_find_id_and_value(tx):
577 tx_id = store.import_tx(tx, False)
578 store.update_tx_cache(tx_id)
582 store.known_mempool_hashes = mempool_hashes
585 def send_tx(self, tx):
586 postdata = dumps({"method": 'sendrawtransaction', 'params': [tx], 'id': 'jsonrpc'})
587 respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
589 if r['error'] is not None:
590 msg = r['error'].get('message')
591 out = "error: transaction rejected by memorypool: " + msg + "\n" + tx
596 def main_iteration(self):
601 time_catch_up = t2 - t1
602 n = self.memorypool_update()
603 time_mempool = time.time() - t2
604 height = self.get_block_number(self.chain_id)
606 with self.cache_lock:
608 self.chunk_cache.pop(height/2016)
612 block_header = self.get_block_header(height)
613 return block_header, time_catch_up, time_mempool, n
616 # if there is an exception, do rollback and then re-raise the exception
617 for dircfg in store.datadirs:
619 store.catch_up_dir(dircfg)
621 store.log.exception("Failed to catch up %s", dircfg)
626 class BlockchainProcessor(Processor):
628 def __init__(self, config, shared):
629 Processor.__init__(self)
630 self.store = AbeStore(config)
631 self.watched_addresses = []
635 self.block_header, time_catch_up, time_mempool, n = self.store.main_iteration()
636 self.block_number = self.block_header.get('block_height')
637 print_log("blockchain: %d blocks" % self.block_number)
639 threading.Timer(10, self.run_store_iteration).start()
641 def add_request(self, request):
642 # see if we can get if from cache. if not, add to queue
643 if self.process(request, cache_only=True) == -1:
644 self.queue.put(request)
646 def process(self, request, cache_only=False):
647 #print_log("abe process", request)
649 message_id = request['id']
650 method = request['method']
651 params = request.get('params', [])
655 if method == 'blockchain.numblocks.subscribe':
656 result = self.block_number
658 elif method == 'blockchain.headers.subscribe':
659 result = self.block_header
661 elif method == 'blockchain.address.subscribe':
664 result = self.store.get_status(address, cache_only)
665 self.watch_address(address)
667 error = str(e) + ': ' + address
668 print_log("error:", error)
670 elif method == 'blockchain.address.get_history':
673 result = self.store.get_history(address, cache_only)
675 error = str(e) + ': ' + address
676 print_log("error:", error)
678 elif method == 'blockchain.block.get_header':
684 result = self.store.get_block_header(height)
686 error = str(e) + ': %d' % height
687 print_log("error:", error)
689 elif method == 'blockchain.block.get_chunk':
695 result = self.store.get_chunk(index)
697 error = str(e) + ': %d' % index
698 print_log("error:", error)
700 elif method == 'blockchain.transaction.broadcast':
701 txo = self.store.send_tx(params[0])
702 print_log("sent tx:", txo)
705 elif method == 'blockchain.transaction.get_merkle':
711 result = self.store.get_tx_merkle(tx_hash)
713 error = str(e) + ': ' + tx_hash
714 print_log("error:", error)
716 elif method == 'blockchain.transaction.get':
720 result = self.store.get_raw_tx(tx_hash)
722 error = str(e) + ': ' + tx_hash
723 print_log("error:", error)
726 error = "unknown method:%s" % method
728 if cache_only and result == -1:
732 response = {'id': message_id, 'error': error}
733 self.push_response(response)
735 response = {'id': message_id, 'result': result}
736 self.push_response(response)
738 def watch_address(self, addr):
739 if addr not in self.watched_addresses:
740 self.watched_addresses.append(addr)
742 def run_store_iteration(self):
744 block_header, time_catch_up, time_mempool, n = self.store.main_iteration()
746 traceback.print_exc(file=sys.stdout)
747 print_log("terminating")
750 if self.shared.stopped():
751 print_log("exit timer")
754 #print_log("block number: %d (%.3fs) mempool:%d (%.3fs)"%(self.block_number, time_catch_up, n, time_mempool))
756 if self.block_number != block_header.get('block_height'):
757 self.block_number = block_header.get('block_height')
758 print_log("block number: %d (%.3fs)" % (self.block_number, time_catch_up))
759 self.push_response({'id': None, 'method': 'blockchain.numblocks.subscribe', 'params': [self.block_number]})
761 if self.block_header != block_header:
762 self.block_header = block_header
763 self.push_response({'id': None, 'method': 'blockchain.headers.subscribe', 'params': [self.block_header]})
767 addr = self.store.address_queue.get(False)
770 if addr in self.watched_addresses:
772 status = self.store.get_status(addr)
773 self.push_response({'id': None, 'method': 'blockchain.address.subscribe', 'params': [addr, status]})
777 threading.Timer(10, self.run_store_iteration).start()