self.tx_cache = {}
self.bitcoind_url = 'http://%s:%s@%s:%s/' % ( config.get('bitcoind','user'), config.get('bitcoind','password'), config.get('bitcoind','host'), config.get('bitcoind','port'))
+ self.chunk_cache = {}
+
self.address_queue = Queue()
- self.dblock = thread.allocate_lock()
+ self.lock = threading.Lock() # for the database
+ self.cache_lock = threading.Lock() # for the cache
self.last_tx_id = 0
self.known_mempool_hashes = []
+
def import_tx(self, tx, is_coinbase):
tx_id = super(AbeStore, self).import_tx(tx, is_coinbase)
continue
address = hash_to_address(chr(self.addrtype), _hash)
- if self.tx_cache.has_key(address):
- print "cache: invalidating", address
- self.tx_cache.pop(address)
+ with self.cache_lock:
+ if self.tx_cache.has_key(address):
+ print "cache: invalidating", address
+ self.tx_cache.pop(address)
+
self.address_queue.put(address)
outrows = self.get_tx_outputs(txid, False)
continue
address = hash_to_address(chr(self.addrtype), _hash)
- if self.tx_cache.has_key(address):
- print "cache: invalidating", address
- self.tx_cache.pop(address)
+ with self.cache_lock:
+ if self.tx_cache.has_key(address):
+ print "cache: invalidating", address
+ self.tx_cache.pop(address)
+
self.address_queue.put(address)
def safe_sql(self,sql, params=(), lock=True):
error = False
try:
- if lock: self.dblock.acquire()
+ if lock: self.lock.acquire()
ret = self.selectall(sql,params)
except:
error = True
traceback.print_exc(file=sys.stdout)
finally:
- if lock: self.dblock.release()
+ if lock: self.lock.release()
if error:
raise BaseException('sql error')
raise BaseException('limit reached')
return out
- def get_history(self, addr):
- cached_version = self.tx_cache.get( addr )
- if cached_version is not None:
- return cached_version
+
+ def get_history(self, addr, cache_only=False):
+ with self.cache_lock:
+ cached_version = self.tx_cache.get( addr )
+ if cached_version is not None:
+ return cached_version
+
+ if cache_only: return -1
version, binaddr = decode_check_address(addr)
if binaddr is None:
# cache result
# do not cache mempool results because statuses are ambiguous
if not address_has_mempool:
- self.tx_cache[addr] = txpoints
+ with self.cache_lock:
+ self.tx_cache[addr] = txpoints
return txpoints
+ def get_history2(self, addr, cache_only=False):
+ h = self.get_history(addr, cache_only)
+ if cache_only and h==-1: return -1
- def get_status(self,addr):
+ out = map(lambda x: {'tx_hash':x['tx_hash'], 'height':x['height']}, h)
+ out2 = []
+ for item in out:
+ if item not in out2: out2.append(item)
+ return out2
+
+
+ def get_status(self, addr, cache_only=False):
# get address status, i.e. the last block for that address.
- tx_points = self.get_history(addr)
+ tx_points = self.get_history(addr, cache_only)
+ if cache_only and tx_points == -1: return -1
+
if not tx_points:
status = None
else:
status = status + ':%d'% len(tx_points)
return status
+ def get_status2(self, addr, cache_only=False):
+ # for 0.5 clients
+ tx_points = self.get_history2(addr, cache_only)
+ if cache_only and tx_points == -1: return -1
+
+ if not tx_points: return None
+ status = ''
+ for tx in tx_points:
+ status += tx.get('tx_hash') + ':%d:' % tx.get('height')
+ return hashlib.sha256( status ).digest().encode('hex')
+
def get_block_header(self, block_height):
out = self.safe_sql("""
def get_chunk(self, index):
+ with self.cache_lock:
+ msg = self.chunk_cache.get(index)
+ if msg: return msg
+
sql = """
SELECT
block_hash,
prev_block_hash,
block_height
FROM chain_summary
- WHERE block_height >= %d AND block_height< %d AND in_longest = 1"""%(index*2016, (index+1)*2016)
+ WHERE block_height >= %d AND block_height< %d AND in_longest = 1 ORDER BY block_height"""%(index*2016, (index+1)*2016)
out = self.safe_sql(sql)
msg = ''
#print "hash", encode(Hash(msg.decode('hex')))
#if h.get('block_height')==1:break
+ with self.cache_lock:
+ self.chunk_cache[index] = msg
print "get_chunk", index, len(msg)
return msg
+ def get_raw_tx(self, tx_hash, height):
+ postdata = dumps({"method": 'getrawtransaction', 'params': [tx_hash, 0, height], 'id':'jsonrpc'})
+ respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
+ r = loads(respdata)
+ if r['error'] != None:
+ raise BaseException(r['error'])
+
+ hextx = r.get('result')
+ return hextx
+
+
def get_tx_merkle(self, tx_hash):
out = self.safe_sql("""
JOIN block_tx on tx.tx_id = block_tx.tx_id
JOIN chain_summary on chain_summary.block_id = block_tx.block_id
WHERE tx_hash='%s' AND in_longest = 1"""%tx_hash)
- block_id = out[0]
+
+ if not out: raise BaseException("not in a block")
+ block_id = int(out[0][0])
# get block height
out = self.safe_sql("SELECT block_height FROM chain_summary WHERE block_id = %d AND in_longest = 1"%block_id)
block_height = int(out[0][0])
merkle = []
+ tx_pos = None
+
# list all tx in block
for row in self.safe_sql("""
SELECT DISTINCT tx_id, tx_pos, tx_hash
FROM txin_detail
WHERE block_id = ?
ORDER BY tx_pos""", (block_id,)):
- tx_id, tx_pos, tx_h = row
- merkle.append(tx_h)
+ _id, _pos, _hash = row
+ merkle.append(_hash)
+ if _hash == tx_hash: tx_pos = int(_pos)
# find subset.
# TODO: do not compute this on client request, better store the hash tree of each block in a database...
while merkle:
new_hash = Hash( merkle[0] + merkle[1] )
if merkle[0] == target_hash:
- s.append( "L" + encode(merkle[1]))
+ s.append( encode(merkle[1]))
target_hash = new_hash
elif merkle[1] == target_hash:
- s.append( "R" + encode(merkle[0]))
+ s.append( encode(merkle[0]))
target_hash = new_hash
n.append( new_hash )
merkle = merkle[2:]
merkle = n
# send result
- return {"block_height":block_height,"merkle":s}
+ return {"block_height":block_height, "merkle":s, "pos":tx_pos}
ds = BCDataStream.BCDataStream()
postdata = dumps({"method": 'getrawmempool', 'params': [], 'id':'jsonrpc'})
respdata = urllib.urlopen(store.bitcoind_url, postdata).read()
+
r = loads(respdata)
if r['error'] != None:
print r['error']
return
mempool_hashes = r.get('result')
+ num_new_tx = 0
+
for tx_hash in mempool_hashes:
if tx_hash in store.known_mempool_hashes: continue
store.known_mempool_hashes.append(tx_hash)
+ num_new_tx += 1
postdata = dumps({"method": 'getrawtransaction', 'params': [tx_hash], 'id':'jsonrpc'})
respdata = urllib.urlopen(store.bitcoind_url, postdata).read()
store.commit()
store.known_mempool_hashes = mempool_hashes
+ return num_new_tx
def send_tx(self,tx):
return out
- def main_iteration(store):
- with store.dblock:
- store.catch_up()
- store.memorypool_update()
- height = store.get_block_number( store.chain_id )
+ def main_iteration(self):
+ with self.lock:
+ t1 = time.time()
+ self.catch_up()
+ t2 = time.time()
+ time_catch_up = t2 - t1
+ n = self.memorypool_update()
+ time_mempool = time.time() - t2
+ height = self.get_block_number( self.chain_id )
+
+ with self.cache_lock:
+ try:
+ self.chunk_cache.pop(height/2016)
+ except:
+ pass
- block_header = store.get_block_header( height )
- return block_header
+ block_header = self.get_block_header( height )
+ return block_header, time_catch_up, time_mempool, n
self.watched_addresses = []
# catch_up first
- self.block_header = self.store.main_iteration()
+ self.block_header, time_catch_up, time_mempool, n = self.store.main_iteration()
self.block_number = self.block_header.get('block_height')
print "blockchain: %d blocks"%self.block_number
threading.Timer(10, self.run_store_iteration).start()
- def process(self, request):
+
+ def add_request(self, request):
+ # see if we can get if from cache. if not, add to queue
+ if self.process( request, cache_only = True) == -1:
+ self.queue.put(request)
+
+
+ def process(self, request, cache_only = False):
#print "abe process", request
message_id = request['id']
elif method == 'blockchain.address.subscribe':
try:
address = params[0]
- result = self.store.get_status(address)
+ result = self.store.get_status(address, cache_only)
self.watch_address(address)
except BaseException, e:
error = str(e) + ': ' + address
print "error:", error
- elif method == 'blockchain.address.get_history':
+ elif method == 'blockchain.address.subscribe2':
try:
address = params[0]
- result = self.store.get_history( address )
+ result = self.store.get_status2(address, cache_only)
+ self.watch_address(address)
except BaseException, e:
error = str(e) + ': ' + address
print "error:", error
- elif method == 'blockchain.block.get_header':
+ elif method == 'blockchain.address.get_history':
try:
- height = params[0]
- result = self.store.get_block_header( height )
+ address = params[0]
+ result = self.store.get_history( address, cache_only )
except BaseException, e:
- error = str(e) + ': %d'% height
+ error = str(e) + ': ' + address
print "error:", error
- elif method == 'blockchain.block.get_chunk':
+ elif method == 'blockchain.address.get_history2':
try:
- index = params[0]
- result = self.store.get_chunk( index )
+ address = params[0]
+ result = self.store.get_history2( address, cache_only )
except BaseException, e:
- error = str(e) + ': %d'% index
+ error = str(e) + ': ' + address
print "error:", error
+ elif method == 'blockchain.block.get_header':
+ if cache_only:
+ result = -1
+ else:
+ try:
+ height = params[0]
+ result = self.store.get_block_header( height )
+ except BaseException, e:
+ error = str(e) + ': %d'% height
+ print "error:", error
+
+ elif method == 'blockchain.block.get_chunk':
+ if cache_only:
+ result = -1
+ else:
+ try:
+ index = params[0]
+ result = self.store.get_chunk( index )
+ except BaseException, e:
+ error = str(e) + ': %d'% index
+ print "error:", error
+
elif method == 'blockchain.transaction.broadcast':
txo = self.store.send_tx(params[0])
print "sent tx:", txo
result = txo
elif method == 'blockchain.transaction.get_merkle':
+ if cache_only:
+ result = -1
+ else:
+ try:
+ tx_hash = params[0]
+ result = self.store.get_tx_merkle(tx_hash )
+ except BaseException, e:
+ error = str(e) + ': ' + tx_hash
+ print "error:", error
+
+ elif method == 'blockchain.transaction.get':
try:
tx_hash = params[0]
- result = self.store.get_tx_merkle(tx_hash )
+ height = params[1]
+ result = self.store.get_raw_tx(tx_hash, height )
except BaseException, e:
error = str(e) + ': ' + tx_hash
print "error:", error
else:
error = "unknown method:%s"%method
+ if cache_only and result == -1: return -1
if error:
response = { 'id':message_id, 'error':error }
def run_store_iteration(self):
try:
- block_header = self.store.main_iteration()
+ block_header, time_catch_up, time_mempool, n = self.store.main_iteration()
except:
traceback.print_exc(file=sys.stdout)
print "terminating"
print "exit timer"
return
+ #print "block number: %d (%.3fs) mempool:%d (%.3fs)"%(self.block_number, time_catch_up, n, time_mempool)
+
if self.block_number != block_header.get('block_height'):
self.block_number = block_header.get('block_height')
- print "block number:", self.block_number
+ print "block number: %d (%.3fs)"%(self.block_number, time_catch_up)
self.push_response({ 'id': None, 'method':'blockchain.numblocks.subscribe', 'params':[self.block_number] })
if self.block_header != block_header:
self.block_header = block_header
self.push_response({ 'id': None, 'method':'blockchain.headers.subscribe', 'params':[self.block_header] })
+
while True:
try:
addr = self.store.address_queue.get(False)
break
if addr in self.watched_addresses:
status = self.store.get_status( addr )
+ status2 = self.store.get_status2( addr )
self.push_response({ 'id': None, 'method':'blockchain.address.subscribe', 'params':[addr, status] })
+ self.push_response({ 'id': None, 'method':'blockchain.address.subscribe2', 'params':[addr, status2] })
threading.Timer(10, self.run_store_iteration).start()