p2pool/web.py

   1 from __future__ import division
   2
   3 import cgi
   4 import errno
   5 import json
   6 import os
   7 import sys
   8 import time
   9
  10 from twisted.internet import reactor, task
  11 from twisted.python import log
  12 from twisted.web import resource, static
  13
  14 from bitcoin import data as bitcoin_data
  15 from . import data as p2pool_data, graphs
  16 from util import graph, math
  17
  18 def _atomic_read(filename):
  19     try:
  20         with open(filename, 'rb') as f:
  21             return f.read()
  22     except IOError, e:
  23         if e.errno != errno.ENOENT:
  24             raise
  25     try:
  26         with open(filename + '.new', 'rb') as f:
  27             return f.read()
  28     except IOError, e:
  29         if e.errno != errno.ENOENT:
  30             raise
  31     return None
  32
  33 def _atomic_write(filename, data):
  34     with open(filename + '.new', 'wb') as f:
  35         f.write(data)
  36         f.flush()
  37         try:
  38             os.fsync(f.fileno())
  39         except:
  40             pass
  41     try:
  42         os.rename(filename + '.new', filename)
  43     except os.error: # windows can't overwrite
  44         os.remove(filename)
  45         os.rename(filename + '.new', filename)
  46
  47 def get_web_root(tracker, current_work, current_work2, get_current_txouts, datadir_path, net, get_stale_counts, my_pubkey_hash, local_rate_monitor, worker_fee, p2p_node, my_share_hashes, recent_blocks, pseudoshare_received):
  48     start_time = time.time()
  49
  50     web_root = resource.Resource()
  51
  52     def get_rate():
  53         if tracker.get_height(current_work.value['best_share_hash']) < 720:
  54             return json.dumps(None)
  55         return json.dumps(p2pool_data.get_pool_attempts_per_second(tracker, current_work.value['best_share_hash'], 720)
  56             / (1 - p2pool_data.get_average_stale_prop(tracker, current_work.value['best_share_hash'], 720)))
  57
  58     def get_users():
  59         height, last = tracker.get_height_and_last(current_work.value['best_share_hash'])
  60         weights, total_weight, donation_weight = tracker.get_cumulative_weights(current_work.value['best_share_hash'], min(height, 720), 65535*2**256, False)
  61         res = {}
  62         for script in sorted(weights, key=lambda s: weights[s]):
  63             res[bitcoin_data.script2_to_human(script, net.PARENT)] = weights[script]/total_weight
  64         return json.dumps(res)
  65
  66     def get_current_scaled_txouts(scale, trunc=0):
  67         txouts = get_current_txouts()
  68         total = sum(txouts.itervalues())
  69         results = dict((script, value*scale//total) for script, value in txouts.iteritems())
  70         if trunc > 0:
  71             total_random = 0
  72             random_set = set()
  73             for s in sorted(results, key=results.__getitem__):
  74                 if results[s] >= trunc:
  75                     break
  76                 total_random += results[s]
  77                 random_set.add(s)
  78             if total_random:
  79                 winner = math.weighted_choice((script, results[script]) for script in random_set)
  80                 for script in random_set:
  81                     del results[script]
  82                 results[winner] = total_random
  83         if sum(results.itervalues()) < int(scale):
  84             results[math.weighted_choice(results.iteritems())] += int(scale) - sum(results.itervalues())
  85         return results
  86
  87     def get_current_payouts():
  88         return json.dumps(dict((bitcoin_data.script2_to_human(script, net.PARENT), value/1e8) for script, value in get_current_txouts().iteritems()))
  89
  90     def get_patron_sendmany(total=None, trunc='0.01'):
  91         if total is None:
  92             return 'need total argument. go to patron_sendmany/<TOTAL>'
  93         total = int(float(total)*1e8)
  94         trunc = int(float(trunc)*1e8)
  95         return json.dumps(dict(
  96             (bitcoin_data.script2_to_address(script, net.PARENT), value/1e8)
  97             for script, value in get_current_scaled_txouts(total, trunc).iteritems()
  98             if bitcoin_data.script2_to_address(script, net.PARENT) is not None
  99         ))
 100
 101     def get_global_stats():
 102         # averaged over last hour
 103         lookbehind = 3600//net.SHARE_PERIOD
 104         if tracker.get_height(current_work.value['best_share_hash']) < lookbehind:
 105             return None
 106
 107         nonstale_hash_rate = p2pool_data.get_pool_attempts_per_second(tracker, current_work.value['best_share_hash'], lookbehind)
 108         stale_prop = p2pool_data.get_average_stale_prop(tracker, current_work.value['best_share_hash'], lookbehind)
 109         return json.dumps(dict(
 110             pool_nonstale_hash_rate=nonstale_hash_rate,
 111             pool_hash_rate=nonstale_hash_rate/(1 - stale_prop),
 112             pool_stale_prop=stale_prop,
 113         ))
 114
 115     def get_local_stats():
 116         lookbehind = 3600//net.SHARE_PERIOD
 117         if tracker.get_height(current_work.value['best_share_hash']) < lookbehind:
 118             return None
 119
 120         global_stale_prop = p2pool_data.get_average_stale_prop(tracker, current_work.value['best_share_hash'], lookbehind)
 121
 122         my_unstale_count = sum(1 for share in tracker.get_chain(current_work.value['best_share_hash'], lookbehind) if share.hash in my_share_hashes)
 123         my_orphan_count = sum(1 for share in tracker.get_chain(current_work.value['best_share_hash'], lookbehind) if share.hash in my_share_hashes and share.share_data['stale_info'] == 253)
 124         my_doa_count = sum(1 for share in tracker.get_chain(current_work.value['best_share_hash'], lookbehind) if share.hash in my_share_hashes and share.share_data['stale_info'] == 254)
 125         my_share_count = my_unstale_count + my_orphan_count + my_doa_count
 126         my_stale_count = my_orphan_count + my_doa_count
 127
 128         my_stale_prop = my_stale_count/my_share_count if my_share_count != 0 else None
 129
 130         my_work = sum(bitcoin_data.target_to_average_attempts(share.target)
 131             for share in tracker.get_chain(current_work.value['best_share_hash'], lookbehind - 1)
 132             if share.hash in my_share_hashes)
 133         actual_time = (tracker.shares[current_work.value['best_share_hash']].timestamp -
 134             tracker.shares[tracker.get_nth_parent_hash(current_work.value['best_share_hash'], lookbehind - 1)].timestamp)
 135         share_att_s = my_work / actual_time
 136
 137         miner_hash_rates = {}
 138         miner_dead_hash_rates = {}
 139         datums, dt = local_rate_monitor.get_datums_in_last()
 140         for datum in datums:
 141             miner_hash_rates[datum['user']] = miner_hash_rates.get(datum['user'], 0) + datum['work']/dt
 142             if datum['dead']:
 143                 miner_dead_hash_rates[datum['user']] = miner_dead_hash_rates.get(datum['user'], 0) + datum['work']/dt
 144
 145         return json.dumps(dict(
 146             my_hash_rates_in_last_hour=dict(
 147                 note="DEPRECATED",
 148                 nonstale=share_att_s,
 149                 rewarded=share_att_s/(1 - global_stale_prop),
 150                 actual=share_att_s/(1 - my_stale_prop) if my_stale_prop is not None else 0, # 0 because we don't have any shares anyway
 151             ),
 152             my_share_counts_in_last_hour=dict(
 153                 shares=my_share_count,
 154                 unstale_shares=my_unstale_count,
 155                 stale_shares=my_stale_count,
 156                 orphan_stale_shares=my_orphan_count,
 157                 doa_stale_shares=my_doa_count,
 158             ),
 159             my_stale_proportions_in_last_hour=dict(
 160                 stale=my_stale_prop,
 161                 orphan_stale=my_orphan_count/my_share_count if my_share_count != 0 else None,
 162                 dead_stale=my_doa_count/my_share_count if my_share_count != 0 else None,
 163             ),
 164             miner_hash_rates=miner_hash_rates,
 165             miner_dead_hash_rates=miner_dead_hash_rates,
 166         ))
 167
 168     def get_peer_addresses():
 169         return ' '.join(peer.transport.getPeer().host + (':' + str(peer.transport.getPeer().port) if peer.transport.getPeer().port != net.P2P_PORT else '') for peer in p2p_node.peers.itervalues())
 170
 171     def get_uptime():
 172         return json.dumps(time.time() - start_time)
 173
 174     class WebInterface(resource.Resource):
 175         def __init__(self, func, mime_type, args=()):
 176             resource.Resource.__init__(self)
 177             self.func, self.mime_type, self.args = func, mime_type, args
 178
 179         def getChild(self, child, request):
 180             return WebInterface(self.func, self.mime_type, self.args + (child,))
 181
 182         def render_GET(self, request):
 183             request.setHeader('Content-Type', self.mime_type)
 184             request.setHeader('Access-Control-Allow-Origin', '*')
 185             return self.func(*self.args)
 186
 187     web_root.putChild('rate', WebInterface(get_rate, 'application/json'))
 188     web_root.putChild('users', WebInterface(get_users, 'application/json'))
 189     web_root.putChild('fee', WebInterface(lambda: json.dumps(worker_fee), 'application/json'))
 190     web_root.putChild('current_payouts', WebInterface(get_current_payouts, 'application/json'))
 191     web_root.putChild('patron_sendmany', WebInterface(get_patron_sendmany, 'text/plain'))
 192     web_root.putChild('global_stats', WebInterface(get_global_stats, 'application/json'))
 193     web_root.putChild('local_stats', WebInterface(get_local_stats, 'application/json'))
 194     web_root.putChild('peer_addresses', WebInterface(get_peer_addresses, 'text/plain'))
 195     web_root.putChild('peer_versions', WebInterface(lambda: ''.join('%s:%i ' % peer.addr + peer.other_sub_version + '\n' for peer in p2p_node.peers.itervalues()), 'text/plain'))
 196     web_root.putChild('payout_addr', WebInterface(lambda: json.dumps(bitcoin_data.pubkey_hash_to_address(my_pubkey_hash, net.PARENT)), 'application/json'))
 197     web_root.putChild('recent_blocks', WebInterface(lambda: json.dumps(recent_blocks), 'application/json'))
 198     web_root.putChild('uptime', WebInterface(get_uptime, 'application/json'))
 199
 200     try:
 201         from . import draw
 202         web_root.putChild('chain_img', WebInterface(lambda: draw.get(tracker, current_work.value['best_share_hash']), 'image/png'))
 203     except ImportError:
 204         print "Install Pygame and PIL to enable visualizations! Visualizations disabled."
 205
 206     new_root = resource.Resource()
 207     web_root.putChild('web', new_root)
 208
 209     stat_log = []
 210     if os.path.exists(os.path.join(datadir_path, 'stats')):
 211         try:
 212             with open(os.path.join(datadir_path, 'stats'), 'rb') as f:
 213                 stat_log = json.loads(f.read())
 214         except:
 215             log.err(None, 'Error loading stats:')
 216     def update_stat_log():
 217         while stat_log and stat_log[0]['time'] < time.time() - 24*60*60:
 218             stat_log.pop(0)
 219
 220         lookbehind = 3600//net.SHARE_PERIOD
 221         if tracker.get_height(current_work.value['best_share_hash']) < lookbehind:
 222             return None
 223
 224         global_stale_prop = p2pool_data.get_average_stale_prop(tracker, current_work.value['best_share_hash'], lookbehind)
 225         (stale_orphan_shares, stale_doa_shares), shares, _ = get_stale_counts()
 226
 227         miner_hash_rates = {}
 228         miner_dead_hash_rates = {}
 229         datums, dt = local_rate_monitor.get_datums_in_last()
 230         for datum in datums:
 231             miner_hash_rates[datum['user']] = miner_hash_rates.get(datum['user'], 0) + datum['work']/dt
 232             if datum['dead']:
 233                 miner_dead_hash_rates[datum['user']] = miner_dead_hash_rates.get(datum['user'], 0) + datum['work']/dt
 234
 235         stat_log.append(dict(
 236             time=time.time(),
 237             pool_hash_rate=p2pool_data.get_pool_attempts_per_second(tracker, current_work.value['best_share_hash'], lookbehind)/(1-global_stale_prop),
 238             pool_stale_prop=global_stale_prop,
 239             local_hash_rates=miner_hash_rates,
 240             local_dead_hash_rates=miner_dead_hash_rates,
 241             shares=shares,
 242             stale_shares=stale_orphan_shares + stale_doa_shares,
 243             stale_shares_breakdown=dict(orphan=stale_orphan_shares, doa=stale_doa_shares),
 244             current_payout=get_current_txouts().get(bitcoin_data.pubkey_hash_to_script2(my_pubkey_hash), 0)*1e-8,
 245             peers=dict(
 246                 incoming=sum(1 for peer in p2p_node.peers.itervalues() if peer.incoming),
 247                 outgoing=sum(1 for peer in p2p_node.peers.itervalues() if not peer.incoming),
 248             ),
 249             attempts_to_share=bitcoin_data.target_to_average_attempts(tracker.shares[current_work.value['best_share_hash']].max_target),
 250             attempts_to_block=bitcoin_data.target_to_average_attempts(current_work.value['bits'].target),
 251             block_value=current_work2.value['subsidy']*1e-8,
 252         ))
 253
 254         with open(os.path.join(datadir_path, 'stats'), 'wb') as f:
 255             f.write(json.dumps(stat_log))
 256     task.LoopingCall(update_stat_log).start(5*60)
 257     new_root.putChild('log', WebInterface(lambda: json.dumps(stat_log), 'application/json'))
 258
 259     class ShareExplorer(resource.Resource):
 260         def __init__(self, share_hash):
 261             self.share_hash = share_hash
 262         def render_GET(self, request):
 263             request.setHeader('Content-Type', 'text/html')
 264             if self.share_hash not in tracker.shares:
 265                 return 'share not known'
 266             share = tracker.shares[self.share_hash]
 267
 268             format_bits = lambda bits: '%f (bits=%#8x) Work required: %sH</p>' % (bitcoin_data.target_to_difficulty(bits.target), bits.bits, math.format(bitcoin_data.target_to_average_attempts(bits.target)))
 269
 270             request.write('<h1>Share <a href="%x">%s</a></h1>' % (share.hash, p2pool_data.format_hash(share.hash)))
 271             if share.previous_hash is not None:
 272                 request.write('<p>Previous: <a href="%x">%s</a>' % (share.previous_hash, p2pool_data.format_hash(share.previous_hash)))
 273             if tracker.get_height(share.hash) >= 100:
 274                 jump_hash = tracker.get_nth_parent_hash(share.hash, 100)
 275                 if jump_hash is not None:
 276                     request.write(' (100 jump <a href="%x">%s</a>)' % (jump_hash, p2pool_data.format_hash(jump_hash)))
 277             request.write('</p>')
 278             request.write('<p>Next: %s</p>' % (', '.join('<a href="%x">%s</a>' % (next, p2pool_data.format_hash(next)) for next in sorted(tracker.reverse_shares.get(share.hash, set()), key=lambda sh: -len(tracker.reverse_shares.get(sh, set())))),))
 279             request.write('<p>Verified: %s</p>' % (share.hash in tracker.verified.shares,))
 280             request.write('<p>Time first seen: %s</p>' % (time.ctime(start_time if share.time_seen == 0 else share.time_seen),))
 281             request.write('<p>Peer first received from: %s</p>' % ('%s:%i' % share.peer.addr if share.peer is not None else 'self or cache',))
 282
 283             request.write('<h2>Share data</h2>')
 284             request.write('<p>Timestamp: %s (%i)</p>' % (time.ctime(share.timestamp), share.timestamp))
 285             request.write('<p>Difficulty: %s</p>' % (format_bits(share.share_info['bits']),))
 286             request.write('<p>Minimum difficulty: %s</p>' % (format_bits(share.share_info.get('max_bits', share.share_info['bits'])),))
 287             request.write('<p>Payout script: %s</p>' % (bitcoin_data.script2_to_human(share.new_script, share.net.PARENT),))
 288             request.write('<p>Donation: %.2f%%</p>' % (share.share_data['donation']/65535*100,))
 289             request.write('<p>Stale info: %s</p>' % ({0: 'none', 253: 'had an orphan', 254: 'had a dead'}.get(share.share_data['stale_info'], 'unknown %i' % (share.share_data['stale_info'],)),))
 290             request.write('<p>Nonce: %s</p>' % (cgi.escape(repr(share.share_data['nonce'])),))
 291
 292             request.write('<h2>Block header</h2>')
 293             request.write('<p>Hash: <a href="%s%064x">%064x</a></p>' % (net.PARENT.BLOCK_EXPLORER_URL_PREFIX, share.header_hash, share.header_hash))
 294             request.write('<p>Version: %i</p>' % (share.header['version'],))
 295             request.write('<p>Previous block: <a href="%s%064x">%064x</a></p>' % (net.PARENT.BLOCK_EXPLORER_URL_PREFIX, share.header['previous_block'], share.header['previous_block']))
 296             request.write('<p>Timestamp: %s (%i)</p>' % (time.ctime(share.header['timestamp']), share.header['timestamp']))
 297             request.write('<p>Difficulty: %f (bits=%#8x) Work: %sH</p>' % (bitcoin_data.target_to_difficulty(share.header['bits'].target), share.header['bits'].bits, math.format(bitcoin_data.target_to_average_attempts(share.header['bits'].target))))
 298             request.write('<p>Nonce: %i</p>' % (share.header['nonce'],))
 299             if share.other_txs is not None:
 300                 tx_count = len(share.other_txs)
 301             elif len(share.merkle_branch) == 0:
 302                 tx_count = 1
 303             else:
 304                 tx_count = 'between %i and %i' % (2**len(share.merkle_branch)//2+1, 2**len(share.merkle_branch))
 305             request.write('<p>Transactions: %s</p>' % (tx_count,))
 306             coinbase = share.share_data['coinbase'].ljust(2, '\x00')
 307             request.write('<p>Coinbase: %s %s</p>' % (cgi.escape(repr(coinbase)), coinbase.encode('hex')))
 308             request.write('<p>Generation value: %.8f %s</p>' % (share.share_data['subsidy']*1e-8, net.PARENT.SYMBOL))
 309             #request.write('<p>Generation txn: %32x</p>' % (share.gentx_hash,))
 310
 311             return ''
 312     class Explorer(resource.Resource):
 313         def render_GET(self, request):
 314             if not request.path.endswith('/'):
 315                 request.redirect(request.path + '/')
 316                 return ''
 317             request.setHeader('Content-Type', 'text/html')
 318             request.write('<h1>P2Pool share explorer</h1>')
 319
 320             request.write('<h2>Verified heads</h2>')
 321             request.write('<ul>')
 322             for h in tracker.verified.heads:
 323                 request.write('<li><a href="%x">%s%s</a></li>' % (h, p2pool_data.format_hash(h), ' BEST' if h == current_work.value['best_share_hash'] else ''))
 324             request.write('</ul>')
 325
 326             request.write('<h2>Verified tails</h2>')
 327             request.write('<ul>')
 328             for tail in tracker.verified.tails:
 329                 for h in tracker.reverse_shares.get(tail, set()):
 330                     request.write('<li><a href="%x">%s%s</a></li>' % (h, p2pool_data.format_hash(h), ' BEST' if h == current_work.value['best_share_hash'] else ''))
 331             request.write('</ul>')
 332
 333             request.write('<h2>Heads</h2>')
 334             request.write('<ul>')
 335             for h in tracker.heads:
 336                 request.write('<li><a href="%x">%s%s</a></li>' % (h, p2pool_data.format_hash(h), ' BEST' if h == current_work.value['best_share_hash'] else ''))
 337             request.write('</ul>')
 338
 339             request.write('<h2>Tails</h2>')
 340             request.write('<ul>')
 341             for tail in tracker.tails:
 342                 for h in tracker.reverse_shares.get(tail, set()):
 343                     request.write('<li><a href="%x">%s%s</a></li>' % (h, p2pool_data.format_hash(h), ' BEST' if h == current_work.value['best_share_hash'] else ''))
 344             request.write('</ul>')
 345
 346             return ''
 347         def getChild(self, child, request):
 348             if not child:
 349                 return self
 350             return ShareExplorer(int(child, 16))
 351     new_root.putChild('explorer', Explorer())
 352
 353     grapher = graphs.Grapher(os.path.join(datadir_path, 'rrd'))
 354     web_root.putChild('graphs', grapher.get_resource())
 355     def add_point():
 356         if tracker.get_height(current_work.value['best_share_hash']) < 720:
 357             return
 358         nonstalerate = p2pool_data.get_pool_attempts_per_second(tracker, current_work.value['best_share_hash'], 720)
 359         poolrate = nonstalerate / (1 - p2pool_data.get_average_stale_prop(tracker, current_work.value['best_share_hash'], 720))
 360         grapher.add_poolrate_point(poolrate, poolrate - nonstalerate)
 361     task.LoopingCall(add_point).start(100)
 362     @pseudoshare_received.watch
 363     def _(work, dead, user):
 364         reactor.callLater(1, grapher.add_localrate_point, work, dead)
 365         if user is not None:
 366             reactor.callLater(1, grapher.add_localminer_point, user, work, dead)
 367
 368     hd_path = os.path.join(datadir_path, 'graph_db')
 369     hd_data = _atomic_read(hd_path)
 370     hd_obj = {}
 371     if hd_data is not None:
 372         try:
 373             hd_obj = json.loads(hd_data)
 374         except Exception:
 375             log.err(None, 'Error reading graph database:')
 376     dataview_descriptions = {
 377         'last_hour': graph.DataViewDescription(150, 60*60),
 378         'last_day': graph.DataViewDescription(300, 60*60*24),
 379         'last_week': graph.DataViewDescription(300, 60*60*24*7),
 380         'last_month': graph.DataViewDescription(300, 60*60*24*30),
 381         'last_year': graph.DataViewDescription(300, 60*60*24*365.25),
 382     }
 383     hd = graph.HistoryDatabase.from_obj({
 384         'local_hash_rate': graph.DataStreamDescription(False, dataview_descriptions),
 385         'local_dead_hash_rate': graph.DataStreamDescription(False, dataview_descriptions),
 386         'pool_rate': graph.DataStreamDescription(True, dataview_descriptions),
 387         'pool_stale_rate': graph.DataStreamDescription(True, dataview_descriptions),
 388         'current_payout': graph.DataStreamDescription(True, dataview_descriptions),
 389     }, hd_obj)
 390     task.LoopingCall(lambda: _atomic_write(hd_path, json.dumps(hd.to_obj()))).start(100)
 391     @pseudoshare_received.watch
 392     def _(work, dead, user):
 393         t = time.time()
 394         hd.datastreams['local_hash_rate'].add_datum(t, work)
 395         if dead:
 396             hd.datastreams['local_dead_hash_rate'].add_datum(t, work)
 397     def add_point():
 398         if tracker.get_height(current_work.value['best_share_hash']) < 720:
 399             return
 400         nonstalerate = p2pool_data.get_pool_attempts_per_second(tracker, current_work.value['best_share_hash'], 720)
 401         poolrate = nonstalerate / (1 - p2pool_data.get_average_stale_prop(tracker, current_work.value['best_share_hash'], 720))
 402         t = time.time()
 403         hd.datastreams['pool_rate'].add_datum(t, poolrate)
 404         hd.datastreams['pool_stale_rate'].add_datum(t, poolrate - nonstalerate)
 405         hd.datastreams['current_payout'].add_datum(t, get_current_txouts().get(bitcoin_data.pubkey_hash_to_script2(my_pubkey_hash), 0)*1e-8)
 406     task.LoopingCall(add_point).start(5)
 407     new_root.putChild('graph_data', WebInterface(lambda source, view: json.dumps(hd.datastreams[source].dataviews[view].get_data(time.time())), 'application/json'))
 408
 409     web_root.putChild('static', static.File(os.path.join(os.path.dirname(sys.argv[0]), 'web-static')))
 410
 411     return web_root