display per-miner current payout in graphs
[p2pool.git] / p2pool / main.py
index 7834662..3fc8858 100644 (file)
@@ -12,6 +12,13 @@ import signal
 import traceback
 import urlparse
 
+try:
+    from twisted.internet import iocpreactor
+    iocpreactor.install()
+except:
+    pass
+else:
+    print 'Using IOCP reactor!'
 from twisted.internet import defer, reactor, protocol, task
 from twisted.web import server
 from twisted.python import log
@@ -38,7 +45,7 @@ def getwork(bitcoind):
         version=work['version'],
         previous_block_hash=int(work['previousblockhash'], 16),
         transactions=map(bitcoin_data.tx_type.unpack, packed_transactions),
-        merkle_branch=bitcoin_data.calculate_merkle_branch([0] + map(bitcoin_data.hash256, packed_transactions), 0),
+        merkle_link=bitcoin_data.calculate_merkle_link([0] + map(bitcoin_data.hash256, packed_transactions), 0), # using 0 is a bit of a hack, but will always work when index=0
         subsidy=work['coinbasevalue'],
         time=work['time'],
         bits=bitcoin_data.FloatingIntegerType().unpack(work['bits'].decode('hex')[::-1]) if isinstance(work['bits'], (str, unicode)) else bitcoin_data.FloatingInteger(work['bits']),
@@ -54,12 +61,19 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
         # connect to bitcoind over JSON-RPC and do initial getmemorypool
         url = 'http://%s:%i/' % (args.bitcoind_address, args.bitcoind_rpc_port)
         print '''Testing bitcoind RPC connection to '%s' with username '%s'...''' % (url, args.bitcoind_rpc_username)
-        bitcoind = jsonrpc.Proxy(url, dict(Authorization='Basic ' + base64.b64encode(args.bitcoind_rpc_username + ':' + args.bitcoind_rpc_password)))
-        good = yield deferral.retry('Error while checking bitcoind identity:', 1)(net.PARENT.RPC_CHECK)(bitcoind)
-        if not good:
-            print >>sys.stderr, "    Check failed! Make sure that you're connected to the right bitcoind with --bitcoind-rpc-port!"
-            return
-        temp_work = yield getwork(bitcoind)
+        bitcoind = jsonrpc.Proxy(url, dict(Authorization='Basic ' + base64.b64encode(args.bitcoind_rpc_username + ':' + args.bitcoind_rpc_password)), timeout=30)
+        @deferral.retry('Error while checking Bitcoin connection:', 1)
+        @defer.inlineCallbacks
+        def check():
+            if not (yield net.PARENT.RPC_CHECK)(bitcoind):
+                print >>sys.stderr, "    Check failed! Make sure that you're connected to the right bitcoind with --bitcoind-rpc-port!"
+                raise deferral.RetrySilentlyException()
+            temp_work = yield getwork(bitcoind)
+            if not net.VERSION_CHECK((yield bitcoind.rpc_getinfo())['version'], temp_work):
+                print >>sys.stderr, '    Bitcoin version too old! BIP16 support required! Upgrade to 0.6.0rc4 or greater!'
+                raise deferral.RetrySilentlyException()
+            defer.returnValue(temp_work)
+        temp_work = yield check()
         print '    ...success!'
         print '    Current block hash: %x' % (temp_work['previous_block_hash'],)
         print
@@ -109,7 +123,6 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
         shared_share_hashes = set()
         ss = p2pool_data.ShareStore(os.path.join(datadir_path, 'shares.'), net)
         known_verified = set()
-        recent_blocks = []
         print "Loading shares..."
         for i, (mode, contents) in enumerate(ss.get_shares()):
             if mode == 'share':
@@ -154,7 +167,7 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
             current_work2.set(dict(
                 time=work['time'],
                 transactions=work['transactions'],
-                merkle_branch=work['merkle_branch'],
+                merkle_link=work['merkle_link'],
                 subsidy=work['subsidy'],
                 clock_offset=time.time() - work['time'],
                 last_update=time.time(),
@@ -285,7 +298,7 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                             break
                         shares.append(share)
                 print 'Sending %i shares to %s:%i' % (len(shares), peer.addr[0], peer.addr[1])
-                peer.sendShares(shares)
+                return shares
         
         @deferral.retry('Error submitting block: (will retry)', 10, 10)
         @defer.inlineCallbacks
@@ -302,7 +315,6 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                 print
                 print 'GOT BLOCK FROM PEER! Passing to bitcoind! %s bitcoin: %s%064x' % (p2pool_data.format_hash(share.hash), net.PARENT.BLOCK_EXPLORER_URL_PREFIX, share.header_hash)
                 print
-                recent_blocks.append(dict(ts=share.timestamp, hash='%064x' % (share.header_hash,)))
         
         print 'Joining p2pool network using port %i...' % (args.p2pool_port,)
         
@@ -393,15 +405,6 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
         
         print 'Listening for workers on %r port %i...' % (worker_endpoint[0], worker_endpoint[1])
         
-        if os.path.exists(os.path.join(datadir_path, 'vip_pass')):
-            with open(os.path.join(datadir_path, 'vip_pass'), 'rb') as f:
-                vip_pass = f.read().strip('\r\n')
-        else:
-            vip_pass = '%016x' % (random.randrange(2**64),)
-            with open(os.path.join(datadir_path, 'vip_pass'), 'wb') as f:
-                f.write(vip_pass)
-        print '    Worker password:', vip_pass, '(only required for generating graphs)'
-        
         # setup worker logic
         
         removed_unstales_var = variable.Variable((0, 0, 0))
@@ -435,6 +438,7 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
         
         
         pseudoshare_received = variable.Event()
+        share_received = variable.Event()
         local_rate_monitor = math.RateMonitor(10*60)
         
         class WorkerBridge(worker_interface.WorkerBridge):
@@ -443,7 +447,7 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                 self.new_work_event = current_work.changed
                 self.recent_shares_ts_work = []
             
-            def preprocess_request(self, request):
+            def get_user_details(self, request):
                 user = request.getUser() if request.getUser() is not None else ''
                 
                 desired_pseudoshare_target = None
@@ -470,6 +474,10 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                     except: # XXX blah
                         pubkey_hash = my_pubkey_hash
                 
+                return user, pubkey_hash, desired_share_target, desired_pseudoshare_target
+            
+            def preprocess_request(self, request):
+                user, pubkey_hash, desired_share_target, desired_pseudoshare_target = self.get_user_details(request)
                 return pubkey_hash, desired_share_target, desired_pseudoshare_target
             
             def get_work(self, pubkey_hash, desired_share_target, desired_pseudoshare_target):
@@ -493,26 +501,29 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                     mm_data = ''
                     mm_later = []
                 
-                share_info, generate_tx = p2pool_data.Share.generate_transaction(
-                    tracker=tracker,
-                    share_data=dict(
-                        previous_share_hash=current_work.value['best_share_hash'],
-                        coinbase=(mm_data + current_work.value['coinbaseflags'])[:100],
-                        nonce=random.randrange(2**32),
-                        pubkey_hash=pubkey_hash,
-                        subsidy=current_work2.value['subsidy'],
-                        donation=math.perfect_round(65535*args.donation_percentage/100),
-                        stale_info=(lambda (orphans, doas), total, (orphans_recorded_in_chain, doas_recorded_in_chain):
-                            253 if orphans > orphans_recorded_in_chain else
-                            254 if doas > doas_recorded_in_chain else
-                            0
-                        )(*get_stale_counts()),
-                    ),
-                    block_target=current_work.value['bits'].target,
-                    desired_timestamp=int(time.time() - current_work2.value['clock_offset']),
-                    desired_target=desired_share_target,
-                    net=net,
-                )
+                if True:
+                    share_info, generate_tx = p2pool_data.Share.generate_transaction(
+                        tracker=tracker,
+                        share_data=dict(
+                            previous_share_hash=current_work.value['best_share_hash'],
+                            coinbase=(mm_data + current_work.value['coinbaseflags'])[:100],
+                            nonce=random.randrange(2**32),
+                            pubkey_hash=pubkey_hash,
+                            subsidy=current_work2.value['subsidy'],
+                            donation=math.perfect_round(65535*args.donation_percentage/100),
+                            stale_info=(lambda (orphans, doas), total, (orphans_recorded_in_chain, doas_recorded_in_chain):
+                                253 if orphans > orphans_recorded_in_chain else
+                                254 if doas > doas_recorded_in_chain else
+                                0
+                            )(*get_stale_counts()),
+                            desired_version=1,
+                        ),
+                        block_target=current_work.value['bits'].target,
+                        desired_timestamp=int(time.time() - current_work2.value['clock_offset']),
+                        desired_target=desired_share_target,
+                        ref_merkle_link=dict(branch=[], index=0),
+                        net=net,
+                    )
                 
                 target = net.PARENT.SANE_MAX_TARGET
                 if desired_pseudoshare_target is None:
@@ -527,10 +538,10 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                 
                 transactions = [generate_tx] + list(current_work2.value['transactions'])
                 packed_generate_tx = bitcoin_data.tx_type.pack(generate_tx)
-                merkle_root = bitcoin_data.check_merkle_branch(bitcoin_data.hash256(packed_generate_tx), 0, current_work2.value['merkle_branch'])
+                merkle_root = bitcoin_data.check_merkle_link(bitcoin_data.hash256(packed_generate_tx), current_work2.value['merkle_link'])
                 
                 getwork_time = time.time()
-                merkle_branch = current_work2.value['merkle_branch']
+                merkle_link = current_work2.value['merkle_link']
                 
                 print 'New work for worker! Difficulty: %.06f Share difficulty: %.06f Total block value: %.6f %s including %i transactions' % (
                     bitcoin_data.target_to_difficulty(target),
@@ -551,6 +562,7 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                 received_header_hashes = set()
                 
                 def got_response(header, request):
+                    user, _, _, _ = self.get_user_details(request)
                     assert header['merkle_root'] == merkle_root
                     
                     header_hash = bitcoin_data.hash256(bitcoin_data.block_header_type.pack(header))
@@ -564,7 +576,6 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                                 print
                                 print 'GOT BLOCK FROM MINER! Passing to bitcoind! %s%064x' % (net.PARENT.BLOCK_EXPLORER_URL_PREFIX, header_hash)
                                 print
-                                recent_blocks.append(dict(ts=time.time(), hash='%064x' % (header_hash,)))
                     except:
                         log.err(None, 'Error while processing potential block:')
                     
@@ -577,11 +588,9 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                                         merkle_tx=dict(
                                             tx=transactions[0],
                                             block_hash=header_hash,
-                                            merkle_branch=merkle_branch,
-                                            index=0,
+                                            merkle_link=merkle_link,
                                         ),
-                                        merkle_branch=bitcoin_data.calculate_merkle_branch(hashes, index),
-                                        index=index,
+                                        merkle_link=bitcoin_data.calculate_merkle_link(hashes, index),
                                         parent_block_header=header,
                                     )).encode('hex'),
                                 )
@@ -597,10 +606,13 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                         except:
                             log.err(None, 'Error while processing merged mining POW:')
                     
-                    if pow_hash <= share_info['bits'].target:
+                    if pow_hash <= share_info['bits'].target and header_hash not in received_header_hashes:
                         min_header = dict(header);del min_header['merkle_root']
                         hash_link = p2pool_data.prefix_to_hash_link(packed_generate_tx[:-32-4], p2pool_data.Share.gentx_before_refhash)
-                        share = p2pool_data.Share(net, None, min_header, share_info, hash_link=hash_link, merkle_branch=merkle_branch, other_txs=transactions[1:] if pow_hash <= header['bits'].target else None)
+                        share = p2pool_data.Share(net, None, dict(
+                            min_header=min_header, share_info=share_info, hash_link=hash_link,
+                            ref_merkle_link=dict(branch=[], index=0),
+                        ), merkle_link=merkle_link, other_txs=transactions[1:] if pow_hash <= header['bits'].target else None)
                         
                         print 'GOT SHARE! %s %s prev %s age %.2fs%s' % (
                             request.getUser(),
@@ -625,6 +637,8 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                                 shared_share_hashes.add(share.hash)
                         except:
                             log.err(None, 'Error forwarding block solution:')
+                        
+                        share_received.happened(bitcoin_data.target_to_average_attempts(share.target), not on_time)
                     
                     if pow_hash > target:
                         print 'Worker %s submitted share with hash > target:' % (request.getUser(),)
@@ -635,11 +649,11 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                     else:
                         received_header_hashes.add(header_hash)
                         
-                        pseudoshare_received.happened(bitcoin_data.target_to_average_attempts(target), not on_time, request.getUser() if request.getPassword() == vip_pass else None)
+                        pseudoshare_received.happened(bitcoin_data.target_to_average_attempts(target), not on_time, user)
                         self.recent_shares_ts_work.append((time.time(), bitcoin_data.target_to_average_attempts(target)))
                         while len(self.recent_shares_ts_work) > 50:
                             self.recent_shares_ts_work.pop(0)
-                        local_rate_monitor.add_datum(dict(work=bitcoin_data.target_to_average_attempts(target), dead=not on_time, user=request.getUser()))
+                        local_rate_monitor.add_datum(dict(work=bitcoin_data.target_to_average_attempts(target), dead=not on_time, user=user))
                     
                     return on_time
                 
@@ -647,8 +661,8 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
         
         get_current_txouts = lambda: p2pool_data.get_expected_payouts(tracker, current_work.value['best_share_hash'], current_work.value['bits'].target, current_work2.value['subsidy'], net)
         
-        web_root = web.get_web_root(tracker, current_work, current_work2, get_current_txouts, datadir_path, net, get_stale_counts, my_pubkey_hash, local_rate_monitor, args.worker_fee, p2p_node, my_share_hashes, recent_blocks, pseudoshare_received)
-        worker_interface.WorkerInterface(WorkerBridge()).attach_to(web_root)
+        web_root = web.get_web_root(tracker, current_work, current_work2, get_current_txouts, datadir_path, net, get_stale_counts, my_pubkey_hash, local_rate_monitor, args.worker_fee, p2p_node, my_share_hashes, pseudoshare_received, share_received)
+        worker_interface.WorkerInterface(WorkerBridge()).attach_to(web_root, get_handler=lambda request: request.redirect('/static/'))
         
         deferral.retry('Error binding to worker port:', traceback=False)(reactor.listenTCP)(worker_endpoint[1], server.Site(web_root), interface=worker_endpoint[0])
         
@@ -673,6 +687,14 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
         
         # done!
         print 'Started successfully!'
+        print 'Go to http://127.0.0.1:%i/ to view graphs and statistics!' % (worker_endpoint[1],)
+        if args.donation_percentage > 0.51:
+            print '''Donating %.1f%% of work towards P2Pool's development. Thanks for the tip!''' % (args.donation_percentage,)
+        elif args.donation_percentage < 0.49:
+            print '''Donating %.1f%% of work towards P2Pool's development. Please donate to encourage further development of P2Pool!''' % (args.donation_percentage,)
+        else:
+            print '''Donating %.1f%% of work towards P2Pool's development. Thank you!''' % (args.donation_percentage,)
+            print 'You can increase this amount with --give-author argument! (or decrease it, if you must)'
         print
         
         
@@ -687,25 +709,32 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
             from twisted.words.protocols import irc
             class IRCClient(irc.IRCClient):
                 nickname = 'p2pool%02i' % (random.randrange(100),)
-                channel = '#p2pool' if net.NAME == 'bitcoin' else '#p2pool-alt'
+                channel = net.ANNOUNCE_CHANNEL
                 def lineReceived(self, line):
-                    print repr(line)
+                    if p2pool.DEBUG:
+                        print repr(line)
                     irc.IRCClient.lineReceived(self, line)
                 def signedOn(self):
                     irc.IRCClient.signedOn(self)
                     self.factory.resetDelay()
                     self.join(self.channel)
-                    self.watch_id = tracker.verified.added.watch(self._new_share)
-                    self.announced_hashes = set()
-                    self.delayed_messages = {}
+                    @defer.inlineCallbacks
+                    def new_share(share):
+                        if share.pow_hash <= share.header['bits'].target and abs(share.timestamp - time.time()) < 10*60:
+                            yield deferral.sleep(random.expovariate(1/60))
+                            message = '\x02%s BLOCK FOUND by %s! %s%064x' % (net.NAME.upper(), bitcoin_data.script2_to_address(share.new_script, net.PARENT), net.PARENT.BLOCK_EXPLORER_URL_PREFIX, share.header_hash)
+                            if message not in self.recent_messages:
+                                self.say(self.channel, message)
+                                self._remember_message(message)
+                    self.watch_id = tracker.verified.added.watch(new_share)
+                    self.recent_messages = []
+                def _remember_message(self, message):
+                    self.recent_messages.append(message)
+                    while len(self.recent_messages) > 100:
+                        self.recent_messages.pop(0)
                 def privmsg(self, user, channel, message):
-                    if channel == self.channel and message in self.delayed_messages:
-                        self.delayed_messages.pop(message).cancel()
-                def _new_share(self, share):
-                    if share.pow_hash <= share.header['bits'].target and share.header_hash not in self.announced_hashes and abs(share.timestamp - time.time()) < 10*60:
-                        self.announced_hashes.add(share.header_hash)
-                        message = '\x02%s BLOCK FOUND by %s! %s%064x' % (net.NAME.upper(), bitcoin_data.script2_to_address(share.new_script, net.PARENT), net.PARENT.BLOCK_EXPLORER_URL_PREFIX, share.header_hash)
-                        self.delayed_messages[message] = reactor.callLater(random.expovariate(1/60), lambda: (self.say(self.channel, message), self.delayed_messages.pop(message)))
+                    if channel == self.channel:
+                        self._remember_message(message)
                 def connectionLost(self, reason):
                     tracker.verified.added.unwatch(self.watch_id)
                     print 'IRC connection lost:', reason.getErrorMessage()
@@ -757,6 +786,15 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                             100*stale_prop,
                             math.format_dt(2**256 / current_work.value['bits'].target / real_att_s),
                         )
+                        
+                        desired_version_counts = p2pool_data.get_desired_version_counts(tracker, current_work.value['best_share_hash'], min(720, height))
+                        majority_desired_version = max(desired_version_counts, key=lambda k: desired_version_counts[k])
+                        if majority_desired_version not in [0, 1]:
+                            print >>sys.stderr, '#'*40
+                            print >>sys.stderr, '>>> WARNING: A MAJORITY OF SHARES CONTAIN A VOTE FOR AN UNSUPPORTED SHARE IMPLEMENTATION! (v%i with %i%% support)' % (
+                                majority_desired_version, 100*desired_version_counts[majority_desired_version]/sum(desired_version_counts.itervalues()))
+                            print >>sys.stderr, '>>> An upgrade is likely necessary. Check http://p2pool.forre.st/ for more information.'
+                            print >>sys.stderr, '#'*40
                     
                     if this_str != last_str or time.time() > last_time + 15:
                         print this_str
@@ -766,8 +804,8 @@ def main(args, net, datadir_path, merged_urls, worker_endpoint):
                     log.err()
         status_thread()
     except:
-        log.err(None, 'Fatal error:')
         reactor.stop()
+        log.err(None, 'Fatal error:')
 
 def run():
     class FixedArgumentParser(argparse.ArgumentParser):
@@ -830,7 +868,7 @@ def run():
         help='call getauxblock on this url to get work for merged mining (example: http://ncuser:ncpass@127.0.0.1:10332/)',
         type=str, action='append', default=[], dest='merged_urls')
     parser.add_argument('--give-author', metavar='DONATION_PERCENTAGE',
-        help='donate this percentage of work to author of p2pool (default: 0.5)',
+        help='donate this percentage of work towards the development of p2pool (default: 0.5)',
         type=float, action='store', default=0.5, dest='donation_percentage')
     parser.add_argument('--irc-announce',
         help='announce any blocks found on irc://irc.freenode.net/#p2pool',
@@ -846,7 +884,7 @@ def run():
     parser.add_argument('--disable-upnp',
         help='''don't attempt to use UPnP to forward p2pool's P2P port from the Internet to this computer''',
         action='store_false', default=True, dest='upnp')
-    p2pool_group.add_argument('--max-conns', metavar='PORT',
+    p2pool_group.add_argument('--max-conns', metavar='CONNS',
         help='maximum incoming connections (default: 40)',
         type=int, action='store', default=40, dest='p2pool_conns')