From 01ee658a1026d877b588f21277ba785e2fb8812c Mon Sep 17 00:00:00 2001 From: forrest Date: Tue, 12 Jul 2011 07:03:48 +0000 Subject: [PATCH] speed optimizations for parsing git-svn-id: svn://forre.st/p2pool@1374 470744a7-cac9-478e-843e-5ec1b25c69e8 --- p2pool/bitcoin/data.py | 86 ++++++++++++++++++++++++++++-------------------- p2pool/bitcoin/p2p.py | 16 +++++--- p2pool/data.py | 71 +++++++++++++++------------------------ p2pool/util/math.py | 6 ++-- 4 files changed, 91 insertions(+), 88 deletions(-) diff --git a/p2pool/bitcoin/data.py b/p2pool/bitcoin/data.py index 551fef9..a16a920 100644 --- a/p2pool/bitcoin/data.py +++ b/p2pool/bitcoin/data.py @@ -1,7 +1,7 @@ from __future__ import division import struct -import StringIO +import cStringIO as StringIO import hashlib import warnings @@ -38,9 +38,10 @@ class Type(object): def unpack(self, data): obj = self._unpack(data) - data2 = self._pack(obj) - if data2 != data: - assert self._unpack(data2) == obj + if __debug__: + data2 = self._pack(obj) + if data2 != data: + assert self._unpack(data2) == obj return obj @@ -79,11 +80,17 @@ class VarIntType(Type): data = file.read(1) if len(data) != 1: raise EarlyEnd() - first, = struct.unpack('= 2**256: + if not 0 <= item < 2**256: raise ValueError("invalid hash value") if item != 0 and item < 2**160: - warnings.warn("very low hash value - maybe you meant to use ShortHashType?") + warnings.warn("very low hash value - maybe you meant to use ShortHashType? %x" % (item,)) file.write(('%064x' % (item,)).decode('hex')[::-1]) class ShortHashType(Type): @@ -173,18 +182,28 @@ class ShortHashType(Type): file.write(('%040x' % (item,)).decode('hex')[::-1]) class ListType(Type): + _inner_size = VarIntType() + def __init__(self, type): self.type = type def read(self, file): - length = VarIntType().read(file) + length = self._inner_size.read(file) return [self.type.read(file) for i in xrange(length)] def write(self, file, item): - VarIntType().write(file, len(item)) + self._inner_size.write(file, len(item)) for subitem in item: self.type.write(file, subitem) +class FastLittleEndianUnsignedInteger(Type): + def read(self, file): + data = map(ord, file.read(4)) + return data[0] + (data[1] << 8) + (data[2] << 16) + (data[3] << 24) + + def write(self, file, item): + StructType("> 24) - 3)) + assert target == self._bits_to_target1(struct.pack("> 24) - 3)) - def _target_to_bits(self, target, _check=True): n = bases.natural_to_string(target) if n and ord(n[0]) >= 128: n = "\x00" + n - bits = (chr(len(n)) + (n + 3*chr(0))[:3])[::-1] + bits2 = (chr(len(n)) + (n + 3*chr(0))[:3])[::-1] + bits = struct.unpack(" share self.reverse_shares = {} # previous_share_hash -> set of share_hashes @@ -385,8 +383,9 @@ if __name__ == '__main__': class OkayTracker(Tracker): def __init__(self, net): - Tracker.__init__(self, net) - self.verified = set() + Tracker.__init__(self) + self.net = net + self.verified = Tracker() """ self.okay_cache = {} # hash -> height @@ -418,25 +417,18 @@ class OkayTracker(Tracker): """ def think(self): desired = set() - best = None - best_score = 0 + + # for each overall head, attempt verification + # if it fails, attempt on parent, and repeat + # if no successful verification because of lack of parents, request parent for head in self.heads: - height, last = self.get_height_and_last(head) - #if height > 1:#, self.shares[head] - if last is not None and height < 2*self.net.CHAIN_LENGTH: - desired.add((random.choice(self.reverse_shares[last]).peers, last)) # XXX - continue - #first_to_verify = math.nth(self.get_chain(head), self.net.CHAIN_LENGTH - 1) - to_verify = [] - last_good_hash = None - for share_hash in self.get_chain_known(head): - if share_hash in self.verified: - last_good = share_hash + head_height, last = self.get_height_and_last(head) + if head_height < a and last is not None: + # request more + + for share in itertools.islice(self.get_chain_known(head), None if last is None else head_height - self.net.CHAIN_LENGTH): # XXX change length for None + in share in self.verified.shares: break - to_verify.append(share_hash) - # recheck for 2*chain_length - for share_hash in reversed(to_verify): - share = self.shares[share_hash] try: share.check(self, self.net) except: @@ -444,31 +436,24 @@ class OkayTracker(Tracker): print "Share check failed:" traceback.print_exc() print + else: + self.verified.add_share(share_hash) break - self.verified.add(share_hash) - last_good_hash = share_hash - - new_height, last = self.get_height_and_last(last_good_hash) - - print head, height, last, new_height - score = new_height - - if score > best_score: - best = head - best_score = score - print "BEST", best + # try to get at least CHAIN_LENGTH height for each verified head, requesting parents if needed + for head in self.verified.heads: + head_height, last = self.verified.get_height_and_last(head) + a + + # decide best verified head + def score(share_hash): + share = self.verified.shares[share_hash] + head_height, last = self.verified.get_height_and_last(share) + return (min(head_height, net.CHAIN_LENGTH), RECENTNESS) + best = max(self.verified.heads, key=score) + return best, desired -class Chain(object): - def __init__(self): - pass - -def get_chain_descriptor(tracker, start): - for item in tracker.get_chain(self.net.CHAIN_LENGTH): - a - pass - class Mainnet(bitcoin_data.Mainnet): SHARE_PERIOD = 5 # seconds diff --git a/p2pool/util/math.py b/p2pool/util/math.py index 67a883d..19f77bd 100644 --- a/p2pool/util/math.py +++ b/p2pool/util/math.py @@ -18,9 +18,9 @@ def shuffled(x): def shift_left(n, m): # python: :( - if m < 0: - return n >> -m - return n << m + if m >= 0: + return n << m + return n >> -m def clip(x, (low, high)): if x < low: -- 1.7.1