unified HashType, ShortHashType, and StructType into IntType

diff --git a/p2pool/bitcoin/data.py b/p2pool/bitcoin/data.py
index 6bf8a63..8cf730f 100644 (file)
--- a/p2pool/bitcoin/data.py
+++ b/p2pool/bitcoin/data.py
@@ -91,14 +91,14 @@ class Type(object):
     
     
     def hash160(self, obj):
-        return ShortHashType().unpack(hashlib.new('ripemd160', hashlib.sha256(self.pack(obj)).digest()).digest())
+        return IntType(160).unpack(hashlib.new('ripemd160', hashlib.sha256(self.pack(obj)).digest()).digest())
     
     def hash256(self, obj):
-        return HashType().unpack(hashlib.sha256(hashlib.sha256(self.pack(obj)).digest()).digest())
+        return IntType(256).unpack(hashlib.sha256(hashlib.sha256(self.pack(obj)).digest()).digest())
     
     def scrypt(self, obj):
         import ltc_scrypt
-        return HashType().unpack(ltc_scrypt.getPoWHash(self.pack(obj)))
+        return IntType(256).unpack(ltc_scrypt.getPoWHash(self.pack(obj)))
 
 class VarIntType(Type):
     # redundancy doesn't matter here because bitcoin and p2pool both reencode before hashing
@@ -171,28 +171,6 @@ class EnumType(Type):
             raise ValueError('enum item (%r) not in values (%r)' % (item, self.values))
         return self.inner.write(file, self.values[item])
 
-class HashType(Type):
-    def read(self, file):
-        data, file = read(file, 256//8)
-        return int(data[::-1].encode('hex'), 16), file
-    
-    def write(self, file, item):
-        if not 0 <= item < 2**256:
-            raise ValueError('invalid hash value - %r' % (item,))
-        if item != 0 and item < 2**160:
-            print 'Very low hash value - maybe you meant to use ShortHashType? %x' % (item,)
-        return file, ('%064x' % (item,)).decode('hex')[::-1]
-
-class ShortHashType(Type):
-    def read(self, file):
-        data, file = read(file, 160//8)
-        return int(data[::-1].encode('hex'), 16), file
-    
-    def write(self, file, item):
-        if not 0 <= item < 2**160:
-            raise ValueError('invalid hash value - %r' % (item,))
-        return file, ('%040x' % (item,)).decode('hex')[::-1]
-
 class ListType(Type):
     _inner_size = VarIntType()
     
@@ -220,15 +198,34 @@ class StructType(Type):
     
     def read(self, file):
         data, file = read(file, self.length)
-        res, = struct.unpack(self.desc, data)
-        return res, file
+        return struct.unpack(self.desc, data)[0], file
     
     def write(self, file, item):
-        data = struct.pack(self.desc, item)
-        if struct.unpack(self.desc, data)[0] != item:
-            # special test because struct doesn't error on some overflows
-            raise ValueError('''item didn't survive pack cycle (%r)''' % (item,))
-        return file, data
+        return file, struct.pack(self.desc, item)
+
+class IntType(Type):
+    def __new__(cls, bits, endianness='little'):
+        assert bits % 8 == 0
+        assert endianness in ['little', 'big']
+        if bits in [8, 16, 32, 64]:
+            return StructType(('<' if endianness == 'little' else '>') + {8: 'B', 16: 'H', 32: 'I', 64: 'Q'}[bits])
+        else:
+            return object.__new__(cls, bits, endianness)
+    
+    def __init__(self, bits, endianness='little'):
+        assert bits % 8 == 0
+        assert endianness in ['little', 'big']
+        self.bytes = bits//8
+        self.step = -1 if endianness == 'little' else 1
+    
+    def read(self, file):
+        data, file = read(file, self.bytes)
+        return int(data[::self.step].encode('hex'), 16), file
+    
+    def write(self, file, item):
+        if not 0 <= item < 2**(8*self.bytes):
+            raise ValueError('invalid int value - %r' % (item,))
+        return file, ('%x' % (item,)).zfill(2*self.bytes).decode('hex')[::self.step]
 
 class IPV6AddressType(Type):
     def read(self, file):
@@ -350,7 +347,7 @@ class FloatingInteger(object):
         return 'FloatingInteger(bits=%s, target=%s)' % (hex(self.bits), hex(self.target))
 
 class FloatingIntegerType(Type):
-    _inner = StructType('<I')
+    _inner = IntType(32)
     
     def read(self, file):
         bits, file = self._inner.read(file)
@@ -374,44 +371,44 @@ class PossiblyNoneType(Type):
         return self.inner.write(file, self.none_value if item is None else item)
 
 address_type = ComposedType([
-    ('services', StructType('<Q')),
+    ('services', IntType(64)),
     ('address', IPV6AddressType()),
-    ('port', StructType('>H')),
+    ('port', IntType(16, 'big')),
 ])
 
 tx_type = ComposedType([
-    ('version', StructType('<I')),
+    ('version', IntType(32)),
     ('tx_ins', ListType(ComposedType([
         ('previous_output', PossiblyNoneType(dict(hash=0, index=2**32 - 1), ComposedType([
-            ('hash', HashType()),
-            ('index', StructType('<I')),
+            ('hash', IntType(256)),
+            ('index', IntType(32)),
         ]))),
         ('script', VarStrType()),
-        ('sequence', PossiblyNoneType(2**32 - 1, StructType('<I'))),
+        ('sequence', PossiblyNoneType(2**32 - 1, IntType(32))),
     ]))),
     ('tx_outs', ListType(ComposedType([
-        ('value', StructType('<Q')),
+        ('value', IntType(64)),
         ('script', VarStrType()),
     ]))),
-    ('lock_time', StructType('<I')),
+    ('lock_time', IntType(32)),
 ])
 
-merkle_branch_type = ListType(HashType())
+merkle_branch_type = ListType(IntType(256))
 
 merkle_tx_type = ComposedType([
     ('tx', tx_type),
-    ('block_hash', HashType()),
+    ('block_hash', IntType(256)),
     ('merkle_branch', merkle_branch_type),
-    ('index', StructType('<i')),
+    ('index', IntType(32)),
 ])
 
 block_header_type = ComposedType([
-    ('version', StructType('<I')),
-    ('previous_block', PossiblyNoneType(0, HashType())),
-    ('merkle_root', HashType()),
-    ('timestamp', StructType('<I')),
+    ('version', IntType(32)),
+    ('previous_block', PossiblyNoneType(0, IntType(256))),
+    ('merkle_root', IntType(256)),
+    ('timestamp', IntType(32)),
     ('bits', FloatingIntegerType()),
-    ('nonce', StructType('<I')),
+    ('nonce', IntType(32)),
 ])
 
 block_type = ComposedType([
@@ -422,14 +419,14 @@ block_type = ComposedType([
 aux_pow_type = ComposedType([
     ('merkle_tx', merkle_tx_type),
     ('merkle_branch', merkle_branch_type),
-    ('index', StructType('<i')),
+    ('index', IntType(32)),
     ('parent_block_header', block_header_type),
 ])
 
 
 merkle_record_type = ComposedType([
-    ('left', HashType()),
-    ('right', HashType()),
+    ('left', IntType(256)),
+    ('right', IntType(256)),
 ])
 
 def merkle_hash(hashes):
@@ -485,8 +482,8 @@ def tx_get_sigop_count(tx):
 # human addresses
 
 human_address_type = ChecksummedType(ComposedType([
-    ('version', StructType('<B')),
-    ('pubkey_hash', ShortHashType()),
+    ('version', IntType(8)),
+    ('pubkey_hash', IntType(160)),
 ]))
 
 pubkey_type = PassthruType()
@@ -509,7 +506,7 @@ def pubkey_to_script2(pubkey):
     return ('\x41' + pubkey_type.pack(pubkey)) + '\xac'
 
 def pubkey_hash_to_script2(pubkey_hash):
-    return '\x76\xa9' + ('\x14' + ShortHashType().pack(pubkey_hash)) + '\x88\xac'
+    return '\x76\xa9' + ('\x14' + IntType(160).pack(pubkey_hash)) + '\x88\xac'
 
 def script2_to_address(script2, net):
     try:
@@ -522,7 +519,7 @@ def script2_to_address(script2, net):
             return pubkey_to_address(pubkey, net)
     
     try:
-        pubkey_hash = ShortHashType().unpack(script2[3:-2])
+        pubkey_hash = IntType(160).unpack(script2[3:-2])
         script2_test2 = pubkey_hash_to_script2(pubkey_hash)
     except:
         pass
@@ -541,7 +538,7 @@ def script2_to_human(script2, net):
             return 'Pubkey. Address: %s' % (pubkey_to_address(pubkey, net),)
     
     try:
-        pubkey_hash = ShortHashType().unpack(script2[3:-2])
+        pubkey_hash = IntType(160).unpack(script2[3:-2])
         script2_test2 = pubkey_hash_to_script2(pubkey_hash)
     except:
         pass

diff --git a/p2pool/bitcoin/getwork.py b/p2pool/bitcoin/getwork.py
index 9beefd8..f3e9bf1 100644 (file)
--- a/p2pool/bitcoin/getwork.py
+++ b/p2pool/bitcoin/getwork.py
@@ -46,7 +46,7 @@ class BlockAttempt(object):
         getwork = {
             'data': _swap4(block_data).encode('hex') + '000000800000000000000000000000000000000000000000000000000000000000000000000000000000000080020000',
             'hash1': '00000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000010000',
-            'target': bitcoin_data.HashType().pack(self.share_target).encode('hex'),
+            'target': bitcoin_data.IntType(256).pack(self.share_target).encode('hex'),
             'midstate': _swap4(sha256.process(block_data[:64])).encode('hex'),
         }
         
@@ -70,7 +70,7 @@ class BlockAttempt(object):
             merkle_root=attrs['merkle_root'],
             timestamp=attrs['timestamp'],
             bits=attrs['bits'],
-            share_target=bitcoin_data.HashType().unpack(getwork['target'].decode('hex')),
+            share_target=bitcoin_data.IntType(256).unpack(getwork['target'].decode('hex')),
         )
         
         if _check:

diff --git a/p2pool/bitcoin/p2p.py b/p2pool/bitcoin/p2p.py
index 6692f2b..c0f4680 100644 (file)
--- a/p2pool/bitcoin/p2p.py
+++ b/p2pool/bitcoin/p2p.py
@@ -136,14 +136,14 @@ class Protocol(BaseProtocol):
         )
     
     message_version = bitcoin_data.ComposedType([
-        ('version', bitcoin_data.StructType('<I')),
-        ('services', bitcoin_data.StructType('<Q')),
-        ('time', bitcoin_data.StructType('<Q')),
+        ('version', bitcoin_data.IntType(32)),
+        ('services', bitcoin_data.IntType(64)),
+        ('time', bitcoin_data.IntType(64)),
         ('addr_to', bitcoin_data.address_type),
         ('addr_from', bitcoin_data.address_type),
-        ('nonce', bitcoin_data.StructType('<Q')),
+        ('nonce', bitcoin_data.IntType(64)),
         ('sub_version_num', bitcoin_data.VarStrType()),
-        ('start_height', bitcoin_data.StructType('<I')),
+        ('start_height', bitcoin_data.IntType(32)),
     ])
     def handle_version(self, version, services, time, addr_to, addr_from, nonce, sub_version_num, start_height):
         #print 'VERSION', locals()
@@ -168,8 +168,8 @@ class Protocol(BaseProtocol):
     
     message_inv = bitcoin_data.ComposedType([
         ('invs', bitcoin_data.ListType(bitcoin_data.ComposedType([
-            ('type', bitcoin_data.EnumType(bitcoin_data.StructType('<I'), {'tx': 1, 'block': 2})),
-            ('hash', bitcoin_data.HashType()),
+            ('type', bitcoin_data.EnumType(bitcoin_data.IntType(32), {'tx': 1, 'block': 2})),
+            ('hash', bitcoin_data.IntType(256)),
         ]))),
     ])
     def handle_inv(self, invs):
@@ -183,25 +183,25 @@ class Protocol(BaseProtocol):
     
     message_getdata = bitcoin_data.ComposedType([
         ('requests', bitcoin_data.ListType(bitcoin_data.ComposedType([
-            ('type', bitcoin_data.EnumType(bitcoin_data.StructType('<I'), {'tx': 1, 'block': 2})),
-            ('hash', bitcoin_data.HashType()),
+            ('type', bitcoin_data.EnumType(bitcoin_data.IntType(32), {'tx': 1, 'block': 2})),
+            ('hash', bitcoin_data.IntType(256)),
         ]))),
     ])
     message_getblocks = bitcoin_data.ComposedType([
-        ('version', bitcoin_data.StructType('<I')),
-        ('have', bitcoin_data.ListType(bitcoin_data.HashType())),
-        ('last', bitcoin_data.PossiblyNoneType(0, bitcoin_data.HashType())),
+        ('version', bitcoin_data.IntType(32)),
+        ('have', bitcoin_data.ListType(bitcoin_data.IntType(256))),
+        ('last', bitcoin_data.PossiblyNoneType(0, bitcoin_data.IntType(256))),
     ])
     message_getheaders = bitcoin_data.ComposedType([
-        ('version', bitcoin_data.StructType('<I')),
-        ('have', bitcoin_data.ListType(bitcoin_data.HashType())),
-        ('last', bitcoin_data.PossiblyNoneType(0, bitcoin_data.HashType())),
+        ('version', bitcoin_data.IntType(32)),
+        ('have', bitcoin_data.ListType(bitcoin_data.IntType(256))),
+        ('last', bitcoin_data.PossiblyNoneType(0, bitcoin_data.IntType(256))),
     ])
     message_getaddr = bitcoin_data.ComposedType([])
     
     message_addr = bitcoin_data.ComposedType([
         ('addrs', bitcoin_data.ListType(bitcoin_data.ComposedType([
-            ('timestamp', bitcoin_data.StructType('<I')),
+            ('timestamp', bitcoin_data.IntType(32)),
             ('address', bitcoin_data.address_type),
         ]))),
     ])
@@ -233,8 +233,8 @@ class Protocol(BaseProtocol):
         self.factory.new_headers.happened([header['header'] for header in headers])
     
     message_reply = bitcoin_data.ComposedType([
-        ('hash', bitcoin_data.HashType()),
-        ('reply',  bitcoin_data.EnumType(bitcoin_data.StructType('<I'), {'success': 0, 'failure': 1, 'denied': 2})),
+        ('hash', bitcoin_data.IntType(256)),
+        ('reply',  bitcoin_data.EnumType(bitcoin_data.IntType(32), {'success': 0, 'failure': 1, 'denied': 2})),
         ('script', bitcoin_data.PossiblyNoneType('', bitcoin_data.VarStrType())),
     ])
     

diff --git a/p2pool/data.py b/p2pool/data.py
index 2d16871..8082daf 100644 (file)
--- a/p2pool/data.py
+++ b/p2pool/data.py
@@ -13,19 +13,19 @@ from p2pool.util import math, forest
 
 
 share_data_type = bitcoin_data.ComposedType([
-    ('previous_share_hash', bitcoin_data.PossiblyNoneType(0, bitcoin_data.HashType())),
+    ('previous_share_hash', bitcoin_data.PossiblyNoneType(0, bitcoin_data.IntType(256))),
     ('coinbase', bitcoin_data.VarStrType()),
     ('nonce', bitcoin_data.VarStrType()),
     ('new_script', bitcoin_data.VarStrType()),
-    ('subsidy', bitcoin_data.StructType('<Q')),
-    ('donation', bitcoin_data.StructType('<H')),
-    ('stale_info', bitcoin_data.StructType('<B')), # 0 nothing, 253 orphan, 254 doa. previously: 254*perfect_round(my_stale_prop), None if no shares
+    ('subsidy', bitcoin_data.IntType(64)),
+    ('donation', bitcoin_data.IntType(16)),
+    ('stale_info', bitcoin_data.IntType(8)), # 0 nothing, 253 orphan, 254 doa. previously: 254*perfect_round(my_stale_prop), None if no shares
 ])
 
 share_info_type = bitcoin_data.ComposedType([
     ('share_data', share_data_type),
     ('bits', bitcoin_data.FloatingIntegerType()),
-    ('timestamp', bitcoin_data.StructType('<I')),
+    ('timestamp', bitcoin_data.IntType(32)),
 ])
 
 share1a_type = bitcoin_data.ComposedType([
@@ -260,7 +260,7 @@ def generate_transaction(tracker, share_data, block_target, desired_timestamp, n
             sequence=None,
             script=share_data['coinbase'].ljust(2, '\x00'),
         )],
-        tx_outs=[dict(value=0, script='\x20' + bitcoin_data.HashType().pack(share_info_type.hash256(share_info)))] + [dict(value=amounts[script], script=script) for script in dests if amounts[script]],
+        tx_outs=[dict(value=0, script='\x20' + bitcoin_data.IntType(256).pack(share_info_type.hash256(share_info)))] + [dict(value=amounts[script], script=script) for script in dests if amounts[script]],
         lock_time=0,
     )
 

diff --git a/p2pool/main.py b/p2pool/main.py
index 2df3111..3b77909 100644 (file)
--- a/p2pool/main.py
+++ b/p2pool/main.py
@@ -204,7 +204,7 @@ def main(args, net, datadir_path):
                 auxblock = yield deferral.retry('Error while calling merged getauxblock:', 1)(merged_proxy.rpc_getauxblock)()
                 pre_merged_work.set(dict(
                     hash=int(auxblock['hash'], 16),
-                    target=bitcoin_data.HashType().unpack(auxblock['target'].decode('hex')),
+                    target=bitcoin_data.IntType(256).unpack(auxblock['target'].decode('hex')),
                     chain_id=auxblock['chainid'],
                 ))
                 yield deferral.sleep(1)
@@ -445,7 +445,7 @@ def main(args, net, datadir_path):
                     share_data=dict(
                         previous_share_hash=current_work.value['best_share_hash'],
                         coinbase=(('' if current_work.value['aux_work'] is None else
-                            '\xfa\xbemm' + bitcoin_data.HashType().pack(current_work.value['aux_work']['hash'])[::-1] + struct.pack('<ii', 1, 0)) + current_work.value['coinbaseflags'])[:100],
+                            '\xfa\xbemm' + bitcoin_data.IntType(256).pack(current_work.value['aux_work']['hash'])[::-1] + struct.pack('<ii', 1, 0)) + current_work.value['coinbaseflags'])[:100],
                         nonce=struct.pack('<Q', random.randrange(2**64)),
                         new_script=payout_script,
                         subsidy=current_work2.value['subsidy'],
@@ -516,9 +516,9 @@ def main(args, net, datadir_path):
                 
                 try:
                     if aux_work is not None and (pow_hash <= aux_work['target'] or p2pool.DEBUG):
-                        assert bitcoin_data.HashType().pack(aux_work['hash'])[::-1].encode('hex') == transactions[0]['tx_ins'][0]['script'][-32-8:-8].encode('hex')
+                        assert bitcoin_data.IntType(256).pack(aux_work['hash'])[::-1].encode('hex') == transactions[0]['tx_ins'][0]['script'][-32-8:-8].encode('hex')
                         df = deferral.retry('Error submitting merged block: (will retry)', 10, 10)(merged_proxy.rpc_getauxblock)(
-                            bitcoin_data.HashType().pack(aux_work['hash'])[::-1].encode('hex'),
+                            bitcoin_data.IntType(256).pack(aux_work['hash'])[::-1].encode('hex'),
                             bitcoin_data.aux_pow_type.pack(dict(
                                 merkle_tx=dict(
                                     tx=transactions[0],

diff --git a/p2pool/p2p.py b/p2pool/p2p.py
index eca642f..ef289fa 100644 (file)
--- a/p2pool/p2p.py
+++ b/p2pool/p2p.py
@@ -92,14 +92,14 @@ class Protocol(bitcoin_p2p.BaseProtocol):
             yield deferral.sleep(random.expovariate(1/(100*len(self.node.peers) + 1)))
     
     message_version = bitcoin_data.ComposedType([
-        ('version', bitcoin_data.StructType('<I')),
-        ('services', bitcoin_data.StructType('<Q')),
+        ('version', bitcoin_data.IntType(32)),
+        ('services', bitcoin_data.IntType(64)),
         ('addr_to', bitcoin_data.address_type),
         ('addr_from', bitcoin_data.address_type),
-        ('nonce', bitcoin_data.StructType('<Q')),
+        ('nonce', bitcoin_data.IntType(64)),
         ('sub_version', bitcoin_data.VarStrType()),
-        ('mode', bitcoin_data.StructType('<I')), # always 1 for legacy compatibility
-        ('best_share_hash', bitcoin_data.PossiblyNoneType(0, bitcoin_data.HashType())),
+        ('mode', bitcoin_data.IntType(32)), # always 1 for legacy compatibility
+        ('best_share_hash', bitcoin_data.PossiblyNoneType(0, bitcoin_data.IntType(256))),
     ])
     def handle_version(self, version, services, addr_to, addr_from, nonce, sub_version, mode, best_share_hash):
         if self.other_version is not None or version < 2:
@@ -134,7 +134,7 @@ class Protocol(bitcoin_p2p.BaseProtocol):
         pass
     
     message_addrme = bitcoin_data.ComposedType([
-        ('port', bitcoin_data.StructType('<H')),
+        ('port', bitcoin_data.IntType(16)),
     ])
     def handle_addrme(self, port):
         host = self.transport.getPeer().host
@@ -158,7 +158,7 @@ class Protocol(bitcoin_p2p.BaseProtocol):
     
     message_addrs = bitcoin_data.ComposedType([
         ('addrs', bitcoin_data.ListType(bitcoin_data.ComposedType([
-            ('timestamp', bitcoin_data.StructType('<Q')),
+            ('timestamp', bitcoin_data.IntType(64)),
             ('address', bitcoin_data.address_type),
         ]))),
     ])
@@ -169,7 +169,7 @@ class Protocol(bitcoin_p2p.BaseProtocol):
                 random.choice(self.node.peers.values()).send_addrs(addrs=[addr_record])
     
     message_getaddrs = bitcoin_data.ComposedType([
-        ('count', bitcoin_data.StructType('<I')),
+        ('count', bitcoin_data.IntType(32)),
     ])
     def handle_getaddrs(self, count):
         if count > 100:
@@ -187,9 +187,9 @@ class Protocol(bitcoin_p2p.BaseProtocol):
         ])
     
     message_getshares = bitcoin_data.ComposedType([
-        ('hashes', bitcoin_data.ListType(bitcoin_data.HashType())),
+        ('hashes', bitcoin_data.ListType(bitcoin_data.IntType(256))),
         ('parents', bitcoin_data.VarIntType()),
-        ('stops', bitcoin_data.ListType(bitcoin_data.HashType())),
+        ('stops', bitcoin_data.ListType(bitcoin_data.IntType(256))),
     ])
     def handle_getshares(self, hashes, parents, stops):
         self.node.handle_get_shares(hashes, parents, stops, self)

diff --git a/p2pool/test/bitcoin/test_data.py b/p2pool/test/bitcoin/test_data.py
index a29c965..4f503a0 100644 (file)
--- a/p2pool/test/bitcoin/test_data.py
+++ b/p2pool/test/bitcoin/test_data.py
@@ -24,13 +24,13 @@ class Test(unittest.TestCase):
             )],
             tx_outs=[dict(
                 value=5003880250,
-                script=data.pubkey_hash_to_script2(data.ShortHashType().unpack('ca975b00a8c203b8692f5a18d92dc5c2d2ebc57b'.decode('hex'))),
+                script=data.pubkey_hash_to_script2(data.IntType(160).unpack('ca975b00a8c203b8692f5a18d92dc5c2d2ebc57b'.decode('hex'))),
             )],
             lock_time=0,
         )) == 0xb53802b2333e828d6532059f46ecf6b313a42d79f97925e457fbbfda45367e5c
     
     def test_address_to_pubkey_hash(self):
-        assert data.address_to_pubkey_hash('1KUCp7YP5FP8ViRxhfszSUJCTAajK6viGy', networks.BitcoinMainnet) == data.ShortHashType().unpack('ca975b00a8c203b8692f5a18d92dc5c2d2ebc57b'.decode('hex'))
+        assert data.address_to_pubkey_hash('1KUCp7YP5FP8ViRxhfszSUJCTAajK6viGy', networks.BitcoinMainnet) == data.IntType(160).unpack('ca975b00a8c203b8692f5a18d92dc5c2d2ebc57b'.decode('hex'))
     
     def test_merkle_hash(self):
         assert data.merkle_hash([