From: Forrest Voight Date: Fri, 27 Jan 2012 15:18:05 +0000 (-0500) Subject: moved generic data types to util.pack X-Git-Tag: 0.8.3~45 X-Git-Url: https://git.novaco.in/?a=commitdiff_plain;h=5b82bccbad62297135345eb4b69c3e75399aaf86;p=p2pool.git moved generic data types to util.pack --- diff --git a/p2pool/bitcoin/data.py b/p2pool/bitcoin/data.py index e47e291..3c5cf6d 100644 --- a/p2pool/bitcoin/data.py +++ b/p2pool/bitcoin/data.py @@ -1,303 +1,10 @@ from __future__ import division -import binascii import hashlib -import struct -from . import base58 -from p2pool.util import bases, math -import p2pool +from p2pool.util import bases, math, pack -class EarlyEnd(Exception): - pass - -class LateEnd(Exception): - pass - -def read((data, pos), length): - data2 = data[pos:pos + length] - if len(data2) != length: - raise EarlyEnd() - return data2, (data, pos + length) - -def size((data, pos)): - return len(data) - pos - -class Type(object): - __slots__ = [] - - # the same data can have only one unpacked representation, but multiple packed binary representations - - def __hash__(self): - rval = getattr(self, '_hash', None) - if rval is None: - try: - rval = self._hash = hash((type(self), frozenset(self.__dict__.items()))) - except: - print self.__dict__ - raise - return rval - - def __eq__(self, other): - return type(other) is type(self) and other.__dict__ == self.__dict__ - - def __ne__(self, other): - return not (self == other) - - def _unpack(self, data): - obj, (data2, pos) = self.read((data, 0)) - - assert data2 is data - - if pos != len(data): - raise LateEnd() - - return obj - - def _pack(self, obj): - f = self.write(None, obj) - - res = [] - while f is not None: - res.append(f[1]) - f = f[0] - res.reverse() - return ''.join(res) - - - def unpack(self, data): - obj = self._unpack(data) - - if p2pool.DEBUG: - data2 = self._pack(obj) - if data2 != data: - if self._unpack(data2) != obj: - raise AssertionError() - - return obj - - def pack(self, obj): - data = self._pack(obj) - - if p2pool.DEBUG: - if self._unpack(data) != obj: - raise AssertionError((self._unpack(data), obj)) - - return data - - - def pack_base58(self, obj): - return base58.encode(self.pack(obj)) - - def unpack_base58(self, base58_data): - return self.unpack(base58.decode(base58_data)) - - - def hash160(self, obj): - return IntType(160).unpack(hashlib.new('ripemd160', hashlib.sha256(self.pack(obj)).digest()).digest()) - - def hash256(self, obj): - return IntType(256).unpack(hashlib.sha256(hashlib.sha256(self.pack(obj)).digest()).digest()) - - def scrypt(self, obj): - import ltc_scrypt - return IntType(256).unpack(ltc_scrypt.getPoWHash(self.pack(obj))) - -class VarIntType(Type): - # redundancy doesn't matter here because bitcoin and p2pool both reencode before hashing - def read(self, file): - data, file = read(file, 1) - first = ord(data) - if first < 0xfd: - return first, file - elif first == 0xfd: - desc, length = '') + {8: 'B', 16: 'H', 32: 'I', 64: 'Q'}[bits]) - else: - return Type.__new__(cls, bits, endianness) - - def __init__(self, bits, endianness='little'): - assert bits % 8 == 0 - assert endianness in ['little', 'big'] - self.bytes = bits//8 - self.step = -1 if endianness == 'little' else 1 - self.format_str = '%%0%ix' % (2*self.bytes) - self.max = 2**bits - - def read(self, file, b2a_hex=binascii.b2a_hex): - data, file = read(file, self.bytes) - return int(b2a_hex(data[::self.step]), 16), file - - def write(self, file, item, a2b_hex=binascii.a2b_hex): - if not 0 <= item < self.max: - raise ValueError('invalid int value - %r' % (item,)) - return file, a2b_hex(self.format_str % (item,))[::self.step] - -class IPV6AddressType(Type): - def read(self, file): - data, file = read(file, 16) - if data[:12] != '00000000000000000000ffff'.decode('hex'): - raise ValueError('ipv6 addresses not supported yet') - return '.'.join(str(ord(x)) for x in data[12:]), file - - def write(self, file, item): - bits = map(int, item.split('.')) - if len(bits) != 4: - raise ValueError('invalid address: %r' % (bits,)) - data = '00000000000000000000ffff'.decode('hex') + ''.join(chr(x) for x in bits) - assert len(data) == 16, len(data) - return file, data - -_record_types = {} - -def get_record(fields): - fields = tuple(sorted(fields)) - if 'keys' in fields: - raise ValueError() - if fields not in _record_types: - class _Record(object): - __slots__ = fields - def __repr__(self): - return repr(dict(self)) - def __getitem__(self, key): - return getattr(self, key) - def __setitem__(self, key, value): - setattr(self, key, value) - #def __iter__(self): - # for field in self.__slots__: - # yield field, getattr(self, field) - def keys(self): - return self.__slots__ - def __eq__(self, other): - if isinstance(other, dict): - return dict(self) == other - elif isinstance(other, _Record): - return all(self[k] == other[k] for k in self.keys()) - raise TypeError() - def __ne__(self, other): - return not (self == other) - _record_types[fields] = _Record - return _record_types[fields]() - -class ComposedType(Type): - def __init__(self, fields): - self.fields = tuple(fields) - - def read(self, file): - item = get_record(k for k, v in self.fields) - for key, type_ in self.fields: - item[key], file = type_.read(file) - return item, file - - def write(self, file, item): - for key, type_ in self.fields: - file = type_.write(file, item[key]) - return file - -class ChecksummedType(Type): +class ChecksummedType(pack.Type): def __init__(self, inner): self.inner = inner @@ -305,7 +12,7 @@ class ChecksummedType(Type): obj, file = self.inner.read(file) data = self.inner.pack(obj) - checksum, file = read(file, 4) + checksum, file = pack.read(file, 4) if checksum != hashlib.sha256(hashlib.sha256(data).digest()).digest()[:4]: raise ValueError('invalid checksum') @@ -324,7 +31,7 @@ class FloatingInteger(object): if n and ord(n[0]) >= 128: n = '\x00' + n bits2 = (chr(len(n)) + (n + 3*chr(0))[:3])[::-1] - bits = struct.unpack(' 100: @@ -186,16 +186,16 @@ class Protocol(bitcoin_p2p.BaseProtocol): self.node.get_good_peers(count) ]) - message_getshares = bitcoin_data.ComposedType([ - ('hashes', bitcoin_data.ListType(bitcoin_data.IntType(256))), - ('parents', bitcoin_data.VarIntType()), - ('stops', bitcoin_data.ListType(bitcoin_data.IntType(256))), + message_getshares = pack.ComposedType([ + ('hashes', pack.ListType(pack.IntType(256))), + ('parents', pack.VarIntType()), + ('stops', pack.ListType(pack.IntType(256))), ]) def handle_getshares(self, hashes, parents, stops): self.node.handle_get_shares(hashes, parents, stops, self) - message_shares = bitcoin_data.ComposedType([ - ('shares', bitcoin_data.ListType(p2pool_data.share_type)), + message_shares = pack.ComposedType([ + ('shares', pack.ListType(p2pool_data.share_type)), ]) def handle_shares(self, shares): res = [] diff --git a/p2pool/test/bitcoin/test_data.py b/p2pool/test/bitcoin/test_data.py index 4f503a0..c98a7ca 100644 --- a/p2pool/test/bitcoin/test_data.py +++ b/p2pool/test/bitcoin/test_data.py @@ -1,6 +1,7 @@ import unittest from p2pool.bitcoin import data, networks +from p2pool.util import pack class Test(unittest.TestCase): @@ -24,13 +25,13 @@ class Test(unittest.TestCase): )], tx_outs=[dict( value=5003880250, - script=data.pubkey_hash_to_script2(data.IntType(160).unpack('ca975b00a8c203b8692f5a18d92dc5c2d2ebc57b'.decode('hex'))), + script=data.pubkey_hash_to_script2(pack.IntType(160).unpack('ca975b00a8c203b8692f5a18d92dc5c2d2ebc57b'.decode('hex'))), )], lock_time=0, )) == 0xb53802b2333e828d6532059f46ecf6b313a42d79f97925e457fbbfda45367e5c def test_address_to_pubkey_hash(self): - assert data.address_to_pubkey_hash('1KUCp7YP5FP8ViRxhfszSUJCTAajK6viGy', networks.BitcoinMainnet) == data.IntType(160).unpack('ca975b00a8c203b8692f5a18d92dc5c2d2ebc57b'.decode('hex')) + assert data.address_to_pubkey_hash('1KUCp7YP5FP8ViRxhfszSUJCTAajK6viGy', networks.BitcoinMainnet) == pack.IntType(160).unpack('ca975b00a8c203b8692f5a18d92dc5c2d2ebc57b'.decode('hex')) def test_merkle_hash(self): assert data.merkle_hash([ diff --git a/p2pool/util/pack.py b/p2pool/util/pack.py new file mode 100644 index 0000000..69cbc7e --- /dev/null +++ b/p2pool/util/pack.py @@ -0,0 +1,309 @@ +import binascii +import hashlib +import struct + +from p2pool.bitcoin import base58 +import p2pool + +class EarlyEnd(Exception): + pass + +class LateEnd(Exception): + pass + +def read((data, pos), length): + data2 = data[pos:pos + length] + if len(data2) != length: + raise EarlyEnd() + return data2, (data, pos + length) + +def size((data, pos)): + return len(data) - pos + +class Type(object): + __slots__ = [] + + # the same data can have only one unpacked representation, but multiple packed binary representations + + def __hash__(self): + rval = getattr(self, '_hash', None) + if rval is None: + try: + rval = self._hash = hash((type(self), frozenset(self.__dict__.items()))) + except: + print self.__dict__ + raise + return rval + + def __eq__(self, other): + return type(other) is type(self) and other.__dict__ == self.__dict__ + + def __ne__(self, other): + return not (self == other) + + def _unpack(self, data): + obj, (data2, pos) = self.read((data, 0)) + + assert data2 is data + + if pos != len(data): + raise LateEnd() + + return obj + + def _pack(self, obj): + f = self.write(None, obj) + + res = [] + while f is not None: + res.append(f[1]) + f = f[0] + res.reverse() + return ''.join(res) + + + def unpack(self, data): + obj = self._unpack(data) + + if p2pool.DEBUG: + data2 = self._pack(obj) + if data2 != data: + if self._unpack(data2) != obj: + raise AssertionError() + + return obj + + def pack(self, obj): + data = self._pack(obj) + + if p2pool.DEBUG: + if self._unpack(data) != obj: + raise AssertionError((self._unpack(data), obj)) + + return data + + + def pack_base58(self, obj): + return base58.encode(self.pack(obj)) + + def unpack_base58(self, base58_data): + return self.unpack(base58.decode(base58_data)) + + + def hash160(self, obj): + return IntType(160).unpack(hashlib.new('ripemd160', hashlib.sha256(self.pack(obj)).digest()).digest()) + + def hash256(self, obj): + return IntType(256).unpack(hashlib.sha256(hashlib.sha256(self.pack(obj)).digest()).digest()) + + def scrypt(self, obj): + import ltc_scrypt + return IntType(256).unpack(ltc_scrypt.getPoWHash(self.pack(obj))) + +class VarIntType(Type): + # redundancy doesn't matter here because bitcoin and p2pool both reencode before hashing + def read(self, file): + data, file = read(file, 1) + first = ord(data) + if first < 0xfd: + return first, file + elif first == 0xfd: + desc, length = '') + {8: 'B', 16: 'H', 32: 'I', 64: 'Q'}[bits]) + else: + return Type.__new__(cls, bits, endianness) + + def __init__(self, bits, endianness='little'): + assert bits % 8 == 0 + assert endianness in ['little', 'big'] + self.bytes = bits//8 + self.step = -1 if endianness == 'little' else 1 + self.format_str = '%%0%ix' % (2*self.bytes) + self.max = 2**bits + + def read(self, file, b2a_hex=binascii.b2a_hex): + data, file = read(file, self.bytes) + return int(b2a_hex(data[::self.step]), 16), file + + def write(self, file, item, a2b_hex=binascii.a2b_hex): + if not 0 <= item < self.max: + raise ValueError('invalid int value - %r' % (item,)) + return file, a2b_hex(self.format_str % (item,))[::self.step] + +class IPV6AddressType(Type): + def read(self, file): + data, file = read(file, 16) + if data[:12] != '00000000000000000000ffff'.decode('hex'): + raise ValueError('ipv6 addresses not supported yet') + return '.'.join(str(ord(x)) for x in data[12:]), file + + def write(self, file, item): + bits = map(int, item.split('.')) + if len(bits) != 4: + raise ValueError('invalid address: %r' % (bits,)) + data = '00000000000000000000ffff'.decode('hex') + ''.join(chr(x) for x in bits) + assert len(data) == 16, len(data) + return file, data + +_record_types = {} + +def get_record(fields): + fields = tuple(sorted(fields)) + if 'keys' in fields: + raise ValueError() + if fields not in _record_types: + class _Record(object): + __slots__ = fields + def __repr__(self): + return repr(dict(self)) + def __getitem__(self, key): + return getattr(self, key) + def __setitem__(self, key, value): + setattr(self, key, value) + #def __iter__(self): + # for field in self.__slots__: + # yield field, getattr(self, field) + def keys(self): + return self.__slots__ + def __eq__(self, other): + if isinstance(other, dict): + return dict(self) == other + elif isinstance(other, _Record): + return all(self[k] == other[k] for k in self.keys()) + raise TypeError() + def __ne__(self, other): + return not (self == other) + _record_types[fields] = _Record + return _record_types[fields]() + +class ComposedType(Type): + def __init__(self, fields): + self.fields = tuple(fields) + + def read(self, file): + item = get_record(k for k, v in self.fields) + for key, type_ in self.fields: + item[key], file = type_.read(file) + return item, file + + def write(self, file, item): + for key, type_ in self.fields: + file = type_.write(file, item[key]) + return file + +class PossiblyNoneType(Type): + def __init__(self, none_value, inner): + self.none_value = none_value + self.inner = inner + + def read(self, file): + value, file = self.inner.read(file) + return None if value == self.none_value else value, file + + def write(self, file, item): + if item == self.none_value: + raise ValueError('none_value used') + return self.inner.write(file, self.none_value if item is None else item)