p2pool/bitcoin/p2p.py

   1 '''
   2 Implementation of Bitcoin's p2p protocol
   3 '''
   4
   5 from __future__ import division
   6
   7 import hashlib
   8 import random
   9 import struct
  10 import time
  11
  12 from twisted.internet import defer, protocol, reactor, task
  13 from twisted.python import log
  14
  15 import p2pool
  16 from . import data as bitcoin_data, getwork
  17 from p2pool.util import variable, datachunker, deferral, forest, pack
  18
  19 class TooLong(Exception):
  20     pass
  21
  22 class BaseProtocol(protocol.Protocol):
  23     def connectionMade(self):
  24         self.dataReceived = datachunker.DataChunker(self.dataReceiver())
  25
  26     def dataReceiver(self):
  27         while True:
  28             start = ''
  29             while start != self._prefix:
  30                 start = (start + (yield 1))[-len(self._prefix):]
  31
  32             command = (yield 12).rstrip('\0')
  33             length, = struct.unpack('<I', (yield 4))
  34
  35             if length > self.max_payload_length:
  36                 print 'length too large'
  37                 continue
  38
  39             if self.use_checksum:
  40                 checksum = yield 4
  41             else:
  42                 checksum = None
  43
  44             payload = yield length
  45
  46             if checksum is not None:
  47                 if hashlib.sha256(hashlib.sha256(payload).digest()).digest()[:4] != checksum:
  48                     print 'invalid hash for', repr(command), checksum.encode('hex') if checksum is not None else None, repr(payload[:100].encode('hex')), len(payload)
  49                     continue
  50
  51             type_ = getattr(self, 'message_' + command, None)
  52             if type_ is None:
  53                 if p2pool.DEBUG:
  54                     print 'no type for', repr(command)
  55                 continue
  56
  57             try:
  58                 payload2 = type_.unpack(payload)
  59             except:
  60                 print 'RECV', command, checksum.encode('hex') if checksum is not None else None, repr(payload.encode('hex')), len(payload)
  61                 log.err(None, 'Error parsing message: (see RECV line)')
  62                 continue
  63
  64             self.packetReceived(command, payload2)
  65
  66     def packetReceived(self, command, payload2):
  67         handler = getattr(self, 'handle_' + command, None)
  68         if handler is None:
  69             if p2pool.DEBUG:
  70                 print 'no handler for', repr(command)
  71             return
  72
  73         try:
  74             handler(**payload2)
  75         except:
  76             print 'RECV', command, repr(payload2)[:100]
  77             log.err(None, 'Error handling message: (see RECV line)')
  78
  79     def sendPacket(self, command, payload2):
  80         if len(command) >= 12:
  81             raise ValueError('command too long')
  82         type_ = getattr(self, 'message_' + command, None)
  83         if type_ is None:
  84             raise ValueError('invalid command')
  85         #print 'SEND', command, repr(payload2)[:500]
  86         payload = type_.pack(payload2)
  87         if len(payload) > self.max_payload_length:
  88             raise TooLong('payload too long')
  89         if self.use_checksum:
  90             checksum = hashlib.sha256(hashlib.sha256(payload).digest()).digest()[:4]
  91         else:
  92             checksum = ''
  93         data = self._prefix + struct.pack('<12sI', command, len(payload)) + checksum + payload
  94         self.transport.write(data)
  95
  96     def __getattr__(self, attr):
  97         prefix = 'send_'
  98         if attr.startswith(prefix):
  99             command = attr[len(prefix):]
 100             return lambda **payload2: self.sendPacket(command, payload2)
 101         #return protocol.Protocol.__getattr__(self, attr)
 102         raise AttributeError(attr)
 103
 104 class Protocol(BaseProtocol):
 105     def __init__(self, net):
 106         self._prefix = net.P2P_PREFIX
 107
 108     version = 0
 109
 110     max_payload_length = 1000000
 111
 112     @property
 113     def use_checksum(self):
 114         return self.version >= 209
 115
 116     def connectionMade(self):
 117         BaseProtocol.connectionMade(self)
 118
 119         self.send_version(
 120             version=32200,
 121             services=1,
 122             time=int(time.time()),
 123             addr_to=dict(
 124                 services=1,
 125                 address=self.transport.getPeer().host,
 126                 port=self.transport.getPeer().port,
 127             ),
 128             addr_from=dict(
 129                 services=1,
 130                 address=self.transport.getHost().host,
 131                 port=self.transport.getHost().port,
 132             ),
 133             nonce=random.randrange(2**64),
 134             sub_version_num='',
 135             start_height=0,
 136         )
 137
 138     message_version = pack.ComposedType([
 139         ('version', pack.IntType(32)),
 140         ('services', pack.IntType(64)),
 141         ('time', pack.IntType(64)),
 142         ('addr_to', bitcoin_data.address_type),
 143         ('addr_from', bitcoin_data.address_type),
 144         ('nonce', pack.IntType(64)),
 145         ('sub_version_num', pack.VarStrType()),
 146         ('start_height', pack.IntType(32)),
 147     ])
 148     def handle_version(self, version, services, time, addr_to, addr_from, nonce, sub_version_num, start_height):
 149         #print 'VERSION', locals()
 150         self.version_after = version
 151         self.send_verack()
 152
 153     message_verack = pack.ComposedType([])
 154     def handle_verack(self):
 155         self.version = self.version_after
 156
 157         self.ready()
 158
 159     def ready(self):
 160         self.get_block = deferral.ReplyMatcher(lambda hash: self.send_getdata(requests=[dict(type='block', hash=hash)]))
 161         self.get_block_header = deferral.ReplyMatcher(lambda hash: self.send_getheaders(version=1, have=[], last=hash))
 162         self.get_tx = deferral.ReplyMatcher(lambda hash: self.send_getdata(requests=[dict(type='tx', hash=hash)]))
 163
 164         if hasattr(self.factory, 'resetDelay'):
 165             self.factory.resetDelay()
 166         if hasattr(self.factory, 'gotConnection'):
 167             self.factory.gotConnection(self)
 168
 169     message_inv = pack.ComposedType([
 170         ('invs', pack.ListType(pack.ComposedType([
 171             ('type', pack.EnumType(pack.IntType(32), {'tx': 1, 'block': 2})),
 172             ('hash', pack.IntType(256)),
 173         ]))),
 174     ])
 175     def handle_inv(self, invs):
 176         for inv in invs:
 177             if inv['type'] == 'tx':
 178                 self.factory.new_tx.happened(inv['hash'])
 179             elif inv['type'] == 'block':
 180                 self.factory.new_block.happened(inv['hash'])
 181             else:
 182                 print 'Unknown inv type', item
 183
 184     message_getdata = pack.ComposedType([
 185         ('requests', pack.ListType(pack.ComposedType([
 186             ('type', pack.EnumType(pack.IntType(32), {'tx': 1, 'block': 2})),
 187             ('hash', pack.IntType(256)),
 188         ]))),
 189     ])
 190     message_getblocks = pack.ComposedType([
 191         ('version', pack.IntType(32)),
 192         ('have', pack.ListType(pack.IntType(256))),
 193         ('last', pack.PossiblyNoneType(0, pack.IntType(256))),
 194     ])
 195     message_getheaders = pack.ComposedType([
 196         ('version', pack.IntType(32)),
 197         ('have', pack.ListType(pack.IntType(256))),
 198         ('last', pack.PossiblyNoneType(0, pack.IntType(256))),
 199     ])
 200     message_getaddr = pack.ComposedType([])
 201
 202     message_addr = pack.ComposedType([
 203         ('addrs', pack.ListType(pack.ComposedType([
 204             ('timestamp', pack.IntType(32)),
 205             ('address', bitcoin_data.address_type),
 206         ]))),
 207     ])
 208     def handle_addr(self, addrs):
 209         for addr in addrs:
 210             pass
 211
 212     message_tx = pack.ComposedType([
 213         ('tx', bitcoin_data.tx_type),
 214     ])
 215     def handle_tx(self, tx):
 216         self.get_tx.got_response(bitcoin_data.tx_type.hash256(tx), tx)
 217
 218     message_block = pack.ComposedType([
 219         ('block', bitcoin_data.block_type),
 220     ])
 221     def handle_block(self, block):
 222         block_hash = bitcoin_data.block_header_type.hash256(block['header'])
 223         self.get_block.got_response(block_hash, block)
 224         self.get_block_header.got_response(block_hash, block['header'])
 225
 226     message_headers = pack.ComposedType([
 227         ('headers', pack.ListType(bitcoin_data.block_type)),
 228     ])
 229     def handle_headers(self, headers):
 230         for header in headers:
 231             header = header['header']
 232             self.get_block_header.got_response(bitcoin_data.block_header_type.hash256(header), header)
 233         self.factory.new_headers.happened([header['header'] for header in headers])
 234
 235     message_ping = pack.ComposedType([])
 236     def handle_ping(self):
 237         pass
 238
 239     message_alert = pack.ComposedType([
 240         ('message', pack.VarStrType()),
 241         ('signature', pack.VarStrType()),
 242     ])
 243     def handle_alert(self, message, signature):
 244         print 'ALERT:', (message, signature)
 245
 246     def connectionLost(self, reason):
 247         if hasattr(self.factory, 'gotConnection'):
 248             self.factory.gotConnection(None)
 249
 250 class ClientFactory(protocol.ReconnectingClientFactory):
 251     protocol = Protocol
 252
 253     maxDelay = 1
 254
 255     def __init__(self, net):
 256         self.net = net
 257         self.conn = variable.Variable(None)
 258
 259         self.new_block = variable.Event()
 260         self.new_tx = variable.Event()
 261         self.new_headers = variable.Event()
 262
 263     def buildProtocol(self, addr):
 264         p = self.protocol(self.net)
 265         p.factory = self
 266         return p
 267
 268     def gotConnection(self, conn):
 269         self.conn.set(conn)
 270
 271     def getProtocol(self):
 272         return self.conn.get_not_none()
 273
 274 class HeaderWrapper(object):
 275     target = 2**256 - 1
 276     __slots__ = 'hash previous_hash'.split(' ')
 277
 278     @classmethod
 279     def from_header(cls, header):
 280         return cls(bitcoin_data.block_header_type.hash256(header), header['previous_block'])
 281
 282     def __init__(self, hash, previous_hash):
 283         self.hash, self.previous_hash = hash, previous_hash
 284
 285 class HeightTracker(object):
 286     '''Point this at a factory and let it take care of getting block heights'''
 287
 288     def __init__(self, rpc_proxy, factory, backlog_needed=1000):
 289         self._rpc_proxy = rpc_proxy
 290         self._factory = factory
 291         self._backlog_needed = backlog_needed
 292
 293         self._tracker = forest.Tracker()
 294
 295         self._watch1 = self._factory.new_headers.watch(self._heard_headers)
 296         self._watch2 = self._factory.new_block.watch(self._heard_block)
 297
 298         self._requested = set()
 299         self._clear_task = task.LoopingCall(self._requested.clear)
 300         self._clear_task.start(60)
 301
 302         self._last_notified_size = 0
 303
 304         self.updated = variable.Event()
 305
 306         self._think_task = task.LoopingCall(self._think)
 307         self._think_task.start(15)
 308         self._think2_task = task.LoopingCall(self._think2)
 309         self._think2_task.start(15)
 310
 311     def _think(self):
 312         try:
 313             highest_head = max(self._tracker.heads, key=lambda h: self._tracker.get_height_and_last(h)[0]) if self._tracker.heads else None
 314             if highest_head is None:
 315                 return # wait for think2
 316             height, last = self._tracker.get_height_and_last(highest_head)
 317             if height < self._backlog_needed:
 318                 self._request(last)
 319         except:
 320             log.err(None, 'Error in HeightTracker._think:')
 321
 322     @defer.inlineCallbacks
 323     def _think2(self):
 324         try:
 325             ba = getwork.BlockAttempt.from_getwork((yield self._rpc_proxy.rpc_getwork()))
 326             self._request(ba.previous_block)
 327         except:
 328             log.err(None, 'Error in HeightTracker._think2:')
 329
 330     def _heard_headers(self, headers):
 331         changed = False
 332         for header in headers:
 333             hw = HeaderWrapper.from_header(header)
 334             if hw.hash in self._tracker.shares:
 335                 continue
 336             changed = True
 337             self._tracker.add(hw)
 338         if changed:
 339             self.updated.happened()
 340         self._think()
 341
 342         if len(self._tracker.shares) >= self._last_notified_size + 100:
 343             print 'Have %i/%i block headers' % (len(self._tracker.shares), self._backlog_needed)
 344             self._last_notified_size = len(self._tracker.shares)
 345
 346     def _heard_block(self, block_hash):
 347         self._request(block_hash)
 348
 349     @defer.inlineCallbacks
 350     def _request(self, last):
 351         if last in self._tracker.shares:
 352             return
 353         if last in self._requested:
 354             return
 355         self._requested.add(last)
 356         (yield self._factory.getProtocol()).send_getheaders(version=1, have=[], last=last)
 357
 358     def get_height_rel_highest(self, block_hash):
 359         # callers: highest height can change during yields!
 360         height, last = self._tracker.get_height_and_last(block_hash)
 361         if last not in self._tracker.tails:
 362             return -1000000000 # XXX hack
 363         return height - max(self._tracker.get_height(head_hash) for head_hash in self._tracker.tails[last])
 364
 365     def stop(self):
 366         self._factory.new_headers.unwatch(self._watch1)
 367         self._factory.new_block.unwatch(self._watch2)
 368         self._clear_task.stop()
 369         self._think_task.stop()
 370         self._think2_task.stop()
 371
 372 if __name__ == '__main__':
 373     from . import networks
 374     factory = ClientFactory(networks.BitcoinMainnet)
 375     reactor.connectTCP('127.0.0.1', 8333, factory)
 376
 377     @repr
 378     @apply
 379     @defer.inlineCallbacks
 380     def think():
 381         try:
 382             print (yield (yield factory.getProtocol()).get_block(0x000000000000003aaaf7638f9f9c0d0c60e8b0eb817dcdb55fd2b1964efc5175))
 383         except defer.TimeoutError:
 384             print "timeout"
 385         reactor.stop()
 386
 387     reactor.run()