2 Implementation of Bitcoin's p2p protocol
5 from __future__ import division
13 from twisted.internet import defer, protocol, reactor, task
14 from twisted.python import log
16 from . import data as bitcoin_data
17 from p2pool.util import variable, datachunker, deferral
19 class TooLong(Exception):
22 class BaseProtocol(protocol.Protocol):
23 def connectionMade(self):
24 self.dataReceived = datachunker.DataChunker(self.dataReceiver())
26 def dataReceiver(self):
29 while start != self._prefix:
30 start = (start + (yield 1))[-len(self._prefix):]
32 command = (yield 12).rstrip('\0')
33 length, = struct.unpack('<I', (yield 4))
35 if length > self.max_net_payload_length:
36 print 'length too long'
44 compressed_payload = yield length
48 d = zlib.decompressobj()
49 payload = d.decompress(compressed_payload, self.max_payload_length)
51 print 'compressed payload expanded too much'
53 assert not len(payload) > self.max_payload_length
55 log.err(None, 'Failure decompressing message:')
58 if len(compressed_payload) > self.max_payload_length:
59 print 'compressed payload expanded too much'
61 payload = compressed_payload
63 if checksum is not None:
64 if hashlib.sha256(hashlib.sha256(payload).digest()).digest()[:4] != checksum:
65 print 'invalid hash for', repr(command), checksum.encode('hex') if checksum is not None else None, repr(payload[:100].encode('hex')), len(payload)
68 type_ = getattr(self, 'message_' + command, None)
70 print 'no type for', repr(command)
74 payload2 = type_.unpack(payload)
76 print 'RECV', command, checksum.encode('hex') if checksum is not None else None, repr(payload.encode('hex')), len(payload)
77 log.err(None, 'Error parsing message: (see RECV line)')
80 handler = getattr(self, 'handle_' + command, None)
82 print 'no handler for', repr(command)
88 print 'RECV', command, repr(payload2)[:100]
89 log.err(None, 'Error handling message: (see RECV line)')
92 def sendPacket(self, command, payload2):
93 if len(command) >= 12:
94 raise ValueError('command too long')
95 type_ = getattr(self, 'message_' + command, None)
97 raise ValueError('invalid command')
98 #print 'SEND', command, repr(payload2)[:500]
99 payload = type_.pack(payload2)
100 if len(payload) > self.max_payload_length:
101 raise TooLong('payload too long')
102 if self.use_checksum:
103 checksum = hashlib.sha256(hashlib.sha256(payload).digest()).digest()[:4]
106 compressed_payload = zlib.compress(payload) if self.compress else payload
107 if len(compressed_payload) > self.max_net_payload_length:
108 raise TooLong('compressed payload too long')
109 data = self._prefix + struct.pack('<12sI', command, len(compressed_payload)) + checksum + compressed_payload
110 self.transport.write(data)
112 def __getattr__(self, attr):
114 if attr.startswith(prefix):
115 command = attr[len(prefix):]
116 return lambda **payload2: self.sendPacket(command, payload2)
117 #return protocol.Protocol.__getattr__(self, attr)
118 raise AttributeError(attr)
120 class Protocol(BaseProtocol):
121 def __init__(self, net):
122 self._prefix = net.BITCOIN_P2P_PREFIX
126 max_payload_length = max_net_payload_length = 1000000
130 def use_checksum(self):
131 return self.version >= 209
136 def connectionMade(self):
137 BaseProtocol.connectionMade(self)
142 time=int(time.time()),
145 address=self.transport.getPeer().host,
146 port=self.transport.getPeer().port,
150 address=self.transport.getHost().host,
151 port=self.transport.getHost().port,
153 nonce=random.randrange(2**64),
158 message_version = bitcoin_data.ComposedType([
159 ('version', bitcoin_data.StructType('<I')),
160 ('services', bitcoin_data.StructType('<Q')),
161 ('time', bitcoin_data.StructType('<Q')),
162 ('addr_to', bitcoin_data.address_type),
163 ('addr_from', bitcoin_data.address_type),
164 ('nonce', bitcoin_data.StructType('<Q')),
165 ('sub_version_num', bitcoin_data.VarStrType()),
166 ('start_height', bitcoin_data.StructType('<I')),
168 def handle_version(self, version, services, time, addr_to, addr_from, nonce, sub_version_num, start_height):
169 #print 'VERSION', locals()
170 self.version_after = version
173 message_verack = bitcoin_data.ComposedType([])
174 def handle_verack(self):
175 self.version = self.version_after
180 self.check_order = deferral.GenericDeferrer(2**256, lambda id, order: self.send_checkorder(id=id, order=order))
181 self.submit_order = deferral.GenericDeferrer(2**256, lambda id, order: self.send_submitorder(id=id, order=order))
182 self.get_block = deferral.ReplyMatcher(lambda hash: self.send_getdata(requests=[dict(type='block', hash=hash)]))
183 self.get_block_header = deferral.ReplyMatcher(lambda hash: self.send_getheaders(version=1, have=[], last=hash))
184 self.get_tx = deferral.ReplyMatcher(lambda hash: self.send_getdata(requests=[dict(type='tx', hash=hash)]))
186 if hasattr(self.factory, 'resetDelay'):
187 self.factory.resetDelay()
188 if hasattr(self.factory, 'gotConnection'):
189 self.factory.gotConnection(self)
191 message_inv = bitcoin_data.ComposedType([
192 ('invs', bitcoin_data.ListType(bitcoin_data.ComposedType([
193 ('type', bitcoin_data.EnumType(bitcoin_data.StructType('<I'), {'tx': 1, 'block': 2})),
194 ('hash', bitcoin_data.HashType()),
197 def handle_inv(self, invs):
199 if inv['type'] == 'tx':
200 self.factory.new_tx.happened(inv['hash'])
201 elif inv['type'] == 'block':
202 self.factory.new_block.happened(inv['hash'])
204 print 'Unknown inv type', item
206 message_getdata = bitcoin_data.ComposedType([
207 ('requests', bitcoin_data.ListType(bitcoin_data.ComposedType([
208 ('type', bitcoin_data.EnumType(bitcoin_data.StructType('<I'), {'tx': 1, 'block': 2})),
209 ('hash', bitcoin_data.HashType()),
212 message_getblocks = bitcoin_data.ComposedType([
213 ('version', bitcoin_data.StructType('<I')),
214 ('have', bitcoin_data.ListType(bitcoin_data.HashType())),
215 ('last', bitcoin_data.PossiblyNone(0, bitcoin_data.HashType())),
217 message_getheaders = bitcoin_data.ComposedType([
218 ('version', bitcoin_data.StructType('<I')),
219 ('have', bitcoin_data.ListType(bitcoin_data.HashType())),
220 ('last', bitcoin_data.PossiblyNone(0, bitcoin_data.HashType())),
222 message_getaddr = bitcoin_data.ComposedType([])
223 message_checkorder = bitcoin_data.ComposedType([
224 ('id', bitcoin_data.HashType()),
225 ('order', bitcoin_data.FixedStrType(60)), # XXX
227 message_submitorder = bitcoin_data.ComposedType([
228 ('id', bitcoin_data.HashType()),
229 ('order', bitcoin_data.FixedStrType(60)), # XXX
232 message_addr = bitcoin_data.ComposedType([
233 ('addrs', bitcoin_data.ListType(bitcoin_data.ComposedType([
234 ('timestamp', bitcoin_data.StructType('<I')),
235 ('address', bitcoin_data.address_type),
238 def handle_addr(self, addrs):
242 message_tx = bitcoin_data.ComposedType([
243 ('tx', bitcoin_data.tx_type),
245 def handle_tx(self, tx):
246 self.get_tx.got_response(bitcoin_data.tx_type.hash256(tx), tx)
248 message_block = bitcoin_data.ComposedType([
249 ('block', bitcoin_data.block_type),
251 def handle_block(self, block):
252 block_hash = bitcoin_data.block_header_type.hash256(block['header'])
253 self.get_block.got_response(block_hash, block)
254 self.get_block_header.got_response(block_hash, block['header'])
256 message_headers = bitcoin_data.ComposedType([
257 ('headers', bitcoin_data.ListType(bitcoin_data.block_type)),
259 def handle_headers(self, headers):
260 for header in headers:
261 header = header['header']
262 self.get_block_header.got_response(bitcoin_data.block_header_type.hash256(header), header)
263 self.factory.new_headers.happened([header['header'] for header in headers])
265 message_reply = bitcoin_data.ComposedType([
266 ('hash', bitcoin_data.HashType()),
267 ('reply', bitcoin_data.EnumType(bitcoin_data.StructType('<I'), {'success': 0, 'failure': 1, 'denied': 2})),
268 ('script', bitcoin_data.PossiblyNone('', bitcoin_data.VarStrType())),
270 def handle_reply(self, hash, reply, script):
271 self.check_order.got_response(hash, dict(reply=reply, script=script))
272 self.submit_order.got_response(hash, dict(reply=reply, script=script))
274 message_ping = bitcoin_data.ComposedType([])
275 def handle_ping(self):
278 message_alert = bitcoin_data.ComposedType([
279 ('message', bitcoin_data.VarStrType()),
280 ('signature', bitcoin_data.VarStrType()),
282 def handle_alert(self, message, signature):
283 print 'ALERT:', (message, signature)
285 def connectionLost(self, reason):
286 if hasattr(self.factory, 'gotConnection'):
287 self.factory.gotConnection(None)
289 class ClientFactory(protocol.ReconnectingClientFactory):
294 def __init__(self, net):
296 self.conn = variable.Variable(None)
298 self.new_block = variable.Event()
299 self.new_tx = variable.Event()
300 self.new_headers = variable.Event()
302 def buildProtocol(self, addr):
303 p = self.protocol(self.net)
307 def gotConnection(self, conn):
310 def getProtocol(self):
311 return self.conn.get_not_none()
313 class HeaderWrapper(object):
315 __slots__ = 'hash previous_hash'.split(' ')
318 def from_header(cls, header):
319 return cls(bitcoin_data.block_header_type.hash256(header), header['previous_block'])
321 def __init__(self, hash, previous_hash):
322 self.hash, self.previous_hash = hash, previous_hash
324 class HeightTracker(object):
325 '''Point this at a factory and let it take care of getting block heights'''
327 def __init__(self, factory, backing):
328 self.factory = factory
329 self.tracker = bitcoin_data.Tracker()
330 self.backing = backing
331 self.most_recent = None
333 self._watch1 = self.factory.new_headers.watch(self.heard_headers)
334 self._watch2 = self.factory.new_block.watch(self.heard_block)
336 self.requested = set()
337 self._clear_task = task.LoopingCall(self.requested.clear)
338 self._clear_task.start(60)
340 self.last_notified_size = 0
342 self.updated = variable.Event()
348 def _load_backing(self):
349 open(self.backing, 'ab').close()
350 with open(self.backing, 'rb') as f:
354 hash, previous_hash, checksum = (int(x, 16) for x in line.strip().split(' '))
356 print "skipping over bad data in headers.dat"
358 if (hash - previous_hash) % 2**256 != checksum:
359 print "checksum failed"
361 if previous_hash == 0: previous_hash = None
363 if count % 10000 == 0 and count: print count
364 if hash not in self.tracker.shares:
365 self.tracker.add(HeaderWrapper(hash, previous_hash))
368 highest_head = max(self.tracker.heads, key=lambda h: self.tracker.get_height_and_last(h)[0]) if self.tracker.heads else None
369 height, last = self.tracker.get_height_and_last(highest_head)
374 while cur is not None:
376 if step > cur_height:
378 cur = self.tracker.get_nth_parent_hash(cur, step)
383 have.append(self.tracker.get_nth_parent_hash(highest_head, height - 1))
386 self.request(have, None)
388 for tail in self.tracker.tails:
391 self.request([], tail)
392 for head in self.tracker.heads:
393 if head == highest_head:
395 self.request([head], None)
397 def heard_headers(self, headers):
399 b = open(self.backing, 'ab')
400 for header in headers:
401 hw = HeaderWrapper.from_header(header)
402 if hw.hash in self.tracker.shares:
406 hash, prev = hw.hash, 0 if hw.previous_hash is None else hw.previous_hash
407 b.write('%x %x %x\n' % (hash, prev, (hash - prev) % 2**256))
410 self.updated.happened()
413 if len(self.tracker.shares) > self.last_notified_size + 10:
414 print 'Have %i block headers' % len(self.tracker.shares)
415 self.last_notified_size = len(self.tracker.shares)
417 def heard_block(self, block_hash):
418 self.request([], block_hash)
420 @defer.inlineCallbacks
421 def request(self, have, last):
422 if (tuple(have), last) in self.requested:
424 self.requested.add((tuple(have), last))
425 (yield self.factory.getProtocol()).send_getheaders(version=1, have=have, last=last)
427 def getHeight(self, block_hash):
428 height, last = self.tracker.get_height_and_last(block_hash)
430 #self.request([], last)
434 def get_min_height(self, block_hash):
435 height, last = self.tracker.get_height_and_last(block_hash)
436 #if last is not None:
437 # self.request([], last)
440 def get_highest_height(self):
441 return self.tracker.get_highest_height()
444 self.factory.new_headers.unwatch(self._watch1)
445 self.factory.new_block.unwatch(self._watch2)
446 self._clear_task.stop()
448 if __name__ == '__main__':
449 factory = ClientFactory(bitcoin_data.Mainnet)
450 reactor.connectTCP('127.0.0.1', 8333, factory)
451 h = HeightTracker(factory)
455 @defer.inlineCallbacks
458 yield deferral.sleep(1)
459 print h.get_min_height(0xa285c3cb2a90ac7194cca034512748289e2526d9d7ae6ee7523)