From 1e086b3786e3708cb71205962d9db1684f2271f2 Mon Sep 17 00:00:00 2001 From: slush Date: Sun, 9 Sep 2012 17:28:06 +0200 Subject: [PATCH] Initial commit --- conf/config_sample.py | 74 +++++++ launcher_demo.tac | 23 ++ lib/bitcoin_rpc.py | 64 ++++++ lib/block_template.py | 140 ++++++++++++ lib/block_updater.py | 64 ++++++ lib/coinbaser.py | 51 +++++ lib/coinbasetx.py | 49 +++++ lib/extranonce_counter.py | 25 +++ lib/halfnode.py | 523 +++++++++++++++++++++++++++++++++++++++++++++ lib/merkletree.py | 103 +++++++++ lib/template_registry.py | 262 +++++++++++++++++++++++ lib/util.py | 201 +++++++++++++++++ mining/__init__.py | 38 ++++ mining/interfaces.py | 58 +++++ mining/service.py | 118 ++++++++++ mining/subscription.py | 53 +++++ 16 files changed, 1846 insertions(+), 0 deletions(-) create mode 100644 conf/__init__.py create mode 100644 conf/config_sample.py create mode 100644 launcher_demo.tac create mode 100644 lib/__init__.py create mode 100644 lib/bitcoin_rpc.py create mode 100644 lib/block_template.py create mode 100644 lib/block_updater.py create mode 100644 lib/coinbaser.py create mode 100644 lib/coinbasetx.py create mode 100644 lib/extranonce_counter.py create mode 100644 lib/halfnode.py create mode 100644 lib/merkletree.py create mode 100644 lib/template_registry.py create mode 100644 lib/util.py create mode 100644 mining/__init__.py create mode 100644 mining/interfaces.py create mode 100644 mining/service.py create mode 100644 mining/subscription.py diff --git a/conf/__init__.py b/conf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/conf/config_sample.py b/conf/config_sample.py new file mode 100644 index 0000000..02c7d30 --- /dev/null +++ b/conf/config_sample.py @@ -0,0 +1,74 @@ +''' +This is example configuration for Stratum server. +Please rename it to settings.py and fill correct values. +''' + +# ******************** GENERAL SETTINGS *************** + +# Enable some verbose debug (logging requests and responses). +DEBUG = False + +# Destination for application logs, files rotated once per day. +LOGDIR = 'log/' + +# Main application log file. +LOGFILE = None#'stratum.log' + +# Possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL +LOGLEVEL = 'INFO' + +# How many threads use for synchronous methods (services). +# 30 is enough for small installation, for real usage +# it should be slightly more, say 100-300. +THREAD_POOL_SIZE = 10 + +ENABLE_EXAMPLE_SERVICE = True + +# ******************** TRANSPORTS ********************* + +# Hostname or external IP to expose +HOSTNAME = 'localhost' + +# Port used for Socket transport. Use 'None' for disabling the transport. +LISTEN_SOCKET_TRANSPORT = 3333 + +# Port used for HTTP Poll transport. Use 'None' for disabling the transport +LISTEN_HTTP_TRANSPORT = None + +# Port used for HTTPS Poll transport +LISTEN_HTTPS_TRANSPORT = None + +# Port used for WebSocket transport, 'None' for disabling WS +LISTEN_WS_TRANSPORT = None + +# Port used for secure WebSocket, 'None' for disabling WSS +LISTEN_WSS_TRANSPORT = None + +# Hostname and credentials for one trusted Bitcoin node ("Satoshi's client"). +# Stratum uses both P2P port (which is 8333 already) and RPC port +BITCOIN_TRUSTED_HOST = 'localhost' +BITCOIN_TRUSTED_PORT = 8332 +BITCOIN_TRUSTED_USER = 'user' +BITCOIN_TRUSTED_PASSWORD = 'somepassword' + +# Use "echo -n '' | sha256sum | cut -f1 -d' ' " +# for calculating SHA256 of your preferred password +ADMIN_PASSWORD_SHA256 = None +#ADMIN_PASSWORD_SHA256 = '9e6c0c1db1e0dfb3fa5159deb4ecd9715b3c8cd6b06bd4a3ad77e9a8c5694219' # SHA256 of the password + +IRC_NICK = None + +''' +DATABASE_DRIVER = 'MySQLdb' +DATABASE_HOST = 'localhost' +DATABASE_DBNAME = 'pooldb' +DATABASE_USER = 'pooldb' +DATABASE_PASSWORD = '**empty**' +''' + +# Pool related settings +INSTANCE_ID = 31 +CENTRAL_WALLET = 'set_valid_addresss_in_config!' +PREVHASH_REFRESH_INTERVAL = 5 # in sec +MERKLE_REFRESH_INTERVAL = 60 # How often check memorypool +COINBASE_EXTRAS = '/stratum/' diff --git a/launcher_demo.tac b/launcher_demo.tac new file mode 100644 index 0000000..5e66817 --- /dev/null +++ b/launcher_demo.tac @@ -0,0 +1,23 @@ +# Run me with "twistd -ny launcher_demo.tac" + +# Add conf directory to python path. +# Configuration file is standard python module. +import os, sys +sys.path = [os.path.join(os.getcwd(), 'conf'),] + sys.path + +# Bootstrap Stratum framework +import stratum +from stratum import settings +application = stratum.setup() + +# Load mining service into stratum framework +import mining + +from mining.interfaces import Interfaces +from mining.interfaces import WorkerManagerInterface, ShareManagerInterface, TimestamperInterface + +Interfaces.set_share_manager(ShareManagerInterface()) +Interfaces.set_worker_manager(WorkerManagerInterface()) +Interfaces.set_timestamper(TimestamperInterface()) + +mining.setup() diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/bitcoin_rpc.py b/lib/bitcoin_rpc.py new file mode 100644 index 0000000..3517b6d --- /dev/null +++ b/lib/bitcoin_rpc.py @@ -0,0 +1,64 @@ +''' + Implements simple interface to bitcoind's RPC. +''' + +import simplejson as json +import base64 +from twisted.internet import defer +from twisted.web import client + +import stratum.logger +log = stratum.logger.get_logger('bitcoin_rpc') + +class BitcoinRPC(object): + + def __init__(self, host, port, username, password): + self.bitcoin_url = 'http://%s:%d' % (host, port) + self.credentials = base64.b64encode("%s:%s" % (username, password)) + self.headers = { + 'Content-Type': 'text/json', + 'Authorization': 'Basic %s' % self.credentials, + } + + def _call_raw(self, data): + return client.getPage( + url=self.bitcoin_url, + method='POST', + headers=self.headers, + postdata=data, + ) + + def _call(self, method, params): + return self._call_raw(json.dumps({ + 'jsonrpc': '2.0', + 'method': method, + 'params': params, + 'id': '1', + })) + + @defer.inlineCallbacks + def submitblock(self, block_hex): + resp = (yield self._call('submitblock', [block_hex,])) + if json.loads(resp)['result'] == None: + defer.returnValue(True) + else: + defer.returnValue(False) + + @defer.inlineCallbacks + def getblocktemplate(self): + resp = (yield self._call('getblocktemplate', [])) + defer.returnValue(json.loads(resp)['result']) + + @defer.inlineCallbacks + def prevhash(self): + resp = (yield self._call('getwork', [])) + try: + defer.returnValue(json.loads(resp)['result']['data'][8:72]) + except Exception as e: + log.exception("Cannot decode prevhash %s" % str(e)) + raise + + @defer.inlineCallbacks + def validateaddress(self, address): + resp = (yield self._call('validateaddress', [address,])) + defer.returnValue(json.loads(resp)['result']) \ No newline at end of file diff --git a/lib/block_template.py b/lib/block_template.py new file mode 100644 index 0000000..4573c0d --- /dev/null +++ b/lib/block_template.py @@ -0,0 +1,140 @@ +import StringIO +import binascii +import struct + +import util +import merkletree +import halfnode +from coinbasetx import CoinbaseTransaction + +# Remove dependency to settings, coinbase extras should be +# provided from coinbaser +from stratum import settings + +class BlockTemplate(halfnode.CBlock): + '''Template is used for generating new jobs for clients. + Let's iterate extranonce1, extranonce2, ntime and nonce + to find out valid bitcoin block!''' + + coinbase_transaction_class = CoinbaseTransaction + + def __init__(self, timestamper, coinbaser, job_id): + super(BlockTemplate, self).__init__() + + self.job_id = job_id + self.timestamper = timestamper + self.coinbaser = coinbaser + + self.prevhash_bin = '' # reversed binary form of prevhash + self.prevhash_hex = '' + self.timedelta = 0 + self.curtime = 0 + self.target = 0 + #self.coinbase_hex = None + self.merkletree = None + + self.broadcast_args = [] + + # List of 4-tuples (extranonce1, extranonce2, ntime, nonce) + # registers already submitted and checked shares + # There may be registered also invalid shares inside! + self.submits = [] + + def fill_from_rpc(self, data): + '''Convert getblocktemplate result into BlockTemplate instance''' + + #txhashes = [None] + [ binascii.unhexlify(t['hash']) for t in data['transactions'] ] + txhashes = [None] + [ util.ser_uint256(int(t['hash'], 16)) for t in data['transactions'] ] + mt = merkletree.MerkleTree(txhashes) + + coinbase = self.coinbase_transaction_class(self.timestamper, self.coinbaser, data['coinbasevalue'], + data['coinbaseaux']['flags'], data['height'], settings.COINBASE_EXTRAS) + + self.height = data['height'] + self.nVersion = data['version'] + self.hashPrevBlock = int(data['previousblockhash'], 16) + self.nBits = int(data['bits'], 16) + self.hashMerkleRoot = 0 + self.nTime = 0 + self.nNonce = 0 + self.vtx = [ coinbase, ] + + for tx in data['transactions']: + t = halfnode.CTransaction() + t.deserialize(StringIO.StringIO(binascii.unhexlify(tx['data']))) + self.vtx.append(t) + + self.curtime = data['curtime'] + self.timedelta = self.curtime - int(self.timestamper.time()) + self.merkletree = mt + self.target = util.uint256_from_compact(self.nBits) + + # Reversed prevhash + self.prevhash_bin = binascii.unhexlify(util.reverse_hash(data['previousblockhash'])) + self.prevhash_hex = "%064x" % self.hashPrevBlock + + self.broadcast_args = self.build_broadcast_args() + + def register_submit(self, extranonce1, extranonce2, ntime, nonce): + '''Client submitted some solution. Let's register it to + prevent double submissions.''' + + t = (extranonce1, extranonce2, ntime, nonce) + if t not in self.submits: + self.submits.append(t) + return True + return False + + def build_broadcast_args(self): + '''Build parameters of mining.notify call. All clients + may receive the same params, because they include + their unique extranonce1 into the coinbase, so every + coinbase_hash (and then merkle_root) will be unique as well.''' + job_id = self.job_id + prevhash = binascii.hexlify(self.prevhash_bin) + (coinb1, coinb2) = [ binascii.hexlify(x) for x in self.vtx[0]._serialized ] + merkle_branch = [ binascii.hexlify(x) for x in self.merkletree._steps ] + version = binascii.hexlify(struct.pack(">i", self.nVersion)) + nbits = binascii.hexlify(struct.pack(">I", self.nBits)) + ntime = binascii.hexlify(struct.pack(">I", self.curtime)) + clean_jobs = True + + return (job_id, prevhash, coinb1, coinb2, merkle_branch, version, nbits, ntime, clean_jobs) + + def serialize_coinbase(self, extranonce1, extranonce2): + '''Serialize coinbase with given extranonce1 and extranonce2 + in binary form''' + (part1, part2) = self.vtx[0]._serialized + return part1 + extranonce1 + extranonce2 + part2 + + def check_ntime(self, ntime): + '''Check for ntime restrictions.''' + if ntime < self.curtime: + return False + + if ntime > (self.timestamper.time() + 60): + # Be strict on ntime into the near future + # may be unnecessary + return False + + return True + + def serialize_header(self, merkle_root_int, ntime_bin, nonce_bin): + '''Serialize header for calculating block hash''' + r = struct.pack(">i", self.nVersion) + r += self.prevhash_bin + r += util.ser_uint256_be(merkle_root_int) + r += ntime_bin + r += struct.pack(">I", self.nBits) + r += nonce_bin + return r + + def finalize(self, merkle_root_int, extranonce1_bin, extranonce2_bin, ntime, nonce): + '''Take all parameters required to compile block candidate. + self.is_valid() should return True then...''' + + self.hashMerkleRoot = merkle_root_int + self.nTime = ntime + self.nNonce = nonce + self.vtx[0].set_extranonce(extranonce1_bin + extranonce2_bin) + self.sha256 = None # We changed block parameters, let's reset sha256 cache \ No newline at end of file diff --git a/lib/block_updater.py b/lib/block_updater.py new file mode 100644 index 0000000..3ed44b8 --- /dev/null +++ b/lib/block_updater.py @@ -0,0 +1,64 @@ +from twisted.internet import reactor, defer +from stratum import settings + +import util +from mining.interfaces import Interfaces + +import stratum.logger +log = stratum.logger.get_logger('block_updater') + +class BlockUpdater(object): + ''' + Polls upstream's getinfo() and detecting new block on the network. + This will call registry.update_block when new prevhash appear. + + This is just failback alternative when something + with ./bitcoind -blocknotify will go wrong. + ''' + + def __init__(self, registry, bitcoin_rpc): + self.bitcoin_rpc = bitcoin_rpc + self.registry = registry + self.clock = None + self.schedule() + + def schedule(self): + when = self._get_next_time() + log.debug("Next prevhash update in %.03f sec" % when) + log.debug("Merkle update in next %.03f sec" % \ + ((self.registry.last_update + settings.MERKLE_REFRESH_INTERVAL)-Interfaces.timestamper.time())) + self.clock = reactor.callLater(when, self.run) + + def _get_next_time(self): + when = settings.PREVHASH_REFRESH_INTERVAL - (Interfaces.timestamper.time() - self.registry.last_update) % \ + settings.PREVHASH_REFRESH_INTERVAL + return when + + @defer.inlineCallbacks + def run(self): + update = False + + try: + if self.registry.last_block: + current_prevhash = "%064x" % self.registry.last_block.hashPrevBlock + else: + current_prevhash = None + + prevhash = util.reverse_hash((yield self.bitcoin_rpc.prevhash())) + if prevhash and prevhash != current_prevhash: + log.info("New block! Prevhash: %s" % prevhash) + update = True + + elif Interfaces.timestamper.time() - self.registry.last_update >= settings.MERKLE_REFRESH_INTERVAL: + log.info("Merkle update! Prevhash: %s" % prevhash) + update = True + + if update: + self.registry.update_block() + + except Exception: + log.exception("UpdateWatchdog.run failed") + finally: + self.schedule() + + \ No newline at end of file diff --git a/lib/coinbaser.py b/lib/coinbaser.py new file mode 100644 index 0000000..04b7d80 --- /dev/null +++ b/lib/coinbaser.py @@ -0,0 +1,51 @@ +import util + +import stratum.logger +log = stratum.logger.get_logger('coinbaser') + +# TODO: Add on_* hooks in the app + +class SimpleCoinbaser(object): + '''This very simple coinbaser uses constant bitcoin address + for all generated blocks.''' + + def __init__(self, bitcoin_rpc, address): + self.address = address + self.is_valid = False # We need to check if pool can use this address + + self.bitcoin_rpc = bitcoin_rpc + + self._validate() + + def _validate(self): + d = self.bitcoin_rpc.validateaddress(self.address) + d.addCallback(self._address_check) + d.addErrback(self._failure) + + def _address_check(self, result): + if result['isvalid'] and result['ismine']: + self.is_valid = True + log.info("Coinbase address '%s' is valid" % self.address) + else: + self.is_valid = False + log.error("Coinbase address '%s' is NOT valid!" % self.address) + + def _failure(self, failure): + log.error("Cannot validate Bitcoin address '%s'" % self.address) + raise + + #def on_new_block(self): + # pass + + #def on_new_template(self): + # pass + + def get_script_pubkey(self): + if not self.is_valid: + # Try again, maybe bitcoind was down? + self._validate() + raise Exception("Coinbase address is not validated!") + return util.script_to_address(self.address) + + def get_coinbase_data(self): + return '' \ No newline at end of file diff --git a/lib/coinbasetx.py b/lib/coinbasetx.py new file mode 100644 index 0000000..11c7e36 --- /dev/null +++ b/lib/coinbasetx.py @@ -0,0 +1,49 @@ +import binascii +import halfnode +import struct +import util + +class CoinbaseTransaction(halfnode.CTransaction): + '''Construct special transaction used for coinbase tx. + It also implements quick serialization using pre-cached + scriptSig template.''' + + extranonce_type = '>Q' + extranonce_placeholder = struct.pack(extranonce_type, int('f000000ff111111f', 16)) + extranonce_size = struct.calcsize(extranonce_type) + + def __init__(self, timestamper, coinbaser, value, flags, height, data): + super(CoinbaseTransaction, self).__init__() + + #self.extranonce = 0 + + if len(self.extranonce_placeholder) != self.extranonce_size: + raise Exception("Extranonce placeholder don't match expected length!") + + tx_in = halfnode.CTxIn() + tx_in.prevout.hash = 0L + tx_in.prevout.n = 2**32-1 + tx_in._scriptSig_template = ( + util.ser_number(height) + binascii.unhexlify(flags) + util.ser_number(int(timestamper.time())) + \ + chr(self.extranonce_size), + util.ser_string(coinbaser.get_coinbase_data() + data) + ) + + tx_in.scriptSig = tx_in._scriptSig_template[0] + self.extranonce_placeholder + tx_in._scriptSig_template[1] + + tx_out = halfnode.CTxOut() + tx_out.nValue = value + tx_out.scriptPubKey = coinbaser.get_script_pubkey() + + self.vin.append(tx_in) + self.vout.append(tx_out) + + # Two parts of serialized coinbase, just put part1 + extranonce + part2 to have final serialized tx + self._serialized = super(CoinbaseTransaction, self).serialize().split(self.extranonce_placeholder) + + def set_extranonce(self, extranonce): + if len(extranonce) != self.extranonce_size: + raise Exception("Incorrect extranonce size") + + (part1, part2) = self.vin[0]._scriptSig_template + self.vin[0].scriptSig = part1 + extranonce + part2 \ No newline at end of file diff --git a/lib/extranonce_counter.py b/lib/extranonce_counter.py new file mode 100644 index 0000000..19254f4 --- /dev/null +++ b/lib/extranonce_counter.py @@ -0,0 +1,25 @@ +import struct + +class ExtranonceCounter(object): + '''Implementation of a counter producing + unique extranonce across all pool instances. + This is just dumb "quick&dirty" solution, + but it can be changed at any time without breaking anything.''' + + def __init__(self, instance_id): + if instance_id < 0 or instance_id > 31: + raise Exception("Current ExtranonceCounter implementation needs an instance_id in <0, 31>.") + + # Last 5 most-significant bits represents instance_id + # The rest is just an iterator of jobs. + self.counter = instance_id << 27 + self.size = struct.calcsize('>L') + + def get_size(self): + '''Return expected size of generated extranonce in bytes''' + return self.size + + def get_new_bin(self): + self.counter += 1 + return struct.pack('>L', self.counter) + \ No newline at end of file diff --git a/lib/halfnode.py b/lib/halfnode.py new file mode 100644 index 0000000..233a5c1 --- /dev/null +++ b/lib/halfnode.py @@ -0,0 +1,523 @@ +#!/usr/bin/python +# Public Domain +# Original author: ArtForz +# Twisted integration: slush + +import struct +import socket +import binascii +import time +import sys +import random +import cStringIO +from Crypto.Hash import SHA256 + +from twisted.internet.protocol import Protocol +from util import * + +MY_VERSION = 31402 +MY_SUBVERSION = ".4" + +class CAddress(object): + def __init__(self): + self.nTime = 0 + self.nServices = 1 + self.pchReserved = "\x00" * 10 + "\xff" * 2 + self.ip = "0.0.0.0" + self.port = 0 + def deserialize(self, f): + #self.nTime = struct.unpack("H", f.read(2))[0] + def serialize(self): + r = "" + #r += struct.pack("H", self.port) + return r + def __repr__(self): + return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices, self.ip, self.port) + +class CInv(object): + typemap = { + 0: "Error", + 1: "TX", + 2: "Block"} + def __init__(self): + self.type = 0 + self.hash = 0L + def deserialize(self, f): + self.type = struct.unpack(" 21000000L * 100000000L: + return False + return True + def __repr__(self): + return "CTransaction(nVersion=%i vin=%s vout=%s nLockTime=%i)" % (self.nVersion, repr(self.vin), repr(self.vout), self.nLockTime) + +class CBlock(object): + def __init__(self): + self.nVersion = 1 + self.hashPrevBlock = 0 + self.hashMerkleRoot = 0 + self.nTime = 0 + self.nBits = 0 + self.nNonce = 0 + self.vtx = [] + self.sha256 = None + def deserialize(self, f): + self.nVersion = struct.unpack(" target: + return False + hashes = [] + for tx in self.vtx: + tx.sha256 = None + if not tx.is_valid(): + return False + tx.calc_sha256() + hashes.append(ser_uint256(tx.sha256)) + + while len(hashes) > 1: + newhashes = [] + for i in xrange(0, len(hashes), 2): + i2 = min(i+1, len(hashes)-1) + newhashes.append(SHA256.new(SHA256.new(hashes[i] + hashes[i2]).digest()).digest()) + hashes = newhashes + + if uint256_from_str(hashes[0]) != self.hashMerkleRoot: + return False + return True + def __repr__(self): + return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx)) + +class msg_version(object): + command = "version" + def __init__(self): + self.nVersion = MY_VERSION + self.nServices = 0 + self.nTime = time.time() + self.addrTo = CAddress() + self.addrFrom = CAddress() + self.nNonce = random.getrandbits(64) + self.strSubVer = MY_SUBVERSION + self.nStartingHeight = 0 + + def deserialize(self, f): + self.nVersion = struct.unpack(" +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from hashlib import sha256 +from util import doublesha + +class MerkleTree: + def __init__(self, data, detailed=False): + self.data = data + self.recalculate(detailed) + self._hash_steps = None + + def recalculate(self, detailed=False): + L = self.data + steps = [] + if detailed: + detail = [] + PreL = [] + StartL = 0 + else: + detail = None + PreL = [None] + StartL = 2 + Ll = len(L) + if detailed or Ll > 1: + while True: + if detailed: + detail += L + if Ll == 1: + break + steps.append(L[1]) + if Ll % 2: + L += [L[-1]] + L = PreL + [doublesha(L[i] + L[i + 1]) for i in range(StartL, Ll, 2)] + Ll = len(L) + self._steps = steps + self.detail = detail + + def hash_steps(self): + if self._hash_steps == None: + self._hash_steps = doublesha(''.join(self._steps)) + return self._hash_steps + + def withFirst(self, f): + steps = self._steps + for s in steps: + f = doublesha(f + s) + return f + + def merkleRoot(self): + return self.withFirst(self.data[0]) + +# MerkleTree tests +def _test(): + import binascii + import time + + mt = MerkleTree([None] + [binascii.unhexlify(a) for a in [ + '999d2c8bb6bda0bf784d9ebeb631d711dbbbfe1bc006ea13d6ad0d6a2649a971', + '3f92594d5a3d7b4df29d7dd7c46a0dac39a96e751ba0fc9bab5435ea5e22a19d', + 'a5633f03855f541d8e60a6340fc491d49709dc821f3acb571956a856637adcb6', + '28d97c850eaf917a4c76c02474b05b70a197eaefb468d21c22ed110afe8ec9e0', + ]]) + assert( + b'82293f182d5db07d08acf334a5a907012bbb9990851557ac0ec028116081bd5a' == + binascii.b2a_hex(mt.withFirst(binascii.unhexlify('d43b669fb42cfa84695b844c0402d410213faa4f3e66cb7248f688ff19d5e5f7'))) + ) + + print '82293f182d5db07d08acf334a5a907012bbb9990851557ac0ec028116081bd5a' + txes = [binascii.unhexlify(a) for a in [ + 'd43b669fb42cfa84695b844c0402d410213faa4f3e66cb7248f688ff19d5e5f7', + '999d2c8bb6bda0bf784d9ebeb631d711dbbbfe1bc006ea13d6ad0d6a2649a971', + '3f92594d5a3d7b4df29d7dd7c46a0dac39a96e751ba0fc9bab5435ea5e22a19d', + 'a5633f03855f541d8e60a6340fc491d49709dc821f3acb571956a856637adcb6', + '28d97c850eaf917a4c76c02474b05b70a197eaefb468d21c22ed110afe8ec9e0', + ]] + + s = time.time() + mt = MerkleTree(txes) + for x in range(100): + y = int('d43b669fb42cfa84695b844c0402d410213faa4f3e66cb7248f688ff19d5e5f7', 16) + #y += x + coinbasehash = binascii.unhexlify("%x" % y) + x = binascii.b2a_hex(mt.withFirst(coinbasehash)) + + print x + print time.time() - s + +if __name__ == '__main__': + _test() \ No newline at end of file diff --git a/lib/template_registry.py b/lib/template_registry.py new file mode 100644 index 0000000..4d3f061 --- /dev/null +++ b/lib/template_registry.py @@ -0,0 +1,262 @@ +import weakref +import binascii +import util +import StringIO + +import stratum.logger +log = stratum.logger.get_logger('template_registry') + +from mining.interfaces import Interfaces +from extranonce_counter import ExtranonceCounter + +class JobIdGenerator(object): + '''Generate pseudo-unique job_id. It does not need to be absolutely unique, + because pool sends "clean_jobs" flag to clients and they should drop all previous jobs.''' + counter = 0 + + @classmethod + def get_new_id(cls): + cls.counter += 1 + if cls.counter % 0xffff == 0: + cls.counter = 1 + return "%x" % cls.counter + +class TemplateRegistry(object): + '''Implements the main logic of the pool. Keep track + on valid block templates, provide internal interface for stratum + service and implements block validation and submits.''' + + def __init__(self, block_template_class, coinbaser, bitcoin_rpc, instance_id, + on_block_callback): + self.prevhashes = {} + self.jobs = weakref.WeakValueDictionary() + + self.extranonce_counter = ExtranonceCounter(instance_id) + self.extranonce2_size = block_template_class.coinbase_transaction_class.extranonce_size \ + - self.extranonce_counter.get_size() + + self.coinbaser = coinbaser + self.block_template_class = block_template_class + self.bitcoin_rpc = bitcoin_rpc + self.on_block_callback = on_block_callback + + self.last_block = None + self.update_in_progress = False + self.last_update = None + + # Create first block template on startup + self.update_block() + + def get_new_extranonce1(self): + '''Generates unique extranonce1 (e.g. for newly + subscribed connection.''' + return self.extranonce_counter.get_new_bin() + + def get_last_broadcast_args(self): + '''Returns arguments for mining.notify + from last known template.''' + return self.last_block.broadcast_args + + def add_template(self, block): + '''Adds new template to the registry. + It also clean up templates which should + not be used anymore.''' + + prevhash = block.prevhash_hex + + if prevhash in self.prevhashes.keys(): + new_block = False + else: + new_block = True + self.prevhashes[prevhash] = [] + + # Blocks sorted by prevhash, so it's easy to drop + # them on blockchain update + self.prevhashes[prevhash].append(block) + + # Weak reference for fast lookup using job_id + self.jobs[block.job_id] = block + + # Use this template for every new request + self.last_block = block + + # Drop templates of obsolete blocks + for ph in self.prevhashes.keys(): + if ph != prevhash: + del self.prevhashes[ph] + + log.info("New template for %s" % prevhash) + self.on_block_callback(new_block) + #from twisted.internet import reactor + #reactor.callLater(10, self.on_block_callback, new_block) + + def update_block(self): + '''Registry calls the getblocktemplate() RPC + and build new block template.''' + + if self.update_in_progress: + # Block has been already detected + return + + self.update_in_progress = True + self.last_update = Interfaces.timestamper.time() + + d = self.bitcoin_rpc.getblocktemplate() + d.addCallback(self._update_block) + d.addErrback(self._update_block_failed) + + def _update_block_failed(self, failure): + log.error(str(failure)) + self.update_in_progress = False + + def _update_block(self, data): + start = Interfaces.timestamper.time() + + template = self.block_template_class(Interfaces.timestamper, self.coinbaser, JobIdGenerator.get_new_id()) + template.fill_from_rpc(data) + self.add_template(template) + + log.info("Update finished, %.03f sec, %d txes" % \ + (Interfaces.timestamper.time() - start, len(template.vtx))) + + self.update_in_progress = False + return data + + def diff_to_target(self, difficulty): + '''Converts difficulty to target''' + diff1 = 0x00000000ffff0000000000000000000000000000000000000000000000000000 + return diff1 / difficulty + + def get_job(self, job_id): + '''For given job_id returns BlockTemplate instance or None''' + try: + j = self.jobs[job_id] + except: + log.info("Job id '%s' not found" % job_id) + return False + + # Now we have to check if job is still valid. + # Unfortunately weak references are not bulletproof and + # old reference can be found until next run of garbage collector. + if j.prevhash_hex not in self.prevhashes: + log.info("Prevhash of job '%s' is unknown" % job_id) + return False + + if j not in self.prevhashes[j.prevhash_hex]: + log.info("Job %s is unknown" % job_id) + return False + + return True + + def submit_share(self, job_id, worker_name, extranonce1_bin, extranonce2, ntime, nonce, + difficulty, submitblock_callback): + '''Check parameters and finalize block template. If it leads + to valid block candidate, asynchronously submits the block + back to the bitcoin network. + + - extranonce1_bin is binary. No checks performed, it should be from session data + - job_id, extranonce2, ntime, nonce - in hex form sent by the client + - difficulty - decimal number from session, again no checks performed + - submitblock_callback - reference to method which receive result of submitblock() + ''' + + # Check if extranonce2 looks correctly. extranonce2 is in hex form... + if len(extranonce2) != self.extranonce2_size * 2: + return (False, "Incorrect size of extranonce2. Expected %d chars" % (self.extranonce2_size*2), None, None) + + # Check for job + if not self.get_job(job_id): + return (False, "Job '%s' not found" % job_id, None, None) + + try: + job = self.jobs[job_id] + except KeyError: + return (False, "Job '%s' not found" % job_id, None, None) + + # Check if ntime looks correct + if len(ntime) != 8: + return (False, "Incorrect size of ntime. Expected 8 chars", None, None) + + if not job.check_ntime(int(ntime, 16)): + return (False, "Ntime out of range", None, None) + + # Check nonce + if len(nonce) != 8: + return (False, "Incorrect size of nonce. Expected 8 chars", None, None) + + # Check for duplicated submit + if not job.register_submit(extranonce1_bin, extranonce2, ntime, nonce): + log.info("Duplicate from %s, (%s %s %s %s)" % \ + (worker_name, binascii.hexlify(extranonce1_bin), extranonce2, ntime, nonce)) + return (False, "Duplicate share", None, None) + + # Now let's do the hard work! + # --------------------------- + + # 0. Some sugar + extranonce2_bin = binascii.unhexlify(extranonce2) + ntime_bin = binascii.unhexlify(ntime) + nonce_bin = binascii.unhexlify(nonce) + + # 1. Build coinbase + coinbase_bin = job.serialize_coinbase(extranonce1_bin, extranonce2_bin) + coinbase_hash = util.doublesha(coinbase_bin) + + # 2. Calculate merkle root + merkle_root_bin = job.merkletree.withFirst(coinbase_hash) + merkle_root_int = util.uint256_from_str(merkle_root_bin) + + # 3. Serialize header with given merkle, ntime and nonce + header_bin = job.serialize_header(merkle_root_int, ntime_bin, nonce_bin) + + # 4. Reverse header and compare it with target of the user + hash_bin = util.doublesha(''.join([ header_bin[i*4:i*4+4][::-1] for i in range(0, 20) ])) + hash_int = util.uint256_from_str(hash_bin) + block_hash_hex = "%064x" % hash_int + header_hex = binascii.hexlify(header_bin) + + target = self.diff_to_target(difficulty) + + if hash_int > target: + return (False, "Share is above target", None, None) + + # 5. Compare hash with target of the network + if hash_int <= job.target: + # Yay! It is block candidate! + log.info("We found a block candidate! %s" % block_hash_hex) + + # 6. Finalize and serialize block object + job.finalize(merkle_root_int, extranonce1_bin, extranonce2_bin, int(ntime, 16), int(nonce, 16)) + + if not job.is_valid(): + # Should not happen + log.error("Final job validation failed!") + return (False, 'Job validation failed', header_hex, block_hash_hex) + + # 7. Submit block to the network + submit_time = Interfaces.timestamper.time() + serialized = binascii.hexlify(job.serialize()) + d = self.bitcoin_rpc.submitblock(serialized) + + # Submit is lazy, we don't need to wait for the result + # Callback will just register success or failure to share manager + d.addCallback(self._on_submitblock, submitblock_callback, + worker_name, header_hex, block_hash_hex, submit_time) + d.addErrback(self._on_submitblock_failure, submitblock_callback, + worker_name, header_hex, block_hash_hex, submit_time) + + return (True, '', header_hex, block_hash_hex) + + return (True, '', header_hex, block_hash_hex) + + def _on_submitblock(self, is_accepted, callback, worker_name, block_header, block_hash, timestamp): + '''Helper method, bridges call from deferred to method reference given in submit()''' + # Forward submitblock result to share manager + callback(worker_name, block_header, block_hash, timestamp, is_accepted) + return is_accepted + + def _on_submitblock_failure(self, failure, callback, worker_name, block_header, block_hash, timestamp): + '''Helper method, bridges call from deferred to method reference given in submit()''' + # Forward submitblock failure to share manager + callback(worker_name, block_header, block_hash, timestamp, False) + log.exception(failure) \ No newline at end of file diff --git a/lib/util.py b/lib/util.py new file mode 100644 index 0000000..3e88fd8 --- /dev/null +++ b/lib/util.py @@ -0,0 +1,201 @@ +'''Various helper methods. It probably needs some cleanup.''' + +import struct +import StringIO +import binascii +from hashlib import sha256 + +def deser_string(f): + nit = struct.unpack(">= 32 + return rs + +def uint256_from_str(s): + r = 0L + t = struct.unpack("IIIIIIII", s[:32]) + for i in xrange(8): + r += t[i] << (i * 32) + return r + +def uint256_from_compact(c): + nbytes = (c >> 24) & 0xFF + v = (c & 0xFFFFFFL) << (8 * (nbytes - 3)) + return v + +def deser_vector(f, c): + nit = struct.unpack("= 256: + div, mod = divmod(long_value, 256) + result = chr(mod) + result + long_value = div + result = chr(long_value) + result + + nPad = 0 + for c in v: + if c == __b58chars[0]: nPad += 1 + else: break + + result = chr(0)*nPad + result + if length is not None and len(result) != length: + return None + + return result + +def reverse_hash(h): + # This only revert byte order, nothing more + if len(h) != 64: + raise Exception('hash must have 64 hexa chars') + + return ''.join([ h[56-i:64-i] for i in range(0, 64, 8) ]) + +def doublesha(b): + return sha256(sha256(b).digest()).digest() + +def bits_to_target(bits): + return struct.unpack('I", u & 0xFFFFFFFFL) + u >>= 32 + return rs + +def deser_uint256_be(f): + r = 0L + for i in xrange(8): + t = struct.unpack(">I", f.read(4))[0] + r += t << (i * 32) + return r + +def ser_number(num): + # For encoding nHeight into coinbase + d = struct.pack("= ntime provided by mining,notify and <= current time'), + ('nonce', 'string', '32bit integer, hex-encoded, big-endian'),] + \ No newline at end of file diff --git a/mining/subscription.py b/mining/subscription.py new file mode 100644 index 0000000..cbbeea5 --- /dev/null +++ b/mining/subscription.py @@ -0,0 +1,53 @@ +from stratum.pubsub import Pubsub, Subscription +from mining.interfaces import Interfaces + +import stratum.logger +log = stratum.logger.get_logger('subscription') + +class MiningSubscription(Subscription): + '''This subscription object implements + logic for broadcasting new jobs to the clients.''' + + event = 'mining.notify' + + @classmethod + def on_block(cls, is_new_block): + '''This is called when TemplateRegistry registers + new block which we have to broadcast clients.''' + + start = Interfaces.timestamper.time() + + clean_jobs = is_new_block + (job_id, prevhash, coinb1, coinb2, merkle_branch, version, nbits, ntime, clean_jobs) = \ + Interfaces.template_registry.get_last_broadcast_args() + + # Push new job to subscribed clients + cls.emit(job_id, prevhash, coinb1, coinb2, merkle_branch, version, nbits, ntime, clean_jobs) + + cnt = Pubsub.get_subscription_count(cls.event) + log.info("BROADCASTED to %d connections in %.03f sec" % (cnt, (Interfaces.timestamper.time() - start))) + + def _finish_after_subscribe(self, result): + '''Send new job to newly subscribed client''' + try: + (job_id, prevhash, coinb1, coinb2, merkle_branch, version, nbits, ntime, clean_jobs) = \ + Interfaces.template_registry.get_last_broadcast_args() + except Exception: + log.error("Template not ready yet") + return result + + # Force set higher difficulty + # TODO + #self.connection_ref().rpc('mining.set_difficulty', [2,], is_notification=True) + + # Force client to remove previous jobs if any (eg. from previous connection) + clean_jobs = True + self.emit_single(job_id, prevhash, coinb1, coinb2, merkle_branch, version, nbits, ntime, clean_jobs) + + return result + + def after_subscribe(self, *args): + '''This will send new job to the client *after* he receive subscription details. + on_finish callback solve the issue that job is broadcasted *during* + the subscription request and client receive messages in wrong order.''' + self.connection_ref().on_finish.addCallback(self._finish_after_subscribe) \ No newline at end of file -- 1.7.1