X-Git-Url: https://git.novaco.in/?a=blobdiff_plain;f=server.py;h=973f932f8edca0f92548ff6975da24843491c1ea;hb=2b521da6a9f9fb386706d86a3d412986c58eba18;hp=693e45cbc7db023efcad5ad49e4008ff355133d1;hpb=fdeb0fddcbe2cfafb80febeac1e4c3f35ceb6ee3;p=electrum-server.git diff --git a/server.py b/server.py index 693e45c..973f932 100755 --- a/server.py +++ b/server.py @@ -19,27 +19,38 @@ Todo: * server should check and return bitcoind status.. * improve txpoint sorting + * command to check cache + + mempool transactions do not need to be added to the database; it slows it down """ import time, socket, operator, thread, ast, sys,re import psycopg2, binascii -import bitcoinrpc from Abe.abe import hash_to_address, decode_check_address from Abe.DataStore import DataStore as Datastore_class from Abe import DataStore, readconf, BCDataStream, deserialize, util, base58 import ConfigParser +from json import dumps, loads +import urllib + +# we need to import electrum +sys.path.append('../client/') +from wallet import Wallet +from interface import Interface + config = ConfigParser.ConfigParser() # set some defaults, which will be overwritten by the config file config.add_section('server') config.set('server','banner', 'Welcome to Electrum!') -config.set('server', 'host', 'ecdsa.org') +config.set('server', 'host', 'localhost') config.set('server', 'port', 50000) config.set('server', 'password', '') config.set('server', 'irc', 'yes') +config.set('server', 'cache', 'no') config.set('server', 'ircname', 'Electrum server') config.add_section('database') config.set('database', 'type', 'psycopg2') @@ -50,29 +61,52 @@ try: config.readfp(f) f.close() except: - print "Could not read electrum.conf. I will use the dafault values." + print "Could not read electrum.conf. I will use the default values." + +password = config.get('server','password') +bitcoind_url = 'http://%s:%s@%s:%s/' % ( config.get('bitcoind','user'), config.get('bitcoind','password'), config.get('bitcoind','host'), config.get('bitcoind','port')) stopping = False block_number = -1 sessions = {} -sessions_last_time = {} dblock = thread.allocate_lock() - peer_list = {} +wallets = {} # for ultra-light clients such as bccapi + class MyStore(Datastore_class): - def safe_sql(self,sql, params=()): + def import_tx(self, tx, is_coinbase): + tx_id = super(MyStore, self).import_tx(tx, is_coinbase) + if config.get('server', 'cache') == 'yes': self.update_tx_cache(tx_id) + + def update_tx_cache(self, txid): + inrows = self.get_tx_inputs(txid, False) + for row in inrows: + _hash = store.binout(row[6]) + address = hash_to_address(chr(0), _hash) + if self.tx_cache.has_key(address): + print "cache: invalidating", address + self.tx_cache.pop(address) + outrows = self.get_tx_outputs(txid, False) + for row in outrows: + _hash = store.binout(row[6]) + address = hash_to_address(chr(0), _hash) + if self.tx_cache.has_key(address): + print "cache: invalidating", address + self.tx_cache.pop(address) + + def safe_sql(self,sql, params=(), lock=True): try: - dblock.acquire() + if lock: dblock.acquire() ret = self.selectall(sql,params) - dblock.release() + if lock: dblock.release() return ret except: print "sql error", sql return [] - def get_tx_outputs(self, tx_id): + def get_tx_outputs(self, tx_id, lock=True): return self.safe_sql("""SELECT txout.txout_pos, txout.txout_scriptPubKey, @@ -87,9 +121,9 @@ class MyStore(Datastore_class): LEFT JOIN tx nexttx ON (txin.tx_id = nexttx.tx_id) WHERE txout.tx_id = %d ORDER BY txout.txout_pos - """%(tx_id)) + """%(tx_id), (), lock) - def get_tx_inputs(self, tx_id): + def get_tx_inputs(self, tx_id, lock=True): return self.safe_sql(""" SELECT txin.txin_pos, txin.txin_scriptSig, @@ -105,7 +139,7 @@ class MyStore(Datastore_class): LEFT JOIN unlinked_txin u ON (u.txin_id = txin.txin_id) WHERE txin.tx_id = %d ORDER BY txin.txin_pos - """%(tx_id,)) + """%(tx_id,), (), lock) def get_address_out_rows(self, dbhash): return self.safe_sql(""" SELECT @@ -173,10 +207,17 @@ class MyStore(Datastore_class): JOIN pubkey ON (pubkey.pubkey_id = txout.pubkey_id) WHERE pubkey.pubkey_hash = ? """, (dbhash,)) - def get_txpoints(self, addr): + def get_history(self, addr): + + if config.get('server','cache') == 'yes': + cached_version = self.tx_cache.get( addr ) + if cached_version is not None: + return cached_version + version, binaddr = decode_check_address(addr) if binaddr is None: - return "err" + return None + dbhash = self.binin(binaddr) rows = [] rows += self.get_address_out_rows( dbhash ) @@ -215,18 +256,28 @@ class MyStore(Datastore_class): rows = [] rows += self.get_address_in_rows_memorypool( dbhash ) rows += self.get_address_out_rows_memorypool( dbhash ) + address_has_mempool = False + for row in rows: is_in, tx_hash, tx_id, pos, value = row tx_hash = self.hashout_hex(tx_hash) if tx_hash in known_tx: continue + + # this means that pending transactions were added to the db, even if they are not returned by getmemorypool + address_has_mempool = True + + # this means pending transactions are returned by getmemorypool + if tx_hash not in self.mempool_keys: + continue + #print "mempool", tx_hash txpoint = { "nTime": 0, - "chain_id": 1, + #"chain_id": 1, "height": 0, "is_in": int(is_in), - "blk_hash": 'mempool', + "blk_hash": 'mempool', "tx_hash": tx_hash, "tx_id": int(tx_id), "pos": int(pos), @@ -266,29 +317,132 @@ class MyStore(Datastore_class): if row: if not row[4]: txpoint['raw_scriptPubKey'] = row[1] - + # cache result + if config.get('server','cache') == 'yes' and not address_has_mempool: + self.tx_cache[addr] = txpoints + return txpoints - def get_status(self, addr): - # last block for an address - tx_points = self.get_txpoints(addr) - if not tx_points: - return None - else: - return tx_points[-1]['blk_hash'] + +class Direct_Interface(Interface): + def __init__(self): + pass + + def handler(self, method, params = ''): + cmds = {'session.new':new_session, + 'session.poll':poll_session, + 'session.update':update_session, + 'blockchain.transaction.broadcast':send_tx, + 'blockchain.address.get_history':store.get_history + } + func = cmds[method] + return func( params ) + def send_tx(tx): - import bitcoinrpc - conn = bitcoinrpc.connect_to_local() - try: - v = conn.importtransaction(tx) - except: - v = "error: transaction rejected by memorypool" - return v + postdata = dumps({"method": 'importtransaction', 'params': [tx], 'id':'jsonrpc'}) + respdata = urllib.urlopen(bitcoind_url, postdata).read() + r = loads(respdata) + if r['error'] != None: + out = "error: transaction rejected by memorypool" + else: + out = r['result'] + return out + +def random_string(N): + import random, string + return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(N)) + + + +def cmd_stop(data): + global stopping + if password == data: + stopping = True + return 'ok' + else: + return 'wrong password' + +def cmd_load(pw): + if password == pw: + return repr( len(sessions) ) + else: + return 'wrong password' + + +def clear_cache(pw): + if password == pw: + store.tx_cache = {} + return 'ok' + else: + return 'wrong password' + +def get_cache(pw,addr): + if password == pw: + return store.tx_cache.get(addr) + else: + return 'wrong password' + + +def poll_session(session_id): + session = sessions.get(session_id) + if session is None: + print time.asctime(), "session not found", session_id + out = repr( (-1, {})) + else: + t1 = time.time() + addresses = session['addresses'] + session['last_time'] = time.time() + ret = {} + k = 0 + for addr in addresses: + if store.tx_cache.get( addr ) is not None: k += 1 + + # get addtess status, i.e. the last block for that address. + tx_points = store.get_history(addr) + if not tx_points: + status = None + else: + lastpoint = tx_points[-1] + status = lastpoint['blk_hash'] + # this is a temporary hack; move it up once old clients have disappeared + if status == 'mempool' and session['version'] != "old": + status = status + ':%d'% len(tx_points) + + last_status = addresses.get( addr ) + if last_status != status: + addresses[addr] = status + ret[addr] = status + if ret: + sessions[session_id]['addresses'] = addresses + out = repr( (block_number, ret ) ) + t2 = time.time() - t1 + if t2 > 10: + print "high load:", session_id, "%d/%d"%(k,len(addresses)), t2 + + return out + + +def new_session(version, addresses): + session_id = random_string(10) + sessions[session_id] = { 'addresses':{}, 'version':version } + for a in addresses: + sessions[session_id]['addresses'][a] = '' + out = repr( (session_id, config.get('server','banner').replace('\\n','\n') ) ) + sessions[session_id]['last_time'] = time.time() + return out + +def update_session(session_id,addresses): + sessions[session_id]['addresses'] = {} + for a in addresses: + sessions[session_id]['addresses'][a] = '' + sessions[session_id]['last_time'] = time.time() + return 'ok' + def listen_thread(store): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -298,9 +452,7 @@ def listen_thread(store): conn, addr = s.accept() thread.start_new_thread(client_thread, (addr, conn,)) -def random_string(N): - import random, string - return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(N)) + def client_thread(ipaddr,conn): #print "client thread", ipaddr @@ -310,83 +462,19 @@ def client_thread(ipaddr,conn): while 1: d = conn.recv(1024) msg += d - if d[-1]=='#': + if not d: + break + if '#' in msg: + msg = msg.split('#', 1)[0] break - - #print msg - try: - cmd, data = ast.literal_eval(msg[:-1]) + cmd, data = ast.literal_eval(msg) except: - print "syntax error", repr(msg) + print "syntax error", repr(msg), ipaddr conn.close() return - if cmd=='b': - out = "%d"%block_number - elif cmd=='session': - session_id = random_string(10) - try: - addresses = ast.literal_eval(data) - except: - print "error" - conn.close() - return - - print time.asctime(), "session", ipaddr, session_id, addresses[0], len(addresses) - - sessions[session_id] = {} - for a in addresses: - sessions[session_id][a] = '' - out = repr( (session_id, config.get('server','banner').replace('\\n','\n') ) ) - sessions_last_time[session_id] = time.time() - - elif cmd=='poll': - session_id = data - addresses = sessions.get(session_id) - if not addresses: - print "session not found", ipaddr - out = repr( (-1, {})) - else: - sessions_last_time[session_id] = time.time() - ret = {} - for addr in addresses: - status = store.get_status( addr ) - last_status = sessions[session_id].get( addr ) - if last_status != status: - sessions[session_id][addr] = status - ret[addr] = status - out = repr( (block_number, ret ) ) - - elif cmd == 'h': - # history - addr = data - h = store.get_txpoints( addr ) - out = repr(h) - - elif cmd == 'load': - if config.get('server','password') == data: - out = repr( len(sessions) ) - else: - out = 'wrong password' - - elif cmd =='tx': - out = send_tx(data) - - elif cmd == 'stop': - global stopping - if config.get('server','password') == data: - stopping = True - out = 'ok' - else: - out = 'wrong password' - - elif cmd == 'peers': - out = repr(peer_list.values()) - - else: - out = None - + out = do_command(cmd, data, ipaddr) if out: #print ipaddr, cmd, len(out) try: @@ -396,33 +484,138 @@ def client_thread(ipaddr,conn): finally: conn.close() - -ds = BCDataStream.BCDataStream() + +def do_command(cmd, data, ipaddr): + + timestr = time.strftime("[%d/%m/%Y-%H:%M:%S]") + + if cmd=='b': + out = "%d"%block_number + + elif cmd in ['session','new_session']: + try: + if cmd == 'session': + addresses = ast.literal_eval(data) + version = "old" + else: + version, addresses = ast.literal_eval(data) + if version[0]=="0": version = "v" + version + except: + print "error", data + return None + print timestr, "new session", ipaddr, addresses[0] if addresses else addresses, len(addresses), version + out = new_session(version, addresses) + + elif cmd=='update_session': + try: + session_id, addresses = ast.literal_eval(data) + except: + print "error" + return None + print timestr, "update session", ipaddr, addresses[0] if addresses else addresses, len(addresses) + out = update_session(session_id,addresses) + + elif cmd == 'bccapi_login': + import electrum + print "data",data + v, k = ast.literal_eval(data) + master_public_key = k.decode('hex') # todo: sanitize. no need to decode twice... + print master_public_key + wallet_id = random_string(10) + w = Wallet( Direct_Interface() ) + w.master_public_key = master_public_key.decode('hex') + w.synchronize() + wallets[wallet_id] = w + out = wallet_id + print "wallets", wallets + + elif cmd == 'bccapi_getAccountInfo': + from wallet import int_to_hex + v, wallet_id = ast.literal_eval(data) + w = wallets.get(wallet_id) + if w is not None: + num = len(w.addresses) + c, u = w.get_balance() + out = int_to_hex(num,4) + int_to_hex(c,8) + int_to_hex( c+u, 8 ) + out = out.decode('hex') + else: + print "error",data + out = "error" + + elif cmd == 'bccapi_getAccountStatement': + from wallet import int_to_hex + v, wallet_id = ast.literal_eval(data) + w = wallets.get(wallet_id) + if w is not None: + num = len(w.addresses) + c, u = w.get_balance() + total_records = num_records = 0 + out = int_to_hex(num,4) + int_to_hex(c,8) + int_to_hex( c+u, 8 ) + int_to_hex( total_records ) + int_to_hex( num_records ) + out = out.decode('hex') + else: + print "error",data + out = "error" + + elif cmd == 'bccapi_getSendCoinForm': + out = '' + + elif cmd == 'bccapi_submitTransaction': + out = '' + + elif cmd=='poll': + out = poll_session(data) + + elif cmd == 'h': + # history + address = data + out = repr( store.get_history( address ) ) + + elif cmd == 'load': + out = cmd_load(data) + + elif cmd =='tx': + out = send_tx(data) + print timestr, "sent tx:", ipaddr, out + + elif cmd == 'stop': + out = cmd_stop(data) + + elif cmd == 'peers': + out = repr(peer_list.values()) + + else: + out = None + + return out + + def memorypool_update(store): + ds = BCDataStream.BCDataStream() + store.mempool_keys = [] - conn = bitcoinrpc.connect_to_local() - try: - v = conn.getmemorypool() - except: - print "cannot contact bitcoin daemmon" + postdata = dumps({"method": 'getmemorypool', 'params': [], 'id':'jsonrpc'}) + respdata = urllib.urlopen(bitcoind_url, postdata).read() + r = loads(respdata) + if r['error'] != None: return - v = v['transactions'] + + v = r['result'].get('transactions') for hextx in v: ds.clear() ds.write(hextx.decode('hex')) tx = deserialize.parse_Transaction(ds) tx['hash'] = util.double_sha256(tx['tx']) - + tx_hash = tx['hash'][::-1].encode('hex') + store.mempool_keys.append(tx_hash) if store.tx_find_id_and_value(tx): pass else: store.import_tx(tx, False) - #print tx['hash'][::-1].encode('hex') - store.commit() + store.commit() @@ -430,11 +623,10 @@ def clean_session_thread(): while not stopping: time.sleep(30) t = time.time() - for k,t0 in sessions_last_time.items(): - if t - t0 > 60: - print "lost session",k + for k,s in sessions.items(): + t0 = s['last_time'] + if t - t0 > 5*60: sessions.pop(k) - sessions_last_time.pop(k) def irc_thread(): @@ -468,9 +660,10 @@ def irc_thread(): name = line[k+6] host = line[k+9] peer_list[name] = (ip,host) - elif time.time() - t > 5*60: + if time.time() - t > 5*60: s.send('NAMES #electrum\n') t = time.time() + peer_list = {} except: traceback.print_exc(file=sys.stdout) finally: @@ -478,34 +671,59 @@ def irc_thread(): s.close() + +def jsonrpc_thread(store): + # see http://code.google.com/p/jsonrpclib/ + from SocketServer import ThreadingMixIn + from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + class SimpleThreadedJSONRPCServer(ThreadingMixIn, SimpleJSONRPCServer): pass + server = SimpleThreadedJSONRPCServer(( config.get('server','host'), 8081)) + server.register_function(lambda : peer_list.values(), 'peers') + server.register_function(cmd_stop, 'stop') + server.register_function(cmd_load, 'load') + server.register_function(lambda : block_number, 'blocks') + server.register_function(clear_cache, 'clear_cache') + server.register_function(get_cache, 'get_cache') + server.register_function(send_tx, 'blockchain.transaction.broadcast') + server.register_function(store.get_history, 'blockchain.address.get_history') + server.register_function(new_session, 'session.new') + server.register_function(update_session, 'session.update') + server.register_function(poll_session, 'session.poll') + server.serve_forever() + + import traceback if __name__ == '__main__': if len(sys.argv)>1: + import jsonrpclib + server = jsonrpclib.Server('http://%s:8081'%config.get('server','host')) cmd = sys.argv[1] if cmd == 'load': - request = "('load','%s')#"%config.get('server','password') + out = server.load(password) elif cmd == 'peers': - request = "('peers','')#" + out = server.peers() elif cmd == 'stop': - request = "('stop','%s')#"%config.get('server','password') - - s = socket.socket( socket.AF_INET, socket.SOCK_STREAM) - s.connect((config.get('server','host'), config.getint('server','port'))) - s.send( request ) - out = '' - while 1: - msg = s.recv(1024) - if msg: out += msg - else: break - s.close() + out = server.stop(password) + elif cmd == 'clear_cache': + out = server.clear_cache(password) + elif cmd == 'get_cache': + out = server.get_cache(password,sys.argv[2]) + elif cmd == 'h': + out = server.blockchain.address.get_history(sys.argv[2]) + elif cmd == 'tx': + out = server.blockchain.transaction.broadcast(sys.argv[2]) + elif cmd == 'b': + out = server.blocks() print out sys.exit(0) print "starting Electrum server" + print "cache:", config.get('server', 'cache') + conf = DataStore.CONFIG_DEFAULTS args, argv = readconf.parse_argv( [], conf) args.dbtype= config.get('database','type') @@ -516,8 +734,11 @@ if __name__ == '__main__': elif args.dbtype == 'psycopg2': args.connect_args = { 'database' : config.get('database','database') } store = MyStore(args) + store.tx_cache = {} + store.mempool_keys = {} thread.start_new_thread(listen_thread, (store,)) + thread.start_new_thread(jsonrpc_thread, (store,)) thread.start_new_thread(clean_session_thread, ()) if (config.get('server','irc') == 'yes' ): thread.start_new_thread(irc_thread, ()) @@ -528,9 +749,14 @@ if __name__ == '__main__': store.catch_up() memorypool_update(store) block_number = store.get_block_number(1) - dblock.release() + except IOError: + print "IOError: cannot reach bitcoind" + block_number = -1 except: traceback.print_exc(file=sys.stdout) + block_number = -1 + finally: + dblock.release() time.sleep(10) print "server stopped"