X-Git-Url: https://git.novaco.in/?p=electrum-server.git;a=blobdiff_plain;f=processor.py;h=0bdb994564060388d346e1fe46b09bfd6b61b7d7;hp=dfe61285ba17a63206aad8821086aa87284cade4;hb=4ce69b7ea24ead59ebbcc7ed335ea9762ae3724b;hpb=827d26f24ef2148c4eb22bab764a7b2798233334 diff --git a/processor.py b/processor.py index dfe6128..0bdb994 100644 --- a/processor.py +++ b/processor.py @@ -1,26 +1,23 @@ import json +import Queue as queue import socket import threading import time -import traceback, sys -import Queue as queue - -def random_string(N): - import random, string - return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(N)) +import traceback +import sys -def timestr(): - return time.strftime("[%d/%m/%Y-%H:%M:%S]") +from utils import random_string, timestr, print_log class Shared: - def __init__(self): + def __init__(self, config): self.lock = threading.Lock() self._stopped = False + self.config = config def stop(self): - print "Stopping Stratum" + print_log("Stopping Stratum") with self.lock: self._stopped = True @@ -37,32 +34,31 @@ class Processor(threading.Thread): self.dispatcher = None self.queue = queue.Queue() - def process(self, request): + def process(self, session, request): pass - def add_request(self, request): - self.queue.put(request) + def add_request(self, session, request): + self.queue.put((session, request)) - def push_response(self, response): + def push_response(self, session, response): #print "response", response - self.dispatcher.request_dispatcher.push_response(response) + self.dispatcher.request_dispatcher.push_response(session, response) def run(self): while not self.shared.stopped(): - request = self.queue.get(10000000000) + request, session = self.queue.get(10000000000) try: - self.process(request) + self.process(request, session) except: traceback.print_exc(file=sys.stdout) - print "processor terminating" - + print_log("processor terminating") class Dispatcher: - def __init__(self): - self.shared = Shared() + def __init__(self, config): + self.shared = Shared(config) self.request_dispatcher = RequestDispatcher(self.shared) self.request_dispatcher.start() self.response_dispatcher = \ @@ -76,7 +72,6 @@ class Dispatcher: self.request_dispatcher.processors[prefix] = processor - class RequestDispatcher(threading.Thread): def __init__(self, shared): @@ -85,50 +80,44 @@ class RequestDispatcher(threading.Thread): self.daemon = True self.request_queue = queue.Queue() self.response_queue = queue.Queue() - self.internal_ids = {} - self.internal_id = 1 self.lock = threading.Lock() - self.sessions = [] + self.idlock = threading.Lock() + self.sessions = {} self.processors = {} - def push_response(self, item): - self.response_queue.put(item) + def push_response(self, session, item): + self.response_queue.put((session, item)) def pop_response(self): return self.response_queue.get() def push_request(self, session, item): - self.request_queue.put((session,item)) + self.request_queue.put((session, item)) def pop_request(self): return self.request_queue.get() def get_session_by_address(self, address): - for x in self.sessions: + for x in self.sessions.values(): if x.address == address: return x - def get_session_id(self, internal_id): - with self.lock: - return self.internal_ids.pop(internal_id) - - def store_session_id(self, session, msgid): - with self.lock: - self.internal_ids[self.internal_id] = session, msgid - r = self.internal_id - self.internal_id += 1 - return r - def run(self): if self.shared is None: raise TypeError("self.shared not set in Processor") + + lastgc = 0 + while not self.shared.stopped(): session, request = self.pop_request() try: self.do_dispatch(session, request) except: traceback.print_exc(file=sys.stdout) - + + if time.time() - lastgc > 60.0: + self.collect_garbage() + lastgc = time.time() self.stop() @@ -139,33 +128,21 @@ class RequestDispatcher(threading.Thread): """ dispatch request to the relevant processor """ method = request['method'] - params = request.get('params',[]) + params = request.get('params', []) suffix = method.split('.')[-1] - is_new = session.protocol_version >= 0.5 - - if is_new and method == 'blockchain.address.get_history': - method = 'blockchain.address.get_history2' - request['method'] = method - - if suffix == 'subscribe': - if is_new and method == 'blockchain.address.subscribe': - method = 'blockchain.address.subscribe2' - request['method'] = method - - session.subscribe_to_service(method, params) - - # store session and id locally - request['id'] = self.store_session_id(session, request['id']) + if session is not None: + if suffix == 'subscribe': + session.subscribe_to_service(method, params) prefix = request['method'].split('.')[0] try: p = self.processors[prefix] except: - print "error: no processor for", prefix + print_log("error: no processor for", prefix) return - p.add_request(request) + p.add_request(session, request) if method in ['server.version']: session.version = params[0] @@ -174,31 +151,35 @@ class RequestDispatcher(threading.Thread): except: pass + def get_sessions(self): with self.lock: - r = self.sessions[:] + r = self.sessions.values() return r def add_session(self, session): + key = session.key() with self.lock: - self.sessions.append(session) + self.sessions[key] = session - def collect_garbage(self): - # Deep copy entire sessions list and blank it - # This is done to minimise lock contention + def remove_session(self, session): + key = session.key() with self.lock: - sessions = self.sessions[:] - self.sessions = [] - for session in sessions: - if not session.stopped(): - # If session is still alive then re-add it back - # to our internal register - self.add_session(session) + self.sessions.pop(key) + + def collect_garbage(self): + now = time.time() + for session in self.sessions.values(): + if (now - session.time) > session.timeout: + session.stop() + class Session: - def __init__(self): + def __init__(self, dispatcher): + self.dispatcher = dispatcher + self.bp = self.dispatcher.processors['blockchain'] self._stopped = False self.lock = threading.Lock() self.subscriptions = [] @@ -209,6 +190,11 @@ class Session: self.time = time.time() threading.Timer(2, self.info).start() + + def key(self): + return self.name + self.address + + # Debugging method. Doesn't need to be threadsafe. def info(self): for sub in self.subscriptions: @@ -221,82 +207,79 @@ class Session: addr = None if self.subscriptions: - print timestr(), self.name, self.address, addr,\ - len(self.subscriptions), self.version + print_log("%4s" % self.name, + "%15s" % self.address, + "%35s" % addr, + "%3d" % len(self.subscriptions), + self.version) + + def stop(self): + with self.lock: + if self._stopped: + return + self._stopped = True + + self.shutdown() + self.dispatcher.remove_session(self) + self.stop_subscriptions() + + + def shutdown(self): + pass + def stopped(self): with self.lock: return self._stopped + def subscribe_to_service(self, method, params): - subdesc = self.build_subdesc(method, params) with self.lock: - if subdesc is not None: - self.subscriptions.append(subdesc) - - # subdesc = A subscription description - @staticmethod - def build_subdesc(method, params): - if method == "blockchain.numblocks.subscribe": - return method, - elif method == "blockchain.headers.subscribe": - return method, - elif method in ["blockchain.address.subscribe", "blockchain.address.subscribe2"]: - if not params: - return None - else: - return method, params[0] - else: - return None + if (method, params) not in self.subscriptions: + self.subscriptions.append((method,params)) + + + def stop_subscriptions(self): + bp = self.bp - def contains_subscription(self, subdesc): with self.lock: - return subdesc in self.subscriptions - + s = self.subscriptions[:] + + for method, params in s: + with bp.watch_lock: + if method == 'blockchain.numblocks.subscribe': + if self in bp.watch_blocks: + bp.watch_blocks.remove(self) + elif method == 'blockchain.headers.subscribe': + if self in bp.watch_headers: + bp.watch_headers.remove(self) + elif method == "blockchain.address.subscribe": + addr = params[0] + l = bp.watched_addresses.get(addr) + if not l: + continue + if self in l: + l.remove(self) + if self in l: + print "error rc!!" + bp.shared.stop() + + if l == []: + bp.watched_addresses.pop(addr) + + with self.lock: + self.subscriptions = [] + class ResponseDispatcher(threading.Thread): - def __init__(self, shared, processor): + def __init__(self, shared, request_dispatcher): self.shared = shared - self.processor = processor + self.request_dispatcher = request_dispatcher threading.Thread.__init__(self) self.daemon = True def run(self): while not self.shared.stopped(): - self.update() - - def update(self): - response = self.processor.pop_response() - #print "pop response", response - internal_id = response.get('id') - method = response.get('method') - params = response.get('params') - - # A notification - if internal_id is None: # and method is not None and params is not None: - self.notification(method, params, response) - # A response - elif internal_id is not None: # and method is None and params is None: - self.send_response(internal_id, response) - else: - print "no method", response - - def notification(self, method, params, response): - subdesc = Session.build_subdesc(method, params) - for session in self.processor.sessions: - if session.stopped(): - continue - if session.contains_subscription(subdesc): - if response.get('method') == "blockchain.address.subscribe2": - response['method'] = "blockchain.address.subscribe" - session.send_response(response) - - def send_response(self, internal_id, response): - session, message_id = self.processor.get_session_id(internal_id) - if session: - response['id'] = message_id + session, response = self.request_dispatcher.pop_response() session.send_response(response) - else: - print "send_response: no session", message_id, internal_id, response -