X-Git-Url: https://git.novaco.in/?a=blobdiff_plain;f=lib%2Finterface.py;h=4424a0ccec3d3a483f261bed7ba5c8a997cf2c94;hb=HEAD;hp=dfae823a8b8a1a76dbdc62458f4a4a7bf03b7372;hpb=86fd728a47f8b5fcb2deed66c8c769851d9979ba;p=electrum-nvc.git diff --git a/lib/interface.py b/lib/interface.py index dfae823..4424a0c 100644 --- a/lib/interface.py +++ b/lib/interface.py @@ -17,84 +17,88 @@ # along with this program. If not, see . -import random, socket, ast, re, ssl +import random, ast, re, errno, os import threading, traceback, sys, time, json, Queue +import socks +import socket +import ssl from version import ELECTRUM_VERSION, PROTOCOL_VERSION from util import print_error, print_msg +from simple_config import SimpleConfig +import x509 DEFAULT_TIMEOUT = 5 -DEFAULT_PORTS = {'t':'50001', 's':'50002', 'h':'8081', 'g':'8082'} - -DEFAULT_SERVERS = { - 'electrum.thecoinwillprevail.com': {'h': '8081', 's': '50002', 't': '50001'}, - 'the9ull.homelinux.org': {'h': '8082', 't': '50001'}, - 'electra.datemas.de': {'h': '8081', 's': '50002', 't': '50001', 'g': '808'}, - 'electrum.datemas.de': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'}, - 'ecdsa.org': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'}, - 'electrum.mooo.com': {'h': '8081', 't': '50001'}, - 'btcback.com': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'}, - 'electrum.bitcoins.sk': {'h': '8081', 's': '50002', 't': '50001', 'g': '8'}, - 'electrum.no-ip.org': {'h': '80', 's': '50002', 't': '50001', 'g': '443'}, - 'electrum.drollette.com': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'}, - 'btc.it-zone.org': {'h': '80', 's': '110', 't': '50001', 'g': '443'}, - 'electrum.yacoin.com': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'}, - 'electrum.be': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'} -} - - - -def filter_protocol(servers, p): - l = [] - for k, protocols in servers.items(): - if p in protocols: - l.append( ':'.join([k, protocols[p], p]) ) - return l - - - proxy_modes = ['socks4', 'socks5', 'http'] -def pick_random_server(): - return random.choice( filter_protocol(DEFAULT_SERVERS,'s') ) + +def cert_verify_hostname(s): + # hostname verification (disabled) + from backports.ssl_match_hostname import match_hostname, CertificateError + try: + match_hostname(s.getpeercert(True), host) + print_error("hostname matches", host) + except CertificateError, ce: + print_error("hostname did not match", host) class Interface(threading.Thread): - def register_callback(self, event, callback): - with self.lock: - if not self.callbacks.get(event): - self.callbacks[event] = [] - self.callbacks[event].append(callback) - def trigger_callback(self, event): - with self.lock: - callbacks = self.callbacks.get(event,[])[:] - if callbacks: - [callback() for callback in callbacks] + def __init__(self, server, config = None): - def init_server(self, host, port, proxy=None, use_ssl=True): - self.host = host - self.port = port - self.proxy = proxy - self.use_ssl = use_ssl + threading.Thread.__init__(self) + self.daemon = True + self.config = config if config is not None else SimpleConfig() + self.connect_event = threading.Event() + + self.subscriptions = {} + self.lock = threading.Lock() + + self.rtime = 0 + self.bytes_received = 0 + self.is_connected = False self.poll_interval = 1 + self.debug = False # dump network messages. can be changed at runtime using the console + #json self.message_id = 0 self.unanswered_requests = {} - #banner - self.banner = '' + + # parse server + self.server = server + try: + host, port, protocol = self.server.split(':') + port = int(port) + except Exception: + self.server = None + return + + if protocol not in 'ghst': + raise Exception('Unknown protocol: %s'%protocol) + + self.host = host + self.port = port + self.protocol = protocol + self.use_ssl = ( protocol in 'sg' ) + self.proxy = self.parse_proxy_options(self.config.get('proxy')) + if self.proxy: + self.proxy_mode = proxy_modes.index(self.proxy["mode"]) + 1 + + + def queue_json_response(self, c): # uncomment to debug - # print_error( "<--",c ) + if self.debug: + print_error( "<--",c ) msg_id = c.get('id') error = c.get('error') @@ -103,58 +107,18 @@ class Interface(threading.Thread): print_error("received error:", c) if msg_id is not None: with self.lock: - method, params, channel = self.unanswered_requests.pop(msg_id) - response_queue = self.responses[channel] - response_queue.put({'method':method, 'params':params, 'error':error, 'id':msg_id}) + method, params, callback = self.unanswered_requests.pop(msg_id) + callback(self,{'method':method, 'params':params, 'error':error, 'id':msg_id}) return if msg_id is not None: with self.lock: - method, params, channel = self.unanswered_requests.pop(msg_id) + method, params, callback = self.unanswered_requests.pop(msg_id) result = c.get('result') - if method == 'server.version': - self.server_version = result - - elif method == 'server.banner': - self.banner = result - self.trigger_callback('banner') - - elif method == 'server.peers.subscribe': - servers = {} - for item in result: - - host = item[1] - out = {} - - version = None - pruning_level = '-' - if len(item) > 2: - for v in item[2]: - if re.match("[stgh]\d*", v): - protocol, port = v[0], v[1:] - if port == '': port = DEFAULT_PORTS[protocol] - out[protocol] = port - elif re.match("v(.?)+", v): - version = v[1:] - elif re.match("p\d*", v): - pruning_level = v[1:] - if pruning_level == '': pruning_level = '0' - try: - is_recent = float(version)>=float(PROTOCOL_VERSION) - except: - is_recent = False - - if out and is_recent: - out['pruning'] = pruning_level - servers[host] = out - - self.servers = servers - self.trigger_callback('peers') - else: - # notification: find the channel(s) + # notification method = c.get('method') params = c.get('params') @@ -174,37 +138,28 @@ class Interface(threading.Thread): with self.lock: for k,v in self.subscriptions.items(): if (method, params) in v: - channel = k + callback = k break else: print_error( "received unexpected notification", method, params) print_error( self.subscriptions ) return - - response_queue = self.responses[channel] - response_queue.put({'method':method, 'params':params, 'result':result, 'id':msg_id}) + callback(self, {'method':method, 'params':params, 'result':result, 'id':msg_id}) - def get_response(self, channel='default', block=True, timeout=10000000000): - return self.responses[channel].get(block, timeout) - def register_channel(self, channel): - with self.lock: - self.responses[channel] = Queue.Queue() + def on_version(self, i, result): + self.server_version = result - def poke(self, channel): - self.responses[channel].put(None) - - def init_http(self, host, port, proxy=None, use_ssl=True): - self.init_server(host, port, proxy, use_ssl) + def start_http(self): self.session_id = None self.is_connected = True self.connection_msg = ('https' if self.use_ssl else 'http') + '://%s:%d'%( self.host, self.port ) try: self.poll() - except: + except Exception: print_error("http init session failed") self.is_connected = False return @@ -226,7 +181,7 @@ class Interface(threading.Thread): break except socket.error: break - except: + except Exception: traceback.print_exc(file=sys.stdout) break @@ -234,16 +189,15 @@ class Interface(threading.Thread): def poll(self): - self.send([]) + self.send([], None) - def send_http(self, messages, channel='default'): + def send_http(self, messages, callback): import urllib2, json, time, cookielib print_error( "send_http", messages ) if self.proxy: - import socks - socks.setdefaultproxy(proxy_modes.index(self.proxy["mode"]) + 1, self.proxy["host"], int(self.proxy["port"]) ) + socks.setdefaultproxy(self.proxy_mode, self.proxy["host"], int(self.proxy["port"]) ) socks.wrapmodule(urllib2) cj = cookielib.CookieJar() @@ -253,11 +207,13 @@ class Interface(threading.Thread): t1 = time.time() data = [] + ids = [] for m in messages: method, params = m if type(params) != type([]): params = [params] data.append( { 'method':method, 'id':self.message_id, 'params':params } ) - self.unanswered_requests[self.message_id] = method, params, channel + self.unanswered_requests[self.message_id] = method, params, callback + ids.append(self.message_id) self.message_id += 1 if data: @@ -274,7 +230,7 @@ class Interface(threading.Thread): try: req = urllib2.Request(self.connection_msg, data_json, headers) response_stream = urllib2.urlopen(req, timeout=DEFAULT_TIMEOUT) - except: + except Exception: return for index, cookie in enumerate(cj): @@ -300,40 +256,138 @@ class Interface(threading.Thread): self.rtime = time.time() - t1 self.is_connected = True + return ids - def init_tcp(self, host, port, proxy=None, use_ssl=True): - self.init_server(host, port, proxy, use_ssl) + def start_tcp(self): - global proxy_modes - self.connection_msg = "%s:%d"%(self.host,self.port) - if self.proxy is None: - s = socket.socket( socket.AF_INET, socket.SOCK_STREAM ) - else: - self.connection_msg += " using proxy %s:%s:%s"%(self.proxy.get('mode'), self.proxy.get('host'), self.proxy.get('port')) - import socks - s = socks.socksocket() - s.setproxy(proxy_modes.index(self.proxy["mode"]) + 1, self.proxy["host"], int(self.proxy["port"]) ) + self.connection_msg = self.host + ':%d' % self.port + + if self.proxy is not None: + + socks.setdefaultproxy(self.proxy_mode, self.proxy["host"], int(self.proxy["port"])) + socket.socket = socks.socksocket + # prevent dns leaks, see http://stackoverflow.com/questions/13184205/dns-over-proxy + def getaddrinfo(*args): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))] + socket.getaddrinfo = getaddrinfo if self.use_ssl: - s = ssl.wrap_socket(s, ssl_version=ssl.PROTOCOL_SSLv23, do_handshake_on_connect=True) - - s.settimeout(2) - s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + cert_path = os.path.join( self.config.path, 'certs', self.host) + + if not os.path.exists(cert_path): + is_new = True + # get server certificate. + # Do not use ssl.get_server_certificate because it does not work with proxy + try: + l = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except socket.gaierror: + print_error("error: cannot resolve", self.host) + return + + for res in l: + try: + s = socket.socket( res[0], socket.SOCK_STREAM ) + s.connect(res[4]) + except: + s = None + continue + + try: + s = ssl.wrap_socket(s, ssl_version=ssl.PROTOCOL_SSLv3, cert_reqs=ssl.CERT_NONE, ca_certs=None) + except ssl.SSLError, e: + print_error("SSL error retrieving SSL certificate:", self.host, e) + s = None + + break + + if s is None: + return + + dercert = s.getpeercert(True) + s.close() + cert = ssl.DER_cert_to_PEM_cert(dercert) + # workaround android bug + cert = re.sub("([^\n])-----END CERTIFICATE-----","\\1\n-----END CERTIFICATE-----",cert) + temporary_path = cert_path + '.temp' + with open(temporary_path,"w") as f: + f.write(cert) + + else: + is_new = False try: - s.connect(( self.host.encode('ascii'), int(self.port))) - except: - traceback.print_exc(file=sys.stdout) - self.is_connected = False - self.s = None + addrinfo = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except socket.gaierror: + print_error("error: cannot resolve", self.host) + return + + for res in addrinfo: + try: + s = socket.socket( res[0], socket.SOCK_STREAM ) + s.settimeout(2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + s.connect(res[4]) + except: + s = None + continue + break + + if s is None: + print_error("failed to connect", self.host, self.port) return + if self.use_ssl: + try: + s = ssl.wrap_socket(s, + ssl_version=ssl.PROTOCOL_SSLv3, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs= (temporary_path if is_new else cert_path), + do_handshake_on_connect=True) + except ssl.SSLError, e: + print_error("SSL error:", self.host, e) + if e.errno != 1: + return + if is_new: + rej = cert_path + '.rej' + if os.path.exists(rej): + os.unlink(rej) + os.rename(temporary_path, rej) + else: + with open(cert_path) as f: + cert = f.read() + try: + x = x509.X509() + x.parse(cert) + x.slow_parse() + except: + traceback.print_exc(file=sys.stdout) + print_error("wrong certificate", self.host) + return + try: + x.check_date() + except: + print_error("certificate has expired:", cert_path) + os.unlink(cert_path) + return + print_error("wrong certificate", self.host) + return + except Exception: + print_error("wrap_socket failed", self.host) + traceback.print_exc(file=sys.stdout) + return + + if is_new: + print_error("saving certificate for", self.host) + os.rename(temporary_path, cert_path) + s.settimeout(60) self.s = s self.is_connected = True + print_error("connected to", self.host, self.port) + def run_tcp(self): try: @@ -348,7 +402,9 @@ class Interface(threading.Thread): except ssl.SSLError: timeout = True except socket.error, err: - if err.errno in [11, 10035]: + if err.errno == 60: + timeout = True + elif err.errno in [11, 10035]: print_error("socket errno", err.errno) time.sleep(0.1) continue @@ -358,7 +414,7 @@ class Interface(threading.Thread): if timeout: # ping the server with server.version, as a real ping does not exist yet - self.send([('server.version', [ELECTRUM_VERSION, PROTOCOL_VERSION])]) + self.send([('server.version', [ELECTRUM_VERSION, PROTOCOL_VERSION])], self.on_version) continue out += msg @@ -374,120 +430,63 @@ class Interface(threading.Thread): c = json.loads(c) self.queue_json_response(c) - except: + except Exception: traceback.print_exc(file=sys.stdout) self.is_connected = False - def send_tcp(self, messages, channel='default'): + def send_tcp(self, messages, callback): """return the ids of the requests that we sent""" out = '' ids = [] for m in messages: method, params = m request = json.dumps( { 'id':self.message_id, 'method':method, 'params':params } ) - self.unanswered_requests[self.message_id] = method, params, channel + self.unanswered_requests[self.message_id] = method, params, callback ids.append(self.message_id) - # uncomment to debug - # print "-->",request + if self.debug: + print "-->", request self.message_id += 1 out += request + '\n' while out: try: sent = self.s.send( out ) out = out[sent:] - except: - # this happens when we get disconnected - print_error( "Not connected, cannot send" ) - return None + except socket.error,e: + if e[0] in (errno.EWOULDBLOCK,errno.EAGAIN): + print_error( "EAGAIN: retrying") + time.sleep(0.1) + continue + else: + traceback.print_exc(file=sys.stdout) + # this happens when we get disconnected + print_error( "Not connected, cannot send" ) + return None return ids - def __init__(self, config=None, loop=False): - self.server = random.choice(filter_protocol(DEFAULT_SERVERS, 's')) - self.proxy = None - - if config is None: - from simple_config import SimpleConfig - config = SimpleConfig() - - threading.Thread.__init__(self) - self.daemon = True - self.loop = loop - self.config = config - self.connect_event = threading.Event() - self.subscriptions = {} - self.responses = {} - self.responses['default'] = Queue.Queue() - - self.callbacks = {} - self.lock = threading.Lock() - - self.servers = {} # actual list from IRC - self.rtime = 0 - self.bytes_received = 0 - self.is_connected = False + def start_interface(self): - - def init_interface(self): - if self.config.get('server'): - self.init_with_server(self.config) - else: - if self.config.get('auto_cycle') is None: - self.config.set_key('auto_cycle', True, False) - - if not self.is_connected and self.config.get('auto_cycle'): - print_msg("Using random server...") - servers = filter_protocol(DEFAULT_SERVERS, 's') - while servers: - server = random.choice( servers ) - servers.remove(server) - print server - self.config.set_key('server', server, False) - self.init_with_server(self.config) - if self.is_connected: break - - if not self.is_connected: - print 'no server available' - self.connect_event.set() # to finish start - self.server = 'ecdsa.org:50001:t' - self.proxy = None - return + if self.protocol in 'st': + self.start_tcp() + elif self.protocol in 'gh': + self.start_http() self.connect_event.set() - if self.is_connected: - self.send([('server.version', [ELECTRUM_VERSION, PROTOCOL_VERSION])]) - self.send([('server.banner',[])]) - self.trigger_callback('connected') - else: - self.trigger_callback('notconnected') - #print_error("Failed to connect " + self.connection_msg) - def init_with_server(self, config): - - s = config.get('server') - host, port, protocol = s.split(':') - port = int(port) - self.protocol = protocol - proxy = self.parse_proxy_options(config.get('proxy')) - self.server = host + ':%d:%s'%(port, protocol) - - #print protocol, host, port - if protocol in 'st': - self.init_tcp(host, port, proxy, use_ssl=(protocol=='s')) - elif protocol in 'gh': - self.init_http(host, port, proxy, use_ssl=(protocol=='g')) - else: - raise BaseException('Unknown protocol: %s'%protocol) + def stop_subscriptions(self): + for callback in self.subscriptions.keys(): + callback(self, None) + self.subscriptions = {} - def send(self, messages, channel='default'): + def send(self, messages, callback): sub = [] for message in messages: @@ -497,30 +496,25 @@ class Interface(threading.Thread): if sub: with self.lock: - if self.subscriptions.get(channel) is None: - self.subscriptions[channel] = [] + if self.subscriptions.get(callback) is None: + self.subscriptions[callback] = [] for message in sub: - if message not in self.subscriptions[channel]: - self.subscriptions[channel].append(message) + if message not in self.subscriptions[callback]: + self.subscriptions[callback].append(message) if not self.is_connected: + print_error("interface: trying to send while not connected") return if self.protocol in 'st': with self.lock: - out = self.send_tcp(messages, channel) + out = self.send_tcp(messages, callback) else: # do not use lock, http is synchronous - out = self.send_http(messages, channel) + out = self.send_http(messages, callback) return out - def resend_subscriptions(self): - for channel, messages in self.subscriptions.items(): - if messages: - self.send(messages, channel) - - def parse_proxy_options(self, s): if type(s) == type({}): return s # fixme: type should be fixed @@ -542,62 +536,50 @@ class Interface(threading.Thread): return proxy - def set_server(self, server, proxy=None): - # raise an error if the format isnt correct - a,b,c = server.split(':') - b = int(b) - assert c in 'stgh' - # set the server - if server != self.server or proxy != self.proxy: - print "changing server:", server, proxy - self.server = server - self.proxy = proxy - if self.is_connected and self.protocol in 'st' and self.s: - self.s.shutdown(socket.SHUT_RDWR) - self.s.close() - self.is_connected = False # this exits the polling loop - self.trigger_callback('disconnecting') # for actively disconnecting def stop(self): if self.is_connected and self.protocol in 'st' and self.s: self.s.shutdown(socket.SHUT_RDWR) self.s.close() + self.is_connected = False - def get_servers(self): - if not self.servers: - return DEFAULT_SERVERS - else: - return self.servers + def is_up_to_date(self): + return self.unanswered_requests == {} - def is_empty(self, channel): - q = self.responses.get(channel) - if q: - return q.empty() - else: - return True - def get_pending_requests(self, channel): - result = [] - with self.lock: - for k, v in self.unanswered_requests.items(): - a, b, c = v - if c == channel: result.append(k) - return result + def start(self, queue = None, wait = False): + if not self.server: + return + self.queue = queue if queue else Queue.Queue() + threading.Thread.start(self) + if wait: + self.connect_event.wait() + + + def run(self): + self.start_interface() + if self.is_connected: + self.send([('server.version', [ELECTRUM_VERSION, PROTOCOL_VERSION])], self.on_version) + self.change_status() + self.run_tcp() if self.protocol in 'st' else self.run_http() + self.change_status() + - def is_up_to_date(self, channel): - return self.is_empty(channel) and not self.get_pending_requests(channel) + def change_status(self): + #print "change status", self.server, self.is_connected + self.queue.put(self) def synchronous_get(self, requests, timeout=100000000): - # todo: use generators, unanswered_requests should be a list of arrays... - ids = self.send(requests) + queue = Queue.Queue() + ids = self.send(requests, lambda i,r: queue.put(r)) id2 = ids[:] res = {} while ids: - r = self.responses['default'].get(True, timeout) + r = queue.get(True, timeout) _id = r.get('id') if _id in ids: ids.remove(_id) @@ -608,26 +590,38 @@ class Interface(threading.Thread): return out - def start(self, wait=True): - threading.Thread.start(self) - if wait: - # wait until connection is established - self.connect_event.wait() - if not self.is_connected: - return False - return True - def run(self): - while True: - self.init_interface() - if self.is_connected: - self.resend_subscriptions() - self.run_tcp() if self.protocol in 'st' else self.run_http() - self.trigger_callback('disconnected') +def check_cert(host, cert): + try: + x = x509.X509() + x.parse(cert) + x.slow_parse() + except: + traceback.print_exc(file=sys.stdout) + return + + try: + x.check_date() + expired = False + except: + expired = True - if not self.loop: break - time.sleep(5) + m = "host: %s\n"%host + m += "has_expired: %s\n"% expired + print_msg(m) +def test_certificates(): + config = SimpleConfig() + mydir = os.path.join(config.path, "certs") + certs = os.listdir(mydir) + for c in certs: + print c + p = os.path.join(mydir,c) + with open(p) as f: + cert = f.read() + check_cert(c, cert) +if __name__ == "__main__": + test_certificates()