October 2013 - tor-commits - lists.torproject.org

[flashproxy/master] Merge branch 'bug9349_server_endpoints' into merge-next
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 5843e89467a30679a64bd10d52b4c7f0124886d7 Merge: fd692bd 8cc46cf Author: Ximin Luo <infinity0(a)gmx.com> Date: Sat Oct 19 17:19:39 2013 +0100 Merge branch 'bug9349_server_endpoints' into merge-next Conflicts: facilitator/Makefile facilitator/fac.py facilitator/facilitator facilitator/facilitator.cgi doc/design.txt | 36 ++++- facilitator/Makefile | 2 + facilitator/fac.py | 68 +++++++- facilitator/facilitator | 347 +++++++++++++++++++++++++++------------- facilitator/facilitator-test | 201 +++++++++++++++++++++-- facilitator/facilitator.cgi | 60 +++++-- facilitator/facilitator.py | 1 + facilitator/init.d/facilitator | 8 +- facilitator/relays | 2 + 9 files changed, 572 insertions(+), 153 deletions(-) diff --cc facilitator/Makefile index 07ed125,e69f9a7..52bd201 --- a/facilitator/Makefile +++ b/facilitator/Makefile @@@ -6,9 -5,11 +6,11 @@@ all : install: - mkdir -p $(BINDIR) - cp -f facilitator facilitator-email-poller facilitator-reg-daemon facilitator-reg facilitator.cgi fac.py $(BINDIR) - mkdir -p /etc/flashproxy - cp -f relays /etc/flashproxy - cp -f init.d/facilitator init.d/facilitator-email-poller init.d/facilitator-reg-daemon /etc/init.d/ + mkdir -p $(DESTDIR)$(BINDIR) + cp -f facilitator facilitator-email-poller facilitator-reg-daemon facilitator-reg facilitator.cgi fac.py $(DESTDIR)$(BINDIR) ++ mkdir -p $(DESTDIR)/etc/flashproxy ++ cp -f relays $(DESTDIR)/etc/flashproxy + cp -f init.d/facilitator init.d/facilitator-email-poller init.d/facilitator-reg-daemon $(DESTDIR)/etc/init.d/ clean: rm -f *.pyc diff --cc facilitator/fac.py index 5e334a5,8e37d39..88365e5 --- a/facilitator/fac.py +++ b/facilitator/fac.py @@@ -239,11 -281,14 +281,12 @@@ def transact(f, command, *params) raise ValueError("No newline at end of string returned by facilitator") return parse_transaction(line[:-1]) - def put_reg(facilitator_addr, client_addr): -def put_reg(facilitator_addr, client_addr, transport, registrant_addr=None): ++def put_reg(facilitator_addr, client_addr, transport): """Send a registration to the facilitator using a one-time socket. Returns true iff the command was successful.""" f = fac_socket(facilitator_addr) params = [("CLIENT", format_addr(client_addr))] + params.append(("TRANSPORT", transport)) - if registrant_addr is not None: - params.append(("FROM", format_addr(registrant_addr))) try: command, params = transact(f, "PUT", *params) finally: diff --cc facilitator/facilitator index f7ebf10,63272f1..cbe3d98 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@@ -51,10 -57,13 +57,13 @@@ again. Listen on 127.0.0.1 and port POR -d, --debug don't daemonize, log to stdout. -h, --help show this help. -l, --log FILENAME write log to FILENAME (default \"%(log)s\"). - -p, --port PORT listen on PORT (by default %(port)d). + -p, --port PORT listen on PORT (default %(port)d). --pidfile FILENAME write PID to FILENAME after daemonizing. --privdrop-user USER switch UID and GID to those of USER. - -r, --relay RELAY send RELAY (host:port) to proxies as the relay to use. + -r, --relay-file RELAY learn relays from FILE. + --outer-transports TRANSPORTS + comma-sep list of outer transports to accept proxies + for (by default %(outer-transports)s) --unsafe-logging don't scrub IP addresses from logs.\ """ % { "progname": sys.argv[0], @@@ -215,11 -283,8 +281,13 @@@ class Handler(SocketServer.StreamReques def send_error(self): print >> self.wfile, "ERROR" + def error(self, log_msg): + log(log_msg) + self.send_error() + return False + + # Handle a GET request (got flashproxy poll; need to return a proper client registration) + # Example: GET FROM="3.3.3.3:3333" TRANSPORT="websocket" TRANSPORT="webrtc" def do_GET(self, params): proxy_spec = fac.param_first("FROM", params) if proxy_spec is None: @@@ -227,40 -294,78 +295,64 @@@ try: proxy_addr = fac.parse_addr_spec(proxy_spec, defport=0) except ValueError, e: - log(u"syntax error in proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e))))) - self.send_error() - return False + return self.error(u"syntax error in proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e))))) + transport_list = fac.param_getlist("PROXY_TRANSPORT", params) + if not transport_list: - log(u"PROXY TRANSPORT missing FROM param") - self.send_error() - return False ++ return self.error(u"PROXY TRANSPORT missing FROM param") + try: - reg = get_reg_for_proxy(proxy_addr) + client_reg, relay_reg = get_match_for_proxy(proxy_addr, transport_list) except Exception, e: - log(u"error getting reg for proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e))))) - self.send_error() - return False + return self.error(u"error getting reg for proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e))))) + check_back_in = get_check_back_in_for_proxy(proxy_addr) - if reg: - log(u"proxy gets %s, relay %s (now %d)" % - (safe_str(unicode(reg)), options.relay_spec, num_regs())) - print >> self.wfile, fac.render_transaction("OK", ("CLIENT", str(reg)), ("RELAY", options.relay_spec), ("CHECK-BACK-IN", str(check_back_in))) + + if client_reg: + log(u"proxy (%s) gets client '%s' (supported transports: %s) (num relays: %s) (remaining regs: %d/%d)" % + (safe_str(repr(proxy_spec)), safe_str(repr(client_reg.addr)), transport_list, num_relays(), num_unhandled_regs(), num_regs())) + print >> self.wfile, fac.render_transaction("OK", + ("CLIENT", fac.format_addr(client_reg.addr)), + ("RELAY", fac.format_addr(relay_reg.addr)), + ("CHECK-BACK-IN", str(check_back_in))) else: - log(u"proxy gets none") + log(u"proxy (%s) gets none" % safe_str(repr(proxy_spec))) print >> self.wfile, fac.render_transaction("NONE", ("CHECK-BACK-IN", str(check_back_in))) + return True + # Handle a PUT request (client made a registration request; register it.) + # Example: PUT CLIENT="1.1.1.1:5555" TRANSPORT="obfs3|websocket" def do_PUT(self, params): + # Check out if we recognize the transport in this registration request + transport = fac.param_first("TRANSPORT", params) + if transport is None: - log(u"PUT missing TRANSPORT param") - self.send_error() - return False ++ return self.error(u"PUT missing TRANSPORT param") + + transport = Transport.parse(transport) + # See if we have relays that support this transport + if transport.outer not in options.outer_transports: + return self.error(u"Unrecognized transport: %s" % transport) + client_spec = fac.param_first("CLIENT", params) if client_spec is None: - log(u"PUT missing CLIENT param") - self.send_error() - return False + return self.error(u"PUT missing CLIENT param") try: - reg = Reg.parse(client_spec) - except ValueError, e: + reg = Endpoint.parse(client_spec, transport) + except (UnknownTransport, ValueError) as e: + # XXX should we throw a better error message to the client? Is it possible? - log(u"syntax error in %s: %s" % (safe_str(repr(client_spec)), safe_str(repr(str(e))))) - self.send_error() - return False + return self.error(u"syntax error in %s: %s" % (safe_str(repr(client_spec)), safe_str(repr(str(e))))) try: ok = put_reg(reg) except Exception, e: - log(u"error putting reg %s: %s" % (safe_str(repr(client_spec)), safe_str(repr(str(e))))) - self.send_error() - return False + return self.error(u"error putting reg %s: %s" % (safe_str(repr(client_spec)), safe_str(repr(str(e))))) + if ok: - log(u"client %s (now %d)" % (safe_str(unicode(reg)), num_regs())) + log(u"client %s (transports: %s) (remaining regs: %d/%d)" % (safe_str(unicode(reg)), reg.transport, num_unhandled_regs(), num_regs())) else: - log(u"client %s (already present, now %d)" % (safe_str(unicode(reg)), num_regs())) + log(u"client %s (already present) (transports: %s) (remaining regs: %d/%d)" % (safe_str(unicode(reg)), reg.transport, num_unhandled_regs(), num_regs())) self.send_ok() return True @@@ -307,22 -419,30 +406,38 @@@ def get_check_back_in_for_proxy(proxy_a def put_reg(reg): """Add a registration.""" - addr_str = reg.host - af = addr_af(addr_str) - REGS = regs_for_af(af) - return REGS.add(reg) + af = addr_af(reg.addr[0]) + return CLIENTS[af].addEndpoint(reg.addr, reg.transport) + + def parse_relay_file(servers, fp): + """Parse a file containing Tor relays that we can point proxies to. + Throws ValueError on a parsing error. Each line contains a transport chain + and an address, for example + obfs2|websocket 1.4.6.1:4123 + """ + for line in fp.readlines(): + try: + transport_spec, addr_spec = line.strip().split() + except ValueError, e: + raise ValueError("Wrong line format: %s." % repr(line)) + addr = fac.parse_addr_spec(addr_spec, defport=DEFAULT_RELAY_PORT, resolve=True) + transport = Transport.parse(transport_spec) + if transport.outer not in options.outer_transports: + raise ValueError(u"Unrecognized transport: %s" % transport) + af = addr_af(addr[0]) + servers[af].addEndpoint(addr, transport) def main(): - opts, args = getopt.gnu_getopt(sys.argv[1:], "dhl:p:r:", - ["debug", "help", "log=", "port=", "pidfile=", "privdrop-user=", "relay-file=", "unsafe-logging"]) + opts, args = getopt.gnu_getopt(sys.argv[1:], "dhl:p:r:", [ + "debug", + "help", + "log=", + "port=", + "pidfile=", + "privdrop-user=", - "relay=", ++ "relay-file=", + "unsafe-logging", + ]) for o, a in opts: if o == "-d" or o == "--debug": options.daemonize = False diff --cc facilitator/facilitator.cgi index e4e9bdf,a68d84e..624580f --- a/facilitator/facilitator.cgi +++ b/facilitator/facilitator.cgi @@@ -78,18 -88,45 +87,45 @@@ Access-Control-Allow-Origin: *\ exit_error(400) def do_post(): + """Parse client registration.""" + + # Old style client registration: + # client=1.2.3.4:9000 + # New style client registration: + # client-websocket=1.2.3.4:9000&client-obfs3|websocket=1.2.3.4:10000 + if path_info != "/": exit_error(400) - client_specs = fs.getlist("client") - if len(client_specs) != 1: - exit_error(400) - client_spec = client_specs[0].strip() - try: - client_addr = fac.parse_addr_spec(client_spec, defhost=remote_addr[0]) - except ValueError: - exit_error(400) - if not fac.put_reg(FACILITATOR_ADDR, client_addr): - exit_error(500) + + # We iterate through the items in the POST body, and see if any of + # them look like "client-websocket=1.2.3.4:9000". We then split + # all those items and send them as separate registrations to the + # facilitator. + for key in fs.keys(): + if key != "client" and not key.startswith("client-"): + continue + + if key == "client": # reg without transport info -- default to websocket. + transport = "websocket" + else: # reg with transport info -- get the "websocket" part out of "client-websocket". + transport = key[len("client-"):] + + # Get the "1.2.3.4:9000" part of "client-websocket=1.2.3.4:9000". + client_spec = fs[key].value.strip() + try: + client_addr = fac.parse_addr_spec(client_spec, defhost=remote_addr[0]) + except ValueError: + exit_error(400) + + # XXX what if previous registrations passed through + # successfully, but the last one failed and called + # exit_error()? + + # XXX need to link these registrations together, so that + # when one is answerered the rest are invalidated. - if not fac.put_reg(FACILITATOR_ADDR, client_addr, transport, remote_addr): ++ if not fac.put_reg(FACILITATOR_ADDR, client_addr, transport): + exit_error(500) + print """\ Status: 200\r \r"""

1 0

[flashproxy/master] - fix match() trying to access info of deleted address
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 9b155c9b2029ec779666a661e5eecf8226cef6c8 Author: Ximin Luo <infinity0(a)gmx.com> Date: Fri Oct 11 17:12:07 2013 +0100 - fix match() trying to access info of deleted address --- facilitator/facilitator | 16 +++++++--------- facilitator/facilitator-test | 38 ++++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/facilitator/facilitator b/facilitator/facilitator index 9e11826..844836a 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -161,7 +161,7 @@ class Endpoints(object): def _serveReg(self, addrpool): """ :param list addrpool: List of candidate addresses. - :returns: An address of an endpoint from the given pool. The serve + :returns: An Endpoint whose address is from the given pool. The serve counter for that address is also incremented, and if it hits self._maxserve the endpoint is removed from this collection. :raises: KeyError if any address is not registered with this collection @@ -170,9 +170,10 @@ class Endpoints(object): prio_addr = min(addrpool, key=lambda a: self._served[a]) assert self._served[prio_addr] < self._maxserve self._served[prio_addr] += 1 + transport = self._endpoints[prio_addr] if self._served[prio_addr] == self._maxserve: self.delEndpoint(prio_addr) - return prio_addr + return Endpoint(prio_addr, transport) EMPTY_MATCH = (None, None) @staticmethod @@ -195,16 +196,13 @@ class Endpoints(object): # find a client to serve client_pool = [addr for inner in both for addr in client_inner[inner]] assert len(client_pool) - client_addr = ptsClient._serveReg(client_pool) + client_reg = ptsClient._serveReg(client_pool) # find a server to serve that has the same inner transport - inner = ptsClient._endpoints[client_addr].inner + inner = client_reg.transport.inner assert inner in server_inner and len(server_inner[inner]) - server_addr = ptsServer._serveReg(server_inner[inner]) + server_reg = ptsServer._serveReg(server_inner[inner]) # assume servers never run out - client_transport = ptsClient._endpoints[client_addr] - server_transport = ptsServer._endpoints[server_addr] - return (Endpoint(client_addr, client_transport), - Endpoint(server_addr, server_transport)) + return (client_reg, server_reg) class Handler(SocketServer.StreamRequestHandler): diff --git a/facilitator/facilitator-test b/facilitator/facilitator-test index 8143348..3efe34d 100755 --- a/facilitator/facilitator-test +++ b/facilitator/facilitator-test @@ -60,9 +60,9 @@ class EndpointsTest(unittest.TestCase): self.pts.addEndpoint("C", "a|p") for i in xrange(64): # 64 is infinite ;) served = set() - served.add(self.pts._serveReg("ABC")) - served.add(self.pts._serveReg("ABC")) - served.add(self.pts._serveReg("ABC")) + served.add(self.pts._serveReg("ABC").addr) + served.add(self.pts._serveReg("ABC").addr) + served.add(self.pts._serveReg("ABC").addr) self.assertEquals(served, set("ABC")) def test_serveReg_maxserve_finite_exhaustion(self): @@ -74,25 +74,25 @@ class EndpointsTest(unittest.TestCase): # test getNumUnservedEndpoints whilst we're at it self.assertEquals(self.pts.getNumUnservedEndpoints(), 3) served = set() - served.add(self.pts._serveReg("ABC")) + served.add(self.pts._serveReg("ABC").addr) self.assertEquals(self.pts.getNumUnservedEndpoints(), 2) - served.add(self.pts._serveReg("ABC")) + served.add(self.pts._serveReg("ABC").addr) self.assertEquals(self.pts.getNumUnservedEndpoints(), 1) - served.add(self.pts._serveReg("ABC")) + served.add(self.pts._serveReg("ABC").addr) self.assertEquals(self.pts.getNumUnservedEndpoints(), 0) self.assertEquals(served, set("ABC")) for i in xrange(5-2): served = set() - served.add(self.pts._serveReg("ABC")) - served.add(self.pts._serveReg("ABC")) - served.add(self.pts._serveReg("ABC")) + served.add(self.pts._serveReg("ABC").addr) + served.add(self.pts._serveReg("ABC").addr) + served.add(self.pts._serveReg("ABC").addr) self.assertEquals(served, set("ABC")) remaining = set("ABC") - remaining.remove(self.pts._serveReg(remaining)) + remaining.remove(self.pts._serveReg(remaining).addr) self.assertRaises(KeyError, self.pts._serveReg, "ABC") - remaining.remove(self.pts._serveReg(remaining)) + remaining.remove(self.pts._serveReg(remaining).addr) self.assertRaises(KeyError, self.pts._serveReg, "ABC") - remaining.remove(self.pts._serveReg(remaining)) + remaining.remove(self.pts._serveReg(remaining).addr) self.assertRaises(KeyError, self.pts._serveReg, "ABC") self.assertEquals(remaining, set()) self.assertEquals(self.pts.getNumUnservedEndpoints(), 0) @@ -151,6 +151,20 @@ class EndpointsTest(unittest.TestCase): self.assertEquals(empty, Endpoints.match(self.pts, self.pts2, ["x"])) self.assertEquals(empty, Endpoints.match(self.pts, self.pts2, ["x"])) + def test_match_exhaustion(self): + self.pts.addEndpoint("A", "p") + self.pts2 = Endpoints(af=socket.AF_INET, maxserve=2) + self.pts2.addEndpoint("B", "p") + print self.pts2._indexes, self.pts2._served + Endpoints.match(self.pts2, self.pts, ["p"]) + print self.pts2._indexes, self.pts2._served + Endpoints.match(self.pts2, self.pts, ["p"]) + empty = Endpoints.EMPTY_MATCH + self.assertTrue("B" not in self.pts2._endpoints) + self.assertTrue("B" not in self.pts2._indexes["p"][""]) + self.assertEquals(empty, Endpoints.match(self.pts2, self.pts, ["p"])) + + class FacilitatorTest(unittest.TestCase): def test_transport_parse(self):

1 0

[flashproxy/master] Revert "report more detailed errors and show stacktraces when unsafe logging"
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 8cc46cf22d0ff4a5707bf836fcc990ebae9762ff Author: Ximin Luo <infinity0(a)gmx.com> Date: Sat Oct 19 17:16:14 2013 +0100 Revert "report more detailed errors and show stacktraces when unsafe logging" This reverts commit c1af35b2e2d57cb8850b1cf345d29065766a9410. Conflicts: facilitator/facilitator --- facilitator/fac.py | 6 ++--- facilitator/facilitator | 61 ++++++++++++++++++++++++++++------------------- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/facilitator/fac.py b/facilitator/fac.py index 6a670d3..8e37d39 100644 --- a/facilitator/fac.py +++ b/facilitator/fac.py @@ -227,7 +227,7 @@ def parse_transaction(line): if not (pos < len(line)): break if skipped == 0: - raise ValueError("Expected space before key-value pair: %r" % line) + raise ValueError("Expected space before key-value pair") pos, key = get_token(pos, line) if not (pos < len(line) and line[pos] == '='): raise ValueError("No '=' found after key") @@ -278,7 +278,7 @@ def transact(f, command, *params): f.flush() line = f.readline() if not (len(line) > 0 and line[-1] == '\n'): - raise ValueError("No newline at end of string returned by facilitator: %r" % line) + raise ValueError("No newline at end of string returned by facilitator") return parse_transaction(line[:-1]) def put_reg(facilitator_addr, client_addr, transport, registrant_addr=None): @@ -343,7 +343,7 @@ def get_reg(facilitator_addr, proxy_addr, proxy_transport_list): response["relay"] = format_addr(relay) return response else: - raise ValueError("Facilitator response was not \"OK\": %s: %s" % (command, params)) + raise ValueError("Facilitator response was not \"OK\"") def put_reg_base64(b64): """Attempt to add a registration by running a facilitator-reg program diff --git a/facilitator/facilitator b/facilitator/facilitator index 844836a..63272f1 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -7,7 +7,6 @@ import socket import sys import threading import time -import traceback from collections import defaultdict import fac @@ -254,7 +253,7 @@ class Handler(SocketServer.StreamRequestHandler): if not line: break num_lines += 1 - except socket.error as e: + except socket.error, e: log("socket error after reading %d lines: %s" % (num_lines, str(e))) break if not self.handle_line(line): @@ -265,46 +264,52 @@ class Handler(SocketServer.StreamRequestHandler): raise ValueError("No newline at end of string returned by readline") try: command, params = fac.parse_transaction(line[:-1]) - except ValueError as e: - return self.error("fac.parse_transaction: %(cause)s", e) + except ValueError, e: + log("fac.parse_transaction: %s" % e) + self.send_error() + return False if command == "GET": return self.do_GET(params) elif command == "PUT": return self.do_PUT(params) else: - return self.error(u"unrecognized command: %s" % command) + self.send_error() + return False def send_ok(self): print >> self.wfile, "OK" - def error(self, msg="", cause=None): - msg = safe_str(msg % { "cause": cause }) - log(msg) - if not options.safe_logging: - log(traceback.format_exc()) - print >>self.wfile, fac.render_transaction("ERROR", ("MSG", msg)) - return False + def send_error(self): + print >> self.wfile, "ERROR" # Handle a GET request (got flashproxy poll; need to return a proper client registration) # Example: GET FROM="3.3.3.3:3333" TRANSPORT="websocket" TRANSPORT="webrtc" def do_GET(self, params): proxy_spec = fac.param_first("FROM", params) if proxy_spec is None: - return self.error(u"GET missing FROM param") + log(u"GET missing FROM param") + self.send_error() + return False try: proxy_addr = fac.parse_addr_spec(proxy_spec, defport=0) - except ValueError as e: - return self.error(u"syntax error in proxy address %s: %%(cause)s" % safe_str(repr(proxy_spec)), e) + except ValueError, e: + log(u"syntax error in proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e))))) + self.send_error() + return False transport_list = fac.param_getlist("PROXY_TRANSPORT", params) if not transport_list: - return self.error(u"PROXY_TRANSPORT missing FROM param") + log(u"PROXY TRANSPORT missing FROM param") + self.send_error() + return False try: client_reg, relay_reg = get_match_for_proxy(proxy_addr, transport_list) - except Exception as e: - return self.error(u"error getting match for proxy address %s: %%(cause)s" % safe_str(repr(proxy_spec)), e) + except Exception, e: + log(u"error getting reg for proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e))))) + self.send_error() + return False check_back_in = get_check_back_in_for_proxy(proxy_addr) @@ -327,7 +332,9 @@ class Handler(SocketServer.StreamRequestHandler): # Check out if we recognize the transport in this registration request transport = fac.param_first("TRANSPORT", params) if transport is None: - return self.error(u"PUT missing TRANSPORT param") + log(u"PUT missing TRANSPORT param") + self.send_error() + return False transport = Transport.parse(transport) # See if we have relays that support this transport @@ -336,18 +343,24 @@ class Handler(SocketServer.StreamRequestHandler): client_spec = fac.param_first("CLIENT", params) if client_spec is None: - return self.error(u"PUT missing CLIENT param") + log(u"PUT missing CLIENT param") + self.send_error() + return False try: reg = Endpoint.parse(client_spec, transport) except (UnknownTransport, ValueError) as e: # XXX should we throw a better error message to the client? Is it possible? - return self.error(u"syntax error in %s: %%(cause)s" % safe_str(repr(client_spec)), e) + log(u"syntax error in %s: %s" % (safe_str(repr(client_spec)), safe_str(repr(str(e))))) + self.send_error() + return False try: ok = put_reg(reg) - except Exception as e: - return self.error(u"error putting reg %s: %%(cause)s" % safe_str(repr(client_spec)), e) + except Exception, e: + log(u"error putting reg %s: %s" % (safe_str(repr(client_spec)), safe_str(repr(str(e))))) + self.send_error() + return False if ok: log(u"client %s (transports: %s) (remaining regs: %d/%d)" % (safe_str(unicode(reg)), reg.transport, num_unhandled_regs(), num_regs())) @@ -499,7 +512,7 @@ obfs2|websocket 1.4.6.1:4123\ log(u"dropping privileges to those of user %s" % options.privdrop_username) try: fac.drop_privs(options.privdrop_username) - except BaseException as e: + except BaseException, e: print >> sys.stderr, "Can't drop privileges:", str(e) sys.exit(1)

1 0

[flashproxy/master] drop registrations that hit _maxserve, and fix tests to run properly
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit d658db1f29d698a7820ef0d4bdf1120e01fb35af Author: Ximin Luo <infinity0(a)gmx.com> Date: Wed Oct 9 23:49:38 2013 +0100 drop registrations that hit _maxserve, and fix tests to run properly - the old behaviour was based on an incorrect understanding of the previous iteration of the code --- facilitator/facilitator | 20 ++++++++++---------- facilitator/facilitator-test | 23 +++++++++++++++++++---- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/facilitator/facilitator b/facilitator/facilitator index f3ae79b..07038d1 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -173,17 +173,18 @@ class Endpoints(object): def _serveReg(self, addrpool): """ :param list addrpool: List of candidate addresses. - :returns: An address of an endpoint from the given pool, or None if all - endpoints have already been served _maxserve times. The serve - counter for that address is also incremented. + :returns: An address of an endpoint from the given pool. The serve + counter for that address is also incremented, and if it hits + self._maxserve the endpoint is removed from this collection. + :raises: KeyError if any address is not registered with this collection """ - if not addrpool: return None + if not addrpool: raise ValueError("gave empty address pool") prio_addr = min(addrpool, key=lambda a: self._served[a]) - if self._served[prio_addr] < self._maxserve: - self._served[prio_addr] += 1 - return prio_addr - else: - return None + assert self._served[prio_addr] < self._maxserve + self._served[prio_addr] += 1 + if self._served[prio_addr] == self._maxserve: + self.delEndpoint(prio_addr) + return prio_addr def _ensureIndexForSuffix(self, suf): if suf in self._indexes: return @@ -227,7 +228,6 @@ class Endpoints(object): assert all(client_pre.itervalues()) # no pool is empty pre = min(both, key=lambda p: ptsClient._avServed(client_pre[p])) client_addr = ptsClient._serveReg(client_pre[pre]) - if not client_addr: return Endpoints.EMPTY_MATCH server_addr = ptsServer._serveReg(server_pre[pre]) # assume servers never run out client_transport = ptsClient._endpoints[client_addr] diff --git a/facilitator/facilitator-test b/facilitator/facilitator-test index 6709221..3f2fbef 100755 --- a/facilitator/facilitator-test +++ b/facilitator/facilitator-test @@ -80,7 +80,7 @@ class EndpointsTest(unittest.TestCase): self.assertTrue(self.pts.supports("obfs3|unknownwhat")) self.assertFalse(self.pts.supports("obfs2|unknownwhat")) - def _test_serveReg_maxserve_infinite_roundrobin(self): + def test_serveReg_maxserve_infinite_roundrobin(self): # case for servers, they never exhaust self.pts.addEndpoint("A", "a|p") self.pts.addEndpoint("B", "a|p") @@ -92,7 +92,7 @@ class EndpointsTest(unittest.TestCase): served.add(self.pts._serveReg("ABC")) self.assertEquals(served, set("ABC")) - def _test_serveReg_maxserve_finite_exhaustion(self): + def test_serveReg_maxserve_finite_exhaustion(self): # case for clients, we don't want to keep serving them self.pts = Endpoints(af=socket.AF_INET, maxserve=5) self.pts.addEndpoint("A", "a|p") @@ -100,13 +100,28 @@ class EndpointsTest(unittest.TestCase): self.pts.addEndpoint("C", "a|p") # test getNumUnservedEndpoints whilst we're at it self.assertEquals(self.pts.getNumUnservedEndpoints(), 3) - for i in xrange(5): + served = set() + served.add(self.pts._serveReg("ABC")) + self.assertEquals(self.pts.getNumUnservedEndpoints(), 2) + served.add(self.pts._serveReg("ABC")) + self.assertEquals(self.pts.getNumUnservedEndpoints(), 1) + served.add(self.pts._serveReg("ABC")) + self.assertEquals(self.pts.getNumUnservedEndpoints(), 0) + self.assertEquals(served, set("ABC")) + for i in xrange(5-2): served = set() served.add(self.pts._serveReg("ABC")) served.add(self.pts._serveReg("ABC")) served.add(self.pts._serveReg("ABC")) self.assertEquals(served, set("ABC")) - self.assertEquals(None, self.pts._serveReg("ABC")) + remaining = set("ABC") + remaining.remove(self.pts._serveReg(remaining)) + self.assertRaises(KeyError, self.pts._serveReg, "ABC") + remaining.remove(self.pts._serveReg(remaining)) + self.assertRaises(KeyError, self.pts._serveReg, "ABC") + remaining.remove(self.pts._serveReg(remaining)) + self.assertRaises(KeyError, self.pts._serveReg, "ABC") + self.assertEquals(remaining, set()) self.assertEquals(self.pts.getNumUnservedEndpoints(), 0) def test_match_normal(self):

1 0

[flashproxy/master] revert prefix/suffix language back to inner/outer which is more consistent with layer-related language
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 75a0826ca0b46a4878ce1f0aa9aa3345e8e6520b Author: Ximin Luo <infinity0(a)gmx.com> Date: Tue Oct 8 19:45:53 2013 +0100 revert prefix/suffix language back to inner/outer which is more consistent with layer-related language --- facilitator/fac.py | 16 ++++----- facilitator/facilitator | 81 +++++++++++++++++++----------------------- facilitator/facilitator-test | 14 ++++---- 3 files changed, 52 insertions(+), 59 deletions(-) diff --git a/facilitator/fac.py b/facilitator/fac.py index cedadd8..6a670d3 100644 --- a/facilitator/fac.py +++ b/facilitator/fac.py @@ -148,26 +148,26 @@ def format_addr(addr): return u"%s%s" % (host_str, port_str) -class Transport(namedtuple("Transport", "prefix suffix")): +class Transport(namedtuple("Transport", "inner outer")): @classmethod def parse(cls, transport): if isinstance(transport, cls): return transport elif type(transport) == str: if "|" in transport: - prefix, suffix = transport.rsplit("|", 1) + inner, outer = transport.rsplit("|", 1) else: - prefix, suffix = "", transport - return cls(prefix, suffix) + inner, outer = "", transport + return cls(inner, outer) else: raise ValueError("could not parse transport: %s" % transport) - def __init__(self, prefix, suffix): - if not suffix: - raise ValueError("suffix (proxy) part of transport must be non-empty: %s" % str(self)) + def __init__(self, inner, outer): + if not outer: + raise ValueError("outer (proxy) part of transport must be non-empty: %s" % str(self)) def __str__(self): - return "%s|%s" % (self.prefix, self.suffix) if self.prefix else self.suffix + return "%s|%s" % (self.inner, self.outer) if self.inner else self.outer class Endpoint(namedtuple("Endpoint", "addr transport")): diff --git a/facilitator/facilitator b/facilitator/facilitator index 07038d1..d011013 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -87,16 +87,16 @@ class Endpoints(object): matchingLock = threading.Condition() - def __init__(self, af, maxserve=float("inf"), known_suf=("websocket",)): + def __init__(self, af, maxserve=float("inf"), known_outer=("websocket",)): self.af = af self._maxserve = maxserve self._endpoints = {} # address -> transport - self._indexes = {} # suffix -> [ addresses ] + self._indexes = {} # outer -> [ addresses ] self._served = {} # address -> num_times_served self._cv = threading.Condition() - self.known_suf = set(known_suf) - for suf in self.known_suf: - self._ensureIndexForSuffix(suf) + self.known_outer = set(known_outer) + for outer in self.known_outer: + self._ensureIndexForOuter(outer) def getNumEndpoints(self): """:returns: the number of endpoints known to us.""" @@ -149,23 +149,23 @@ class Endpoints(object): """ transport = Transport.parse(transport) with self._cv: - known_pre = self._findPrefixesForSuffixes(*self.known_suf).keys() - pre, suf = transport.prefix, transport.suffix - return pre in known_pre and suf in self.known_suf + known_inner = self._findInnerForOuter(*self.known_outer).keys() + inner, outer = transport.inner, transport.outer + return inner in known_inner and outer in self.known_outer - def _findPrefixesForSuffixes(self, *supported_suf): + def _findInnerForOuter(self, *supported_outer): """ - :returns: { prefix: [addr] }, where each address supports some suffix - from supported_suf. TODO(infinity0): describe better + :returns: { inner: [addr] }, where each address supports some outer + from supported_outer. TODO(infinity0): describe better """ - self.known_suf.update(supported_suf) - prefixes = {} - for suf in supported_suf: - self._ensureIndexForSuffix(suf) - for addr in self._indexes[suf]: - pre = self._endpoints[addr].prefix - prefixes.setdefault(pre, set()).add(addr) - return prefixes + self.known_outer.update(supported_outer) + inners = {} + for outer in supported_outer: + self._ensureIndexForOuter(outer) + for addr in self._indexes[outer]: + inner = self._endpoints[addr].inner + inners.setdefault(inner, set()).add(addr) + return inners def _avServed(self, addrpool): return sum(self._served[a] for a in addrpool) / float(len(addrpool)) @@ -186,49 +186,42 @@ class Endpoints(object): self.delEndpoint(prio_addr) return prio_addr - def _ensureIndexForSuffix(self, suf): - if suf in self._indexes: return + def _ensureIndexForOuter(self, outer): + if outer in self._indexes: return addrs = set(addr for addr, transport in self._endpoints.iteritems() - if transport.suffix == suf) - self._indexes[suf] = addrs + if transport.outer == outer) + self._indexes[outer] = addrs def _addAddrIntoIndexes(self, addr): - suf = self._endpoints[addr].suffix - if suf in self._indexes: self._indexes[suf].add(addr) + outer = self._endpoints[addr].outer + if outer in self._indexes: self._indexes[outer].add(addr) def _delAddrFromIndexes(self, addr): - suf = self._endpoints[addr].suffix - if suf in self._indexes: self._indexes[suf].remove(addr) - - def _prefixesForTransport(self, transport, *supported_suf): - for suf in supported_suf: - if not suf: - yield transport - elif transport[-len(suf):] == suf: - yield transport[:-len(suf)] + outer = self._endpoints[addr].outer + if outer in self._indexes: self._indexes[outer].remove(addr) EMPTY_MATCH = (None, None) @staticmethod - def match(ptsClient, ptsServer, supported_suf): + def match(ptsClient, ptsServer, supported_outer): """ :returns: A tuple (client Reg, server Reg) arbitrarily selected from - the available endpoints that can satisfy supported_suf. + the available endpoints that can satisfy supported_outer. """ if ptsClient.af != ptsServer.af: raise ValueError("address family not equal!") # need to operate on both structures # so hold both locks plus a pair-wise lock with Endpoints.matchingLock, ptsClient._cv, ptsServer._cv: - server_pre = ptsServer._findPrefixesForSuffixes(*supported_suf) - client_pre = ptsClient._findPrefixesForSuffixes(*supported_suf) - both = set(server_pre.keys()) & set(client_pre.keys()) + server_inner = ptsServer._findInnerForOuter(*supported_outer) + client_inner = ptsClient._findInnerForOuter(*supported_outer) + both = set(server_inner.keys()) & set(client_inner.keys()) if not both: return Endpoints.EMPTY_MATCH - # pick the prefix whose client address pool is least well-served + # pick the inner whose client address pool is least well-served # TODO: this may be manipulated by clients, needs research - assert all(client_pre.itervalues()) # no pool is empty - pre = min(both, key=lambda p: ptsClient._avServed(client_pre[p])) - client_addr = ptsClient._serveReg(client_pre[pre]) - server_addr = ptsServer._serveReg(server_pre[pre]) + assert all(client_inner.itervalues()) # no pool is empty + inner = min(both, key=lambda p: ptsClient._avServed(client_inner[p])) + client_addr = ptsClient._serveReg(client_inner[inner]) + server_addr = ptsServer._serveReg(server_inner[inner]) # assume servers never run out client_transport = ptsClient._endpoints[client_addr] server_transport = ptsServer._endpoints[server_addr] diff --git a/facilitator/facilitator-test b/facilitator/facilitator-test index 3f2fbef..8e06053 100755 --- a/facilitator/facilitator-test +++ b/facilitator/facilitator-test @@ -29,10 +29,10 @@ class EndpointsTest(unittest.TestCase): def setUp(self): self.pts = Endpoints(af=socket.AF_INET) - def _observeProxySupporting(self, *supported_suf): + def _observeProxySupporting(self, *supported_outer): # semantically observe the existence of a proxy, to make our intent - # a bit clearer than simply calling findPrefixesForSuffixes - self.pts._findPrefixesForSuffixes(*supported_suf) + # a bit clearer than simply calling _findInnerForOuter + self.pts._findInnerForOuter(*supported_outer) def test_addEndpoints_twice(self): self.pts.addEndpoint("A", "a|b|p") @@ -41,7 +41,7 @@ class EndpointsTest(unittest.TestCase): def test_addEndpoints_lazy_indexing(self): self.pts.addEndpoint("A", "a|b|p") - default_index = {"websocket": set()} # we always index known_suffixes + default_index = {"websocket": set()} # we always index known_outer # no index until we've asked for it self.assertEquals(self.pts._indexes, default_index) @@ -74,7 +74,7 @@ class EndpointsTest(unittest.TestCase): # we are fully capable of supporting them too, but only if we have # an endpoint that also speaks it. self.assertFalse(self.pts.supports("obfs3|unknownwhat")) - suf = self._observeProxySupporting("unknownwhat") + self._observeProxySupporting("unknownwhat") self.assertFalse(self.pts.supports("obfs3|unknownwhat")) self.pts.addEndpoint("A", "obfs3|unknownwhat") self.assertTrue(self.pts.supports("obfs3|unknownwhat")) @@ -155,7 +155,7 @@ class EndpointsTest(unittest.TestCase): self.assertEquals(expected, Endpoints.match(self.pts, self.pts2, ["p"])) self.assertEquals(empty, Endpoints.match(self.pts, self.pts2, ["x"])) - def test_match_many_prefixes(self): + def test_match_many_inners(self): self.pts.addEndpoint("A", "a|p") self.pts.addEndpoint("B", "b|p") self.pts.addEndpoint("C", "p") @@ -163,7 +163,7 @@ class EndpointsTest(unittest.TestCase): self.pts2.addEndpoint("D", "a|p") self.pts2.addEndpoint("E", "b|p") self.pts2.addEndpoint("F", "p") - # this test ensures we have a sane policy for selecting between prefix pools + # this test ensures we have a sane policy for selecting between inners pools expected = set() expected.add((Endpoint("A", Transport("a","p")), Endpoint("D", Transport("a","p")))) expected.add((Endpoint("B", Transport("b","p")), Endpoint("E", Transport("b","p"))))

1 0

[flashproxy/master] Add the (default) missing transport arg to put_reg in reg-daemon.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 86da700dff8902c4974c3f0ea194a3a8f1f23334 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Oct 17 02:37:55 2013 -0700 Add the (default) missing transport arg to put_reg in reg-daemon. --- facilitator/facilitator-reg-daemon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/facilitator/facilitator-reg-daemon b/facilitator/facilitator-reg-daemon index ed9eb7e..0eef4b4 100755 --- a/facilitator/facilitator-reg-daemon +++ b/facilitator/facilitator-reg-daemon @@ -125,7 +125,7 @@ class Handler(SocketServer.StreamRequestHandler): plaintext = rsa.private_decrypt(ciphertext, RSA.pkcs1_oaep_padding) client_addr = find_client_addr(plaintext) log(u"registering %s" % safe_str(fac.format_addr(client_addr))) - if fac.put_reg(FACILITATOR_ADDR, client_addr): + if fac.put_reg(FACILITATOR_ADDR, client_addr, "websocket"): print >> self.wfile, "OK" else: print >> self.wfile, "FAIL"

1 0

[flashproxy/master] Fix a type error in a log message.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit dce34bab36501355389ab18a1bfa1bc700b28838 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Oct 17 03:13:46 2013 -0700 Fix a type error in a log message. File "./facilitator", line 333, in do_PUT return self.error(u"Unrecognized transport: %s" % transport) TypeError: not all arguments converted during string formatting --- facilitator/facilitator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/facilitator/facilitator b/facilitator/facilitator index cbe3d98..71d0293 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -332,7 +332,7 @@ class Handler(SocketServer.StreamRequestHandler): transport = Transport.parse(transport) # See if we have relays that support this transport if transport.outer not in options.outer_transports: - return self.error(u"Unrecognized transport: %s" % transport) + return self.error(u"Unrecognized transport: %s" % transport.outer) client_spec = fac.param_first("CLIENT", params) if client_spec is None:

1 0

[flashproxy/master] Undo facilitator logging in facilitator-test.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 258615b56b7325a6b5c11b654e9463ec1159cf41 Author: David Fifield <david(a)bamsoftware.com> Date: Wed Oct 16 21:31:56 2013 -0700 Undo facilitator logging in facilitator-test. --- facilitator/facilitator-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/facilitator/facilitator-test b/facilitator/facilitator-test index 3efe34d..1c99b56 100755 --- a/facilitator/facilitator-test +++ b/facilitator/facilitator-test @@ -205,7 +205,7 @@ class FacilitatorProcTest(unittest.TestCase): self.relay_file.write("websocket [0:0::1:0]:1\n") self.relay_file.flush() self.relay_file.seek(0) - self.process = subprocess.Popen(["./facilitator", "-p", str(FACILITATOR_PORT), "-r", self.relay_file.name, "-d", "--unsafe-logging"]) + self.process = subprocess.Popen(["./facilitator", "-d", "-p", str(FACILITATOR_PORT), "-r", self.relay_file.name, "-l", "/dev/null"]) time.sleep(0.1) def tearDown(self):

1 0

[flashproxy/master] Update documentation of get_reg.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 37a0f5a1a4c466f0030dcebfd0168e82e5139036 Author: David Fifield <david(a)bamsoftware.com> Date: Wed Oct 16 21:28:00 2013 -0700 Update documentation of get_reg. --- facilitator/fac.py | 12 ++++++------ facilitator/facilitator | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/facilitator/fac.py b/facilitator/fac.py index 88365e5..842668d 100644 --- a/facilitator/fac.py +++ b/facilitator/fac.py @@ -295,13 +295,13 @@ def put_reg(facilitator_addr, client_addr, transport): def get_reg(facilitator_addr, proxy_addr, proxy_transport_list): """ - Get a client registration for proxy 'proxy_addr' from the - facilitator at 'facilitator_addr' using a one-time - socket. 'transports' is a list containing the transport names that + Get a client registration for proxy proxy_addr from the + facilitator at facilitator_addr using a one-time + socket. proxy_transport_list is a list containing the transport names that the flashproxy supports. - Returns a dict with keys "client-<transport>" and - "relay-<transport>" if successful, or a dict with the key "client" + Returns a dict with keys "client" and + "relay" if successful, or a dict with the key "client" mapped to the value "" if there are no registrations available for proxy_addr. Raises an exception otherwise.""" f = fac_socket(facilitator_addr) @@ -309,7 +309,7 @@ def get_reg(facilitator_addr, proxy_addr, proxy_transport_list): # Form a list (in transact() format) with the transports that we # should send to the facilitator. Then pass that list to the # transact() function. - # For example, TRANSPORT=obfs2 TRANSPORT=obfs3. + # For example, PROXY_TRANSPORT=obfs2 PROXY_TRANSPORT=obfs3. transports = [("PROXY_TRANSPORT", tp) for tp in proxy_transport_list] try: diff --git a/facilitator/facilitator b/facilitator/facilitator index 71d0293..670a01d 100755 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@ -287,7 +287,7 @@ class Handler(SocketServer.StreamRequestHandler): return False # Handle a GET request (got flashproxy poll; need to return a proper client registration) - # Example: GET FROM="3.3.3.3:3333" TRANSPORT="websocket" TRANSPORT="webrtc" + # Example: GET FROM="3.3.3.3:3333" PROXY_TRANSPORT="websocket" PROXY_TRANSPORT="webrtc" def do_GET(self, params): proxy_spec = fac.param_first("FROM", params) if proxy_spec is None:

1 0

[flashproxy/master] Remove debugging lines in tests.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 637a0be2873b841a29713fcc0c94d14ca671dcb6 Author: David Fifield <david(a)bamsoftware.com> Date: Wed Oct 16 21:32:35 2013 -0700 Remove debugging lines in tests. --- facilitator/facilitator-test | 2 -- 1 file changed, 2 deletions(-) diff --git a/facilitator/facilitator-test b/facilitator/facilitator-test index 1c99b56..040d4d4 100755 --- a/facilitator/facilitator-test +++ b/facilitator/facilitator-test @@ -155,9 +155,7 @@ class EndpointsTest(unittest.TestCase): self.pts.addEndpoint("A", "p") self.pts2 = Endpoints(af=socket.AF_INET, maxserve=2) self.pts2.addEndpoint("B", "p") - print self.pts2._indexes, self.pts2._served Endpoints.match(self.pts2, self.pts, ["p"]) - print self.pts2._indexes, self.pts2._served Endpoints.match(self.pts2, self.pts, ["p"]) empty = Endpoints.EMPTY_MATCH self.assertTrue("B" not in self.pts2._endpoints)

1 0