commit 5843e89467a30679a64bd10d52b4c7f0124886d7 Merge: fd692bd 8cc46cf Author: Ximin Luo <infinity0@gmx.com> Date: Sat Oct 19 17:19:39 2013 +0100 Merge branch 'bug9349_server_endpoints' into merge-next Conflicts: facilitator/Makefile facilitator/fac.py facilitator/facilitator facilitator/facilitator.cgi doc/design.txt | 36 ++++- facilitator/Makefile | 2 + facilitator/fac.py | 68 +++++++- facilitator/facilitator | 347 +++++++++++++++++++++++++++------------- facilitator/facilitator-test | 201 +++++++++++++++++++++-- facilitator/facilitator.cgi | 60 +++++-- facilitator/facilitator.py | 1 + facilitator/init.d/facilitator | 8 +- facilitator/relays | 2 + 9 files changed, 572 insertions(+), 153 deletions(-) diff --cc facilitator/Makefile index 07ed125,e69f9a7..52bd201 --- a/facilitator/Makefile +++ b/facilitator/Makefile @@@ -6,9 -5,11 +6,11 @@@ all : install: - mkdir -p $(BINDIR) - cp -f facilitator facilitator-email-poller facilitator-reg-daemon facilitator-reg facilitator.cgi fac.py $(BINDIR) - mkdir -p /etc/flashproxy - cp -f relays /etc/flashproxy - cp -f init.d/facilitator init.d/facilitator-email-poller init.d/facilitator-reg-daemon /etc/init.d/ + mkdir -p $(DESTDIR)$(BINDIR) + cp -f facilitator facilitator-email-poller facilitator-reg-daemon facilitator-reg facilitator.cgi fac.py $(DESTDIR)$(BINDIR) ++ mkdir -p $(DESTDIR)/etc/flashproxy ++ cp -f relays $(DESTDIR)/etc/flashproxy + cp -f init.d/facilitator init.d/facilitator-email-poller init.d/facilitator-reg-daemon $(DESTDIR)/etc/init.d/ clean: rm -f *.pyc diff --cc facilitator/fac.py index 5e334a5,8e37d39..88365e5 --- a/facilitator/fac.py +++ b/facilitator/fac.py @@@ -239,11 -281,14 +281,12 @@@ def transact(f, command, *params) raise ValueError("No newline at end of string returned by facilitator") return parse_transaction(line[:-1]) - def put_reg(facilitator_addr, client_addr): -def put_reg(facilitator_addr, client_addr, transport, registrant_addr=None): ++def put_reg(facilitator_addr, client_addr, transport): """Send a registration to the facilitator using a one-time socket. Returns true iff the command was successful.""" f = fac_socket(facilitator_addr) params = [("CLIENT", format_addr(client_addr))] + params.append(("TRANSPORT", transport)) - if registrant_addr is not None: - params.append(("FROM", format_addr(registrant_addr))) try: command, params = transact(f, "PUT", *params) finally: diff --cc facilitator/facilitator index f7ebf10,63272f1..cbe3d98 --- a/facilitator/facilitator +++ b/facilitator/facilitator @@@ -51,10 -57,13 +57,13 @@@ again. Listen on 127.0.0.1 and port POR -d, --debug don't daemonize, log to stdout. -h, --help show this help. -l, --log FILENAME write log to FILENAME (default \"%(log)s\"). - -p, --port PORT listen on PORT (by default %(port)d). + -p, --port PORT listen on PORT (default %(port)d). --pidfile FILENAME write PID to FILENAME after daemonizing. --privdrop-user USER switch UID and GID to those of USER. - -r, --relay RELAY send RELAY (host:port) to proxies as the relay to use. + -r, --relay-file RELAY learn relays from FILE. + --outer-transports TRANSPORTS + comma-sep list of outer transports to accept proxies + for (by default %(outer-transports)s) --unsafe-logging don't scrub IP addresses from logs.\ """ % { "progname": sys.argv[0], @@@ -215,11 -283,8 +281,13 @@@ class Handler(SocketServer.StreamReques def send_error(self): print >> self.wfile, "ERROR" + def error(self, log_msg): + log(log_msg) + self.send_error() + return False + + # Handle a GET request (got flashproxy poll; need to return a proper client registration) + # Example: GET FROM="3.3.3.3:3333" TRANSPORT="websocket" TRANSPORT="webrtc" def do_GET(self, params): proxy_spec = fac.param_first("FROM", params) if proxy_spec is None: @@@ -227,40 -294,78 +295,64 @@@ try: proxy_addr = fac.parse_addr_spec(proxy_spec, defport=0) except ValueError, e: - log(u"syntax error in proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e))))) - self.send_error() - return False + return self.error(u"syntax error in proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e))))) + transport_list = fac.param_getlist("PROXY_TRANSPORT", params) + if not transport_list: - log(u"PROXY TRANSPORT missing FROM param") - self.send_error() - return False ++ return self.error(u"PROXY TRANSPORT missing FROM param") + try: - reg = get_reg_for_proxy(proxy_addr) + client_reg, relay_reg = get_match_for_proxy(proxy_addr, transport_list) except Exception, e: - log(u"error getting reg for proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e))))) - self.send_error() - return False + return self.error(u"error getting reg for proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e))))) + check_back_in = get_check_back_in_for_proxy(proxy_addr) - if reg: - log(u"proxy gets %s, relay %s (now %d)" % - (safe_str(unicode(reg)), options.relay_spec, num_regs())) - print >> self.wfile, fac.render_transaction("OK", ("CLIENT", str(reg)), ("RELAY", options.relay_spec), ("CHECK-BACK-IN", str(check_back_in))) + + if client_reg: + log(u"proxy (%s) gets client '%s' (supported transports: %s) (num relays: %s) (remaining regs: %d/%d)" % + (safe_str(repr(proxy_spec)), safe_str(repr(client_reg.addr)), transport_list, num_relays(), num_unhandled_regs(), num_regs())) + print >> self.wfile, fac.render_transaction("OK", + ("CLIENT", fac.format_addr(client_reg.addr)), + ("RELAY", fac.format_addr(relay_reg.addr)), + ("CHECK-BACK-IN", str(check_back_in))) else: - log(u"proxy gets none") + log(u"proxy (%s) gets none" % safe_str(repr(proxy_spec))) print >> self.wfile, fac.render_transaction("NONE", ("CHECK-BACK-IN", str(check_back_in))) + return True + # Handle a PUT request (client made a registration request; register it.) + # Example: PUT CLIENT="1.1.1.1:5555" TRANSPORT="obfs3|websocket" def do_PUT(self, params): + # Check out if we recognize the transport in this registration request + transport = fac.param_first("TRANSPORT", params) + if transport is None: - log(u"PUT missing TRANSPORT param") - self.send_error() - return False ++ return self.error(u"PUT missing TRANSPORT param") + + transport = Transport.parse(transport) + # See if we have relays that support this transport + if transport.outer not in options.outer_transports: + return self.error(u"Unrecognized transport: %s" % transport) + client_spec = fac.param_first("CLIENT", params) if client_spec is None: - log(u"PUT missing CLIENT param") - self.send_error() - return False + return self.error(u"PUT missing CLIENT param") try: - reg = Reg.parse(client_spec) - except ValueError, e: + reg = Endpoint.parse(client_spec, transport) + except (UnknownTransport, ValueError) as e: + # XXX should we throw a better error message to the client? Is it possible? - log(u"syntax error in %s: %s" % (safe_str(repr(client_spec)), safe_str(repr(str(e))))) - self.send_error() - return False + return self.error(u"syntax error in %s: %s" % (safe_str(repr(client_spec)), safe_str(repr(str(e))))) try: ok = put_reg(reg) except Exception, e: - log(u"error putting reg %s: %s" % (safe_str(repr(client_spec)), safe_str(repr(str(e))))) - self.send_error() - return False + return self.error(u"error putting reg %s: %s" % (safe_str(repr(client_spec)), safe_str(repr(str(e))))) + if ok: - log(u"client %s (now %d)" % (safe_str(unicode(reg)), num_regs())) + log(u"client %s (transports: %s) (remaining regs: %d/%d)" % (safe_str(unicode(reg)), reg.transport, num_unhandled_regs(), num_regs())) else: - log(u"client %s (already present, now %d)" % (safe_str(unicode(reg)), num_regs())) + log(u"client %s (already present) (transports: %s) (remaining regs: %d/%d)" % (safe_str(unicode(reg)), reg.transport, num_unhandled_regs(), num_regs())) self.send_ok() return True @@@ -307,22 -419,30 +406,38 @@@ def get_check_back_in_for_proxy(proxy_a def put_reg(reg): """Add a registration.""" - addr_str = reg.host - af = addr_af(addr_str) - REGS = regs_for_af(af) - return REGS.add(reg) + af = addr_af(reg.addr[0]) + return CLIENTS[af].addEndpoint(reg.addr, reg.transport) + + def parse_relay_file(servers, fp): + """Parse a file containing Tor relays that we can point proxies to. + Throws ValueError on a parsing error. Each line contains a transport chain + and an address, for example + obfs2|websocket 1.4.6.1:4123 + """ + for line in fp.readlines(): + try: + transport_spec, addr_spec = line.strip().split() + except ValueError, e: + raise ValueError("Wrong line format: %s." % repr(line)) + addr = fac.parse_addr_spec(addr_spec, defport=DEFAULT_RELAY_PORT, resolve=True) + transport = Transport.parse(transport_spec) + if transport.outer not in options.outer_transports: + raise ValueError(u"Unrecognized transport: %s" % transport) + af = addr_af(addr[0]) + servers[af].addEndpoint(addr, transport) def main(): - opts, args = getopt.gnu_getopt(sys.argv[1:], "dhl:p:r:", - ["debug", "help", "log=", "port=", "pidfile=", "privdrop-user=", "relay-file=", "unsafe-logging"]) + opts, args = getopt.gnu_getopt(sys.argv[1:], "dhl:p:r:", [ + "debug", + "help", + "log=", + "port=", + "pidfile=", + "privdrop-user=", - "relay=", ++ "relay-file=", + "unsafe-logging", + ]) for o, a in opts: if o == "-d" or o == "--debug": options.daemonize = False diff --cc facilitator/facilitator.cgi index e4e9bdf,a68d84e..624580f --- a/facilitator/facilitator.cgi +++ b/facilitator/facilitator.cgi @@@ -78,18 -88,45 +87,45 @@@ Access-Control-Allow-Origin: *\ exit_error(400) def do_post(): + """Parse client registration.""" + + # Old style client registration: + # client=1.2.3.4:9000 + # New style client registration: + # client-websocket=1.2.3.4:9000&client-obfs3|websocket=1.2.3.4:10000 + if path_info != "/": exit_error(400) - client_specs = fs.getlist("client") - if len(client_specs) != 1: - exit_error(400) - client_spec = client_specs[0].strip() - try: - client_addr = fac.parse_addr_spec(client_spec, defhost=remote_addr[0]) - except ValueError: - exit_error(400) - if not fac.put_reg(FACILITATOR_ADDR, client_addr): - exit_error(500) + + # We iterate through the items in the POST body, and see if any of + # them look like "client-websocket=1.2.3.4:9000". We then split + # all those items and send them as separate registrations to the + # facilitator. + for key in fs.keys(): + if key != "client" and not key.startswith("client-"): + continue + + if key == "client": # reg without transport info -- default to websocket. + transport = "websocket" + else: # reg with transport info -- get the "websocket" part out of "client-websocket". + transport = key[len("client-"):] + + # Get the "1.2.3.4:9000" part of "client-websocket=1.2.3.4:9000". + client_spec = fs[key].value.strip() + try: + client_addr = fac.parse_addr_spec(client_spec, defhost=remote_addr[0]) + except ValueError: + exit_error(400) + + # XXX what if previous registrations passed through + # successfully, but the last one failed and called + # exit_error()? + + # XXX need to link these registrations together, so that + # when one is answerered the rest are invalidated. - if not fac.put_reg(FACILITATOR_ADDR, client_addr, transport, remote_addr): ++ if not fac.put_reg(FACILITATOR_ADDR, client_addr, transport): + exit_error(500) + print """\ Status: 200\r \r"""