October 2013 - tor-commits - lists.torproject.org

[flashproxy/master] Parse CLIENT-TRANSPORT and RELAY-TRANSPORT in get_reg.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 1367540bc467b7fa34d39387d27b2014bb3a67b9 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Oct 17 02:07:24 2013 -0700 Parse CLIENT-TRANSPORT and RELAY-TRANSPORT in get_reg. --- facilitator/fac.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/facilitator/fac.py b/facilitator/fac.py index 842668d..5e917d6 100644 --- a/facilitator/fac.py +++ b/facilitator/fac.py @@ -300,8 +300,8 @@ def get_reg(facilitator_addr, proxy_addr, proxy_transport_list): socket. proxy_transport_list is a list containing the transport names that the flashproxy supports. - Returns a dict with keys "client" and - "relay" if successful, or a dict with the key "client" + Returns a dict with keys "client", "client-transport", "relay", + and "relay-transport" if successful, or a dict with the key "client" mapped to the value "" if there are no registrations available for proxy_addr. Raises an exception otherwise.""" f = fac_socket(facilitator_addr) @@ -329,16 +329,24 @@ def get_reg(facilitator_addr, proxy_addr, proxy_transport_list): return response elif command == "OK": client_spec = param_first("CLIENT", params) + client_transport = param_first("CLIENT-TRANSPORT", params) relay_spec = param_first("RELAY", params) + relay_transport = param_first("RELAY-TRANSPORT", params) if not client_spec: raise ValueError("Facilitator did not return CLIENT") + if not client_transport: + raise ValueError("Facilitator did not return CLIENT-TRANSPORT") if not relay_spec: raise ValueError("Facilitator did not return RELAY") + if not relay_transport: + raise ValueError("Facilitator did not return RELAY-TRANSPORT") # Check the syntax returned by the facilitator. client = parse_addr_spec(client_spec) relay = parse_addr_spec(relay_spec) response["client"] = format_addr(client) + response["client-transport"] = client_transport response["relay"] = format_addr(relay) + response["relay-transport"] = relay_transport return response else: raise ValueError("Facilitator response was not \"OK\"")

1 0

[flashproxy/master] Make find_client_addr return an Endpoint.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 8494d2b226fb56f68652dde8720533d0e7613d55 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Oct 17 02:44:57 2013 -0700 Make find_client_addr return an Endpoint. --- facilitator/fac.py | 4 +++- facilitator/facilitator-reg-daemon | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/facilitator/fac.py b/facilitator/fac.py index 58db4cf..1ad1abd 100644 --- a/facilitator/fac.py +++ b/facilitator/fac.py @@ -160,7 +160,9 @@ def find_client_addr(body): client_specs = qs["client"] if len(client_specs) != 1: continue - return parse_addr_spec(client_specs[0]) + addr = parse_addr_spec(client_specs[0]) + transport = "websocket" + return Endpoint(addr, transport) return None diff --git a/facilitator/facilitator-reg-daemon b/facilitator/facilitator-reg-daemon index 77a6bc8..33921dd 100755 --- a/facilitator/facilitator-reg-daemon +++ b/facilitator/facilitator-reg-daemon @@ -107,9 +107,9 @@ class Handler(SocketServer.StreamRequestHandler): try: ciphertext = b64_ciphertext.decode("base64") plaintext = rsa.private_decrypt(ciphertext, RSA.pkcs1_oaep_padding) - client_addr = fac.find_client_addr(plaintext) - log(u"registering %s" % safe_str(fac.format_addr(client_addr))) - if fac.put_reg(FACILITATOR_ADDR, client_addr, "websocket"): + client_reg = fac.find_client_addr(plaintext) + log(u"registering %s" % safe_str(fac.format_addr(client_reg.addr))) + if fac.put_reg(FACILITATOR_ADDR, client_reg.addr, client_reg.transport): print >> self.wfile, "OK" else: print >> self.wfile, "FAIL"

1 0

[flashproxy/master] Add tests for read_client_registrations.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit e120069b3910213fb70a93be0633d53ecc41b608 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Oct 17 03:28:41 2013 -0700 Add tests for read_client_registrations. --- facilitator/facilitator-test | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/facilitator/facilitator-test b/facilitator/facilitator-test index 6b45f6f..35c2e2e 100755 --- a/facilitator/facilitator-test +++ b/facilitator/facilitator-test @@ -382,5 +382,50 @@ class ParseTransactionTest(unittest.TestCase): def test_newline(self): self.assertRaises(ValueError, fac.parse_transaction, "COMMAND X=\"ABC\" \nY=\"DEF\"") +class ReadClientRegistrationsTest(unittest.TestCase): + def testSingle(self): + l = list(fac.read_client_registrations("")) + self.assertEqual(len(l), 0) + l = list(fac.read_client_registrations("client=1.2.3.4:1111")) + self.assertEqual(len(l), 1) + self.assertEqual(l[0].addr, ("1.2.3.4", 1111)) + l = list(fac.read_client_registrations("client=1.2.3.4:1111\n")) + self.assertEqual(len(l), 1) + self.assertEqual(l[0].addr, ("1.2.3.4", 1111)) + l = list(fac.read_client_registrations("foo=bar&client=1.2.3.4:1111&baz=quux")) + self.assertEqual(len(l), 1) + self.assertEqual(l[0].addr, ("1.2.3.4", 1111)) + l = list(fac.read_client_registrations("foo=b%3dar&client=1.2.3.4%3a1111")) + self.assertEqual(len(l), 1) + self.assertEqual(l[0].addr, ("1.2.3.4", 1111)) + l = list(fac.read_client_registrations("client=%5b1::2%5d:3333")) + self.assertEqual(len(l), 1) + self.assertEqual(l[0].addr, ("1::2", 3333)) + + def testDefaultTransport(self): + l = list(fac.read_client_registrations("client=1.2.3.4:1111")) + self.assertEqual(l[0].transport, "websocket") + + def testMultiple(self): + l = list(fac.read_client_registrations("client=1.2.3.4:1111&foo=bar\nfoo=bar&client=5.6.7.8:2222")) + self.assertEqual(len(l), 2) + self.assertEqual(l[0].addr, ("1.2.3.4", 1111)) + self.assertEqual(l[1].addr, ("5.6.7.8", 2222)) + l = list(fac.read_client_registrations("client=1.2.3.4:1111&foo=bar\nfoo=bar&client=%5b1::2%5d:3333")) + self.assertEqual(len(l), 2) + self.assertEqual(l[0].addr, ("1.2.3.4", 1111)) + self.assertEqual(l[1].addr, ("1::2", 3333)) + + def testInvalid(self): + # Missing "client". + with self.assertRaises(ValueError): + list(fac.read_client_registrations("foo=bar")) + # More than one "client". + with self.assertRaises(ValueError): + list(fac.read_client_registrations("client=1.2.3.4:1111&foo=bar&client=5.6.7.8:2222")) + # Single client with bad syntax. + with self.assertRaises(ValueError): + list(fac.read_client_registrations("client=1.2.3.4,1111")) + if __name__ == "__main__": unittest.main()

1 0

[flashproxy/master] Move find_client_addr to fac.py.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit b7c3727f60420c65fcba6d7c870080887fac7840 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Oct 17 02:18:21 2013 -0700 Move find_client_addr to fac.py. This is going to be used in common by facilitator-reg-daemon and facilitator.cgi (the two front-end interfaces to the facilitator that exist). Currently we're cheating a bit and using cgi.FieldStorage to parse the POST body in facilitator.cgi, which happens to work because of the common URL syntax. --- facilitator/fac.py | 16 ++++++++++++++++ facilitator/facilitator-reg-daemon | 18 +----------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/facilitator/fac.py b/facilitator/fac.py index 5e917d6..58db4cf 100644 --- a/facilitator/fac.py +++ b/facilitator/fac.py @@ -5,6 +5,7 @@ import socket import stat import subprocess import pwd +import urlparse from collections import namedtuple # Return true iff the given fd is readable, writable, and executable only by its @@ -147,6 +148,21 @@ def format_addr(addr): raise ValueError("host and port may not both be None") return u"%s%s" % (host_str, port_str) +def find_client_addr(body): + """Find and parse the first client line of the form + client=... + Returns None if no client line was found.""" + for line in body.splitlines(): + try: + qs = urlparse.parse_qs(line, keep_blank_values=True, strict_parsing=True) + except ValueError: + continue + client_specs = qs["client"] + if len(client_specs) != 1: + continue + return parse_addr_spec(client_specs[0]) + return None + class Transport(namedtuple("Transport", "inner outer")): @classmethod diff --git a/facilitator/facilitator-reg-daemon b/facilitator/facilitator-reg-daemon index 0eef4b4..77a6bc8 100755 --- a/facilitator/facilitator-reg-daemon +++ b/facilitator/facilitator-reg-daemon @@ -7,7 +7,6 @@ import socket import sys import threading import time -import urlparse import fac @@ -77,21 +76,6 @@ def log(msg): finally: log_lock.release() -def find_client_addr(body): - """Find and parse the first client line of the form - client=... - Returns None if no client line was found.""" - for line in body.splitlines(): - try: - qs = urlparse.parse_qs(line, keep_blank_values=True, strict_parsing=True) - except ValueError: - continue - client_specs = qs["client"] - if len(client_specs) != 1: - continue - return fac.parse_addr_spec(client_specs[0]) - return None - class Handler(SocketServer.StreamRequestHandler): def __init__(self, *args, **kwargs): self.deadline = time.time() + CLIENT_TIMEOUT @@ -123,7 +107,7 @@ class Handler(SocketServer.StreamRequestHandler): try: ciphertext = b64_ciphertext.decode("base64") plaintext = rsa.private_decrypt(ciphertext, RSA.pkcs1_oaep_padding) - client_addr = find_client_addr(plaintext) + client_addr = fac.find_client_addr(plaintext) log(u"registering %s" % safe_str(fac.format_addr(client_addr))) if fac.put_reg(FACILITATOR_ADDR, client_addr, "websocket"): print >> self.wfile, "OK"

1 0

[flashproxy/master] Change find_client_addr to read_client_registrations.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 856b6573dec23b6296a5facbd3b71ae9aa1f2fa0 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Oct 17 02:55:36 2013 -0700 Change find_client_addr to read_client_registrations. This new function yields all the registrations that exist in a message, and parses "client-transport". --- facilitator/fac.py | 34 +++++++++++++++++++--------------- facilitator/facilitator-reg-daemon | 11 ++++++----- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/facilitator/fac.py b/facilitator/fac.py index 1ad1abd..8daa73c 100644 --- a/facilitator/fac.py +++ b/facilitator/fac.py @@ -8,6 +8,8 @@ import pwd import urlparse from collections import namedtuple +DEFAULT_CLIENT_TRANSPORT = "websocket" + # Return true iff the given fd is readable, writable, and executable only by its # owner. def check_perms(fd): @@ -148,22 +150,24 @@ def format_addr(addr): raise ValueError("host and port may not both be None") return u"%s%s" % (host_str, port_str) -def find_client_addr(body): - """Find and parse the first client line of the form - client=... - Returns None if no client line was found.""" +def get_single(qs, key, default=None): + try: + vals = qs[key] + except KeyError: + if default is None: + raise ValueError("missing %r key" % key) + vals = (default,) + if len(vals) != 1: + raise ValueError("more than one %r key" % key) + return vals[0] + +def read_client_registrations(body): + """Parse the lines of body and yield an Endpoint for each.""" for line in body.splitlines(): - try: - qs = urlparse.parse_qs(line, keep_blank_values=True, strict_parsing=True) - except ValueError: - continue - client_specs = qs["client"] - if len(client_specs) != 1: - continue - addr = parse_addr_spec(client_specs[0]) - transport = "websocket" - return Endpoint(addr, transport) - return None + qs = urlparse.parse_qs(line, keep_blank_values=True, strict_parsing=True) + addr = parse_addr_spec(get_single(qs, "client")) + transport = get_single(qs, "client-transport", DEFAULT_CLIENT_TRANSPORT) + yield Endpoint(addr, transport) class Transport(namedtuple("Transport", "inner outer")): diff --git a/facilitator/facilitator-reg-daemon b/facilitator/facilitator-reg-daemon index 33921dd..04f2102 100755 --- a/facilitator/facilitator-reg-daemon +++ b/facilitator/facilitator-reg-daemon @@ -107,12 +107,13 @@ class Handler(SocketServer.StreamRequestHandler): try: ciphertext = b64_ciphertext.decode("base64") plaintext = rsa.private_decrypt(ciphertext, RSA.pkcs1_oaep_padding) - client_reg = fac.find_client_addr(plaintext) - log(u"registering %s" % safe_str(fac.format_addr(client_reg.addr))) - if fac.put_reg(FACILITATOR_ADDR, client_reg.addr, client_reg.transport): - print >> self.wfile, "OK" + for client_reg in fac.read_client_registrations(plaintext): + log(u"registering %s" % safe_str(fac.format_addr(client_reg.addr))) + if not fac.put_reg(FACILITATOR_ADDR, client_reg.addr, client_reg.transport): + print >> self.wfile, "FAIL" + break else: - print >> self.wfile, "FAIL" + print >> self.wfile, "OK" except Exception, e: log("error registering: %s" % str(e)) print >> self.wfile, "FAIL"

1 0

[flashproxy/master] Add defhost and defport to read_client_registrations.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 13325d09e1201398a745ac4a03e12e2b597a97c0 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Oct 17 04:27:02 2013 -0700 Add defhost and defport to read_client_registrations. Inspired by "http registration: set default remote address properly - put_reg FROM is never actually read by facilitator" https://github.com/infinity0/flashproxy/commit/4b8d3d65fbc78eca400e6c33e998… --- facilitator/fac.py | 4 ++-- facilitator/facilitator-test | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/facilitator/fac.py b/facilitator/fac.py index 8daa73c..a1ac614 100644 --- a/facilitator/fac.py +++ b/facilitator/fac.py @@ -161,11 +161,11 @@ def get_single(qs, key, default=None): raise ValueError("more than one %r key" % key) return vals[0] -def read_client_registrations(body): +def read_client_registrations(body, defhost=None, defport=None): """Parse the lines of body and yield an Endpoint for each.""" for line in body.splitlines(): qs = urlparse.parse_qs(line, keep_blank_values=True, strict_parsing=True) - addr = parse_addr_spec(get_single(qs, "client")) + addr = parse_addr_spec(get_single(qs, "client"), defhost, defport) transport = get_single(qs, "client-transport", DEFAULT_CLIENT_TRANSPORT) yield Endpoint(addr, transport) diff --git a/facilitator/facilitator-test b/facilitator/facilitator-test index 35c2e2e..024972a 100755 --- a/facilitator/facilitator-test +++ b/facilitator/facilitator-test @@ -402,6 +402,12 @@ class ReadClientRegistrationsTest(unittest.TestCase): self.assertEqual(len(l), 1) self.assertEqual(l[0].addr, ("1::2", 3333)) + def testDefaultAddress(self): + l = list(fac.read_client_registrations("client=:1111&transport=websocket", defhost="1.2.3.4")) + self.assertEqual(l[0].addr, ("1.2.3.4", 1111)) + l = list(fac.read_client_registrations("client=1.2.3.4:&transport=websocket", defport=1111)) + self.assertEqual(l[0].addr, ("1.2.3.4", 1111)) + def testDefaultTransport(self): l = list(fac.read_client_registrations("client=1.2.3.4:1111")) self.assertEqual(l[0].transport, "websocket")

1 0

[flashproxy/master] have the facilitator only respond with transport suffixes to the proxy
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit a259e8d43b39ba059fb3a3f5ad22913ddb234878 Author: Ximin Luo <infinity0(a)gmx.com> Date: Tue Oct 8 19:20:52 2013 +0100 have the facilitator only respond with transport suffixes to the proxy --- doc/design.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/design.txt b/doc/design.txt index 472a188..a0c4d98 100644 --- a/doc/design.txt +++ b/doc/design.txt @@ -178,6 +178,10 @@ Design of flash proxies For example: client-websocket=1.2.3.4:2000&relay-websocket=10.10.10:9902 + As with the request, the response transports are actually outer + transports; inner transports are not the proxy's concern and therefore + not given. + If the value for the client parameter is empty, it means that there are no client registrations for this proxy.

1 0

[flashproxy/master] - move FieldStorage() inside do_get() as it consumes stdin and messes with do_post()
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit d77510152be2edf60572a83926b4d5daee6bd976 Author: Ximin Luo <infinity0(a)gmx.com> Date: Fri Oct 11 18:05:33 2013 +0100 - move FieldStorage() inside do_get() as it consumes stdin and messes with do_post() --- facilitator/facilitator.cgi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/facilitator/facilitator.cgi b/facilitator/facilitator.cgi index 624580f..05286fd 100755 --- a/facilitator/facilitator.cgi +++ b/facilitator/facilitator.cgi @@ -31,7 +31,6 @@ path_info = os.environ.get("PATH_INFO") or "/" if not method or not remote_addr[0]: exit_error(400) -fs = cgi.FieldStorage() # Print the HEAD part of a URL-based registration response, or exit with an # error if appropriate. @@ -55,6 +54,7 @@ def do_get(): """Parses flashproxy polls. Example: GET /?r=1&client=7.1.43.21&client=1.2.3.4&transport=webrtc&transport=websocket """ + fs = cgi.FieldStorage() path_parts = [x for x in path_info.split("/") if x] if len(path_parts) == 2 and path_parts[0] == "reg":

1 0

[flashproxy/master] Add new client-transport and relay-transport to design.txt.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit d5f1ac431f3c5e297a945722e665c725448dfebf Author: David Fifield <david(a)bamsoftware.com> Date: Thu Oct 17 04:42:26 2013 -0700 Add new client-transport and relay-transport to design.txt. This is the design.txt part of "New unified url-param syntax for both facilitator responses and client registrations." https://github.com/infinity0/flashproxy/commit/0377b7aba8026710bd2168b6ceb3… --- doc/design.txt | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/doc/design.txt b/doc/design.txt index a0c4d98..94f8403 100644 --- a/doc/design.txt +++ b/doc/design.txt @@ -126,14 +126,16 @@ Design of flash proxies POST / HTTP/1.0 - client-<transport>=[<address>]:<port>[&client-<transport>=[<address>]:<port>...] + client=[<address>]:<port>[&client-transport=<transport>][ + client=[<address>]:<port>[&client-transport=<transport>] ...] Where 'transport' is the name of the pluggable transport that is listening on <address>:<port>. The default flashproxy transport is named 'websocket'. - For example a registration message would look like this: - client-websocket=1.2.3.4:9000&client-obfs3|websocket=1.2.3.4:10000 + For example a registration message might look like this: + client=1.2.3.4:9000 + client=1.2.3.4:10000&client-transport=obfs3|websocket The facilitator sends a 200 reply if the registration was successful and an error status otherwise. If the transport plugin omits the @@ -163,20 +165,21 @@ Design of flash proxies The 'r' parameter is the protocol revision number (should be '1' for now). The 'client' parameter carries the IP address of a flashproxy - client. The client parameter can can repeat to report multiple + client. The client parameter can repeat to report multiple connected clients. - The 'transport' parameter is optional and signals the transports that - this flashproxy supports. + The 'transport' parameter may be repeated zero or many times and + signals the outer-transports that this flashproxy supports. (See + section 10 for a discussion of inner and outer transports.) For example: GET /?r=1&client=7.1.43.21:9999&client=1.2.3.4:9000&transport=webrtc&transport=websocket HTTP/1.0 The response code is 200 and the body looks like this: - client-<transport>=[<address>:<port>]&relay-<transport>=<address>:<port> + client=<address>:<port>&client-transport=<transport>&relay=<address>:<port>&relay-transport=<transport> For example: - client-websocket=1.2.3.4:2000&relay-websocket=10.10.10:9902 + client=1.2.3.4:2000&client-transport=websocket&relay=10.10.10:9902&relay-transport=websocket As with the request, the response transports are actually outer transports; inner transports are not the proxy's concern and therefore @@ -200,3 +203,7 @@ Design of flash proxies 9. Behavior of the Tor relay. The Tor relay requires no special configuration. + +10. Inner and outer transports + + TODO: import discussion from #9349.

1 0

[flashproxy/master] Use read_client_registrations in facilitator.cgi.
by infinity0＠torproject.org 28 Oct '13

28 Oct '13

commit 6eccd89f1c83b98047e47dd9d9db04a668b1d084 Author: David Fifield <david(a)bamsoftware.com> Date: Thu Oct 17 04:38:05 2013 -0700 Use read_client_registrations in facilitator.cgi. This takes care of the other registration mechanism (apart from facilitator-reg-daemon). Now we're using the client=<addr:port>&client-transport=websocket style, rather than the client-websocket=<addr:port> style. This is basically the same as the facilitator.cgi part of "New unified url-param syntax for both facilitator responses and client registrations." https://github.com/infinity0/flashproxy/commit/0377b7aba8026710bd2168b6ceb3… --- facilitator/facilitator.cgi | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/facilitator/facilitator.cgi b/facilitator/facilitator.cgi index 05286fd..47497b0 100755 --- a/facilitator/facilitator.cgi +++ b/facilitator/facilitator.cgi @@ -92,7 +92,8 @@ def do_post(): # Old style client registration: # client=1.2.3.4:9000 # New style client registration: - # client-websocket=1.2.3.4:9000&client-obfs3|websocket=1.2.3.4:10000 + # client-addr=1.2.3.4:9000&client-transport=websocket + # client-addr=1.2.3.4:9090&client-transport=obfs3|websocket if path_info != "/": exit_error(400) @@ -101,29 +102,15 @@ def do_post(): # them look like "client-websocket=1.2.3.4:9000". We then split # all those items and send them as separate registrations to the # facilitator. - for key in fs.keys(): - if key != "client" and not key.startswith("client-"): - continue - - if key == "client": # reg without transport info -- default to websocket. - transport = "websocket" - else: # reg with transport info -- get the "websocket" part out of "client-websocket". - transport = key[len("client-"):] - - # Get the "1.2.3.4:9000" part of "client-websocket=1.2.3.4:9000". - client_spec = fs[key].value.strip() - try: - client_addr = fac.parse_addr_spec(client_spec, defhost=remote_addr[0]) - except ValueError: - exit_error(400) - - # XXX what if previous registrations passed through - # successfully, but the last one failed and called - # exit_error()? + try: + regs = list(fac.read_client_registrations(sys.stdin.read(), defhost=remote_addr[0])) + except ValueError: + exit_error(400) + for reg in regs: # XXX need to link these registrations together, so that - # when one is answerered the rest are invalidated. - if not fac.put_reg(FACILITATOR_ADDR, client_addr, transport): + # when one is answerered (or errors) the rest are invalidated. + if not fac.put_reg(FACILITATOR_ADDR, reg.addr, reg.transport): exit_error(500) print """\

1 0