April 2012 - tor-commits - lists.torproject.org

[flashproxy/master] Don't allow bufferedAmount to grow too large.
by dcf＠torproject.org 09 Apr '12

09 Apr '12

commit 9428184e0850e623286ed6f289fca24c2ae1b766 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Apr 7 02:09:50 2012 -0700 Don't allow bufferedAmount to grow too large. --- flashproxy.js | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flashproxy.js b/flashproxy.js index 33dd85b..4e42106 100644 --- a/flashproxy.js +++ b/flashproxy.js @@ -450,6 +450,8 @@ function FlashProxy() { /* An instance of a client-relay connection. */ function ProxyPair(client_addr, relay_addr, rate_limit) { + var MAX_BUFFER = 10 * 1024 * 1024; + function log(s) { puts(s) @@ -557,13 +559,13 @@ function ProxyPair(client_addr, relay_addr, rate_limit) { var chunk; busy = false; - if (is_open(this.client_s) && this.r2c_schedule.length > 0) { + if (is_open(this.client_s) && this.client_s.bufferedAmount < MAX_BUFFER && this.r2c_schedule.length > 0) { chunk = this.r2c_schedule.shift(); this.rate_limit.update(chunk.length); this.client_s.send(chunk); busy = true; } - if (is_open(this.relay_s) && this.c2r_schedule.length > 0) { + if (is_open(this.relay_s) && this.relay_s.bufferedAmount < MAX_BUFFER && this.c2r_schedule.length > 0) { chunk = this.c2r_schedule.shift(); this.rate_limit.update(chunk.length); this.relay_s.send(chunk);

1 0

[flashproxy/master] Don't close the other half of the proxy pair if there is buffered data.
by dcf＠torproject.org 09 Apr '12

09 Apr '12

commit b724fe09e39f90eade0f42ac8f5b3ec6da2a9751 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Apr 7 02:25:04 2012 -0700 Don't close the other half of the proxy pair if there is buffered data. --- flashproxy.js | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flashproxy.js b/flashproxy.js index 4e42106..726b3ee 100644 --- a/flashproxy.js +++ b/flashproxy.js @@ -573,11 +573,11 @@ function ProxyPair(client_addr, relay_addr, rate_limit) { } } - if (is_closed(this.relay_s) && !is_closed(this.client_s) && this.r2c_schedule.length === 0) { + if (is_closed(this.relay_s) && !is_closed(this.client_s) && this.client_s.bufferedAmount === 0 && this.r2c_schedule.length === 0) { log("Client: closing."); this.client_s.close(); } - if (is_closed(this.client_s) && !is_closed(this.relay_s) && this.c2r_schedule.length === 0) { + if (is_closed(this.client_s) && !is_closed(this.relay_s) && this.relay_s.bufferedAmount === 0 && this.c2r_schedule.length === 0) { log("Relay: closing."); this.relay_s.close(); }

1 0

[flashproxy/master] More efficient apply_mask.
by dcf＠torproject.org 09 Apr '12

09 Apr '12

commit 73e7ffaed410124daecc8edb3fab662a6414e8ee Author: David Fifield <david(a)bamsoftware.com> Date: Sat Apr 7 04:36:19 2012 -0700 More efficient apply_mask. --- connector.py | 22 +++++++++++++++++----- 1 files changed, 17 insertions(+), 5 deletions(-) diff --git a/connector.py b/connector.py index 23de63d..1689c33 100755 --- a/connector.py +++ b/connector.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import array import base64 import cStringIO import getopt @@ -121,11 +122,22 @@ def format_addr(addr): def apply_mask(payload, mask_key): - result = [] - for i, c in enumerate(payload): - mc = chr(ord(payload[i]) ^ ord(mask_key[i%4])) - result.append(mc) - return "".join(result) + result = array.array("l") + result.fromstring(payload[:len(payload) // 4 * 4]) + m, = struct.unpack("=l", mask_key) + i = 0 + while i < len(result): + result[i] ^= m + i += 1 + result = result.tostring() + i *= 4 + if i < len(payload): + remainder = [] + while i < len(payload): + remainder.append(chr(ord(payload[i]) ^ ord(mask_key[i%4]))) + i += 1 + result = result + "".join(remainder) + return result class WebSocketFrame(object): def __init__(self):

1 0

[flashproxy/master] Recognize a WebSocket close opcode as EOF.
by dcf＠torproject.org 09 Apr '12

09 Apr '12

commit 6904c9fe83c799f9822f82ab466a5ded17e14c86 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Apr 7 03:40:58 2012 -0700 Recognize a WebSocket close opcode as EOF. --- connector.py | 64 +++++++++++++++++++++++++++++++++++---------------------- 1 files changed, 39 insertions(+), 25 deletions(-) diff --git a/connector.py b/connector.py index 5bdbda4..23de63d 100755 --- a/connector.py +++ b/connector.py @@ -325,6 +325,8 @@ class WebSocketBinaryDecoder(object): self.dec.feed(data) def read(self): + """Returns None when there are currently no data to be read. Returns "" + when a close message is received.""" while True: message = self.dec.read_message() if message is None: @@ -332,9 +334,14 @@ class WebSocketBinaryDecoder(object): elif message.opcode == 1: if not self.base64: raise ValueError("Received text message on decoder incapable of base64") - return base64.b64decode(message.payload) + payload = base64.b64decode(message.payload) + if payload: + return payload elif message.opcode == 2: - return message.payload + if message.payload: + return message.payload + elif message.opcode == 8: + return "" # Ignore all other opcodes. return None @@ -557,13 +564,14 @@ def register(): except OSError, e: log(u"Failed to register: %s" % str(e)) -def proxy_chunk_local_to_remote(local, remote): - try: - data = local.recv(65536) - except socket.error, e: # Can be "Connection reset by peer". - log(u"Socket error from local: %s" % repr(str(e))) - remote.close() - return False +def proxy_chunk_local_to_remote(local, remote, data = None): + if data is None: + try: + data = local.recv(65536) + except socket.error, e: # Can be "Connection reset by peer". + log(u"Socket error from local: %s" % repr(str(e))) + remote.close() + return False if not data: log(u"EOF from local %s." % format_peername(local)) local.close() @@ -573,20 +581,31 @@ def proxy_chunk_local_to_remote(local, remote): remote.send_chunk(data) return True -def proxy_chunk_remote_to_local(remote, local): - try: - data = remote.recv(65536) - except socket.error, e: # Can be "Connection reset by peer". - log(u"Socket error from remote: %s" % repr(str(e))) - local.close() - return False +def proxy_chunk_remote_to_local(remote, local, data = None): + if data is None: + try: + data = remote.recv(65536) + except socket.error, e: # Can be "Connection reset by peer". + log(u"Socket error from remote: %s" % repr(str(e))) + local.close() + return False if not data: log(u"EOF from remote %s." % format_peername(remote)) remote.close() local.close() return False else: - local.send_chunk(data) + remote.dec.feed(data) + while True: + data = remote.dec.read() + if data is None: + break + elif not data: + log(u"WebSocket close from remote %s." % format_peername(remote)) + remote.close() + local.close() + return False + local.send_chunk(data) return True def receive_unlinked(fd, label): @@ -623,9 +642,9 @@ def match_proxies(): remote.partner = local local.partner = remote if local.buf: - remote.send_chunk(local.buf) + proxy_chunk_remote_to_local(remote, local, local.buf) if remote.buf: - local.send_chunk(remote.buf) + proxy_chunk_local_to_remote(local, remote, remote.buf) class TimeoutSocket(object): def __init__(self, fd): @@ -659,12 +678,7 @@ class LocalSocket(object): self.partner = None def send_chunk(self, data): - self.partner.dec.feed(data) - while True: - data = self.partner.dec.read() - if not data: - break - self.sendall(data) + self.sendall(data) def __getattr__(self, name): return getattr(self.fd, name)

1 0

[flashproxy/master] Call flush while any data is buffered.
by dcf＠torproject.org 09 Apr '12

09 Apr '12

commit a9491dd508f8bf1fe11a371f99b332bafd428743 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Apr 7 04:44:46 2012 -0700 Call flush while any data is buffered. This is so that an open socket can be closed after sending all its data, after its partner has closed. --- flashproxy.js | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/flashproxy.js b/flashproxy.js index 726b3ee..bbadc46 100644 --- a/flashproxy.js +++ b/flashproxy.js @@ -582,7 +582,8 @@ function ProxyPair(client_addr, relay_addr, rate_limit) { this.relay_s.close(); } - if (this.r2c_schedule.length > 0 || this.c2r_schedule.length > 0) + if (this.r2c_schedule.length > 0 || this.client_s.bufferedAmount > 0 + || this.c2r_schedule.length > 0 || this.relay_s.bufferedAmount > 0) this.flush_timeout_id = setTimeout(this.flush.bind(this), this.rate_limit.when() * 1000); }; }

1 0

[flashproxy/master] Add a test of a long message with an odd number of bytes.
by dcf＠torproject.org 09 Apr '12

09 Apr '12

commit 5e6fdd100f23926050964c6f9885d5f4a53b3b42 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Apr 7 04:59:08 2012 -0700 Add a test of a long message with an odd number of bytes. --- connector-test.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/connector-test.py b/connector-test.py index a15c1ad..36c9208 100755 --- a/connector-test.py +++ b/connector-test.py @@ -47,6 +47,9 @@ class TestWebSocketDecoder(unittest.TestCase): ("\x82\x7f\x00\x00\x00\x00\x00\x01\x00\x00" + "\x00" * 65536, False, [(True, 2, "\x00" * 65536)], [(2, "\x00" * 65536)]), + ("\x82\x7f\x00\x00\x00\x00\x00\x01\x00\x03" + "ABCD" * 16384 + "XYZ", False, + [(True, 2, "ABCD" * 16384 + "XYZ")], + [(2, "ABCD" * 16384 + "XYZ")]), ] for data, use_mask, expected_frames, expected_messages in TESTS: dec = WebSocketDecoder(use_mask = use_mask)

1 0

[flashproxy/master] Don't apply_mask when we don't need to.
by dcf＠torproject.org 09 Apr '12

09 Apr '12

commit 0c99f3dd9fab26a421bf30d9224c0839b9d93b6c Author: David Fifield <david(a)bamsoftware.com> Date: Sat Apr 7 04:36:50 2012 -0700 Don't apply_mask when we don't need to. --- connector.py | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diff --git a/connector.py b/connector.py index 1689c33..9c725be 100755 --- a/connector.py +++ b/connector.py @@ -219,14 +219,17 @@ class WebSocketDecoder(object): # "The server MUST close the connection upon receiving a frame # that is not masked." raise self.MaskingError("Got unmasked payload from client") - mask_key = "\x00\x00\x00\x00" + mask_key = None if payload_len > self.MAX_MESSAGE_LENGTH: raise ValueError("Refusing to buffer payload of %d bytes" % payload_len) if len(self.buf) < offset + payload_len: return None - payload = apply_mask(self.buf[offset:offset+payload_len], mask_key) + if mask_key: + payload = apply_mask(self.buf[offset:offset+payload_len], mask_key) + else: + payload = self.buf[offset:offset+payload_len] self.buf = self.buf[offset+payload_len:] frame = WebSocketFrame()

1 0

[flashproxy/master] Add a --register option to the connector.
by dcf＠torproject.org 09 Apr '12

09 Apr '12

commit 6b54d70dd731952236cd08ef9bb4ba9be209f427 Author: David Fifield <david(a)bamsoftware.com> Date: Fri Apr 6 12:34:47 2012 -0700 Add a --register option to the connector. Now it doesn't register by default but only when you use this option. --- README | 4 ++-- connector.py | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/README b/README index b593328..e6ad198 100644 --- a/README +++ b/README @@ -58,12 +58,12 @@ registration with the facilitator: everything else happens over ephemeral proxies. 1. Run the connector. - $ python connector.py + $ python connector.py --register By default the connector listens on Internet-facing TCP port 9000. If you have to use a different port (to get through a firewall, for example), give it on the command lines like this (here using port 8888): - $ python connector.py 127.0.0.1:9001 :8888 + $ python connector.py --register 127.0.0.1:9001 :8888 2. Run Tor using the included torrc file. $ tor -f flashproxy/torrc diff --git a/connector.py b/connector.py index 9c725be..8653907 100755 --- a/connector.py +++ b/connector.py @@ -34,6 +34,7 @@ class options(object): log_filename = None log_file = sys.stdout daemonize = False + register = False pid_filename = None # We accept up to this many bytes from a socket not yet matched with a partner @@ -42,7 +43,7 @@ UNCONNECTED_BUFFER_LIMIT = 10240 def usage(f = sys.stdout): print >> f, """\ -Usage: %(progname)s -f FACILITATOR[:PORT] [LOCAL][:PORT] [REMOTE][:PORT] +Usage: %(progname)s --register [LOCAL][:PORT] [REMOTE][:PORT] Wait for connections on a local and a remote port. When any pair of connections exists, data is ferried between them until one side is closed. By default LOCAL is "%(local)s" and REMOTE is "%(remote)s". @@ -51,14 +52,17 @@ The local connection acts as a SOCKS4a proxy, but the host and port in the SOCKS request are ignored and the local connection is always linked to a remote connection. -The REMOTE address is registered with the given FACILITATOR:PORT (or whatever -is the default of the flashproxy-reg-http.py program). +If the --register option is used, then your IP address will be sent to the +facilitator so that proxies can connect to you. You need to register in some way +in order to get any service. The --facilitator option allows controlling which +facilitator is used; if omitted, it uses a public default. --daemon daemonize (Unix only). -f, --facilitator=HOST[:PORT] advertise willingness to receive connections to HOST:PORT. -h, --help show this help. -l, --log FILENAME write log to FILENAME (default stdout). - --pidfile FILENAME write PID to FILENAME after daemonizing.\ + --pidfile FILENAME write PID to FILENAME after daemonizing. + -r, --register register with the facilitator.\ """ % { "progname": sys.argv[0], "local": format_addr((DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT)), @@ -567,6 +571,9 @@ def report_pending(): log(u"remotes (%d): %s" % (len(remotes), [format_peername(x) for x in remotes])) def register(): + if not options.register: + return + spec = format_addr((None, options.remote_addr[1])) command = ["./flashproxy-reg-http.py"] if options.facilitator_addr is None: @@ -768,7 +775,7 @@ def main(): report_pending() if __name__ == "__main__": - opts, args = getopt.gnu_getopt(sys.argv[1:], "f:hl:", ["daemon", "facilitator=", "help", "log=", "pidfile="]) + opts, args = getopt.gnu_getopt(sys.argv[1:], "f:hl:r", ["daemon", "facilitator=", "help", "log=", "pidfile=", "register"]) for o, a in opts: if o == "--daemon": options.daemonize = True @@ -781,6 +788,8 @@ if __name__ == "__main__": options.log_filename = a elif o == "--pidfile": options.pid_filename = a + elif o == "-r" or o == "--register": + options.register = True if len(args) == 0: options.local_addr = (DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT)

1 0

[flashproxy/master] Change throughput.sh test for WebSocket proxy.
by dcf＠torproject.org 09 Apr '12

09 Apr '12

commit 04153e9244c49c051d0c39f9f116dcc317f06d27 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Apr 7 05:10:56 2012 -0700 Change throughput.sh test for WebSocket proxy. --- experiments/throughput/throughput.sh | 12 ++++++++++-- 1 files changed, 10 insertions(+), 2 deletions(-) diff --git a/experiments/throughput/throughput.sh b/experiments/throughput/throughput.sh index 672f2ac..fc61fb5 100755 --- a/experiments/throughput/throughput.sh +++ b/experiments/throughput/throughput.sh @@ -8,6 +8,10 @@ # Results are saved in a file called results-NUM_CLIENTS-DATE, where DATE is the # current date. +# plain socks ws ws plain +# httpget <---> socat <---> connector <---> flashproxy <---> websockify <---> thttpd +# 2000 9001 9000 8001 8000 + . ../common.sh NUM_CLIENTS=1 @@ -19,7 +23,7 @@ while getopts "n:" OPTNAME; do done PROFILE=flashexp1 -PROXY_URL="http://127.0.0.1:8000/swfcat.swf?facilitator=127.0.0.1:9002&max_clients=$NU…" +PROXY_URL="http://127.0.0.1:8000/embed.html?facilitator=127.0.0.1:9002&max_clients=$NU…" DATA_FILE_NAME="$FLASHPROXY_DIR/dump" RESULTS_FILE_NAME="results-$NUM_CLIENTS-$(date --iso)" @@ -44,8 +48,12 @@ echo "Start web server." "$THTTPD" -D -d "$FLASHPROXY_DIR" -p 8000 & PIDS_TO_KILL+=($!) +echo "Start websockify." +"$WEBSOCKIFY" -v 8001 127.0.0.1:8000 >/dev/null & +PIDS_TO_KILL+=($!) + echo "Start facilitator." -"$FLASHPROXY_DIR"/facilitator.py -d --relay 127.0.0.1:8000 >/dev/null & +"$FLASHPROXY_DIR"/facilitator.py -d --relay 127.0.0.1:8001 127.0.0.1 9002 >/dev/null & PIDS_TO_KILL+=($!) visible_sleep 1

1 0

[flashproxy/master] Gracefully handle a masking error from a remote.
by dcf＠torproject.org 09 Apr '12

09 Apr '12

commit 9da7c9a065ebb74027d0618327a786227ef3d245 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Apr 7 11:55:12 2012 -0600 Gracefully handle a masking error from a remote. --- connector.py | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diff --git a/connector.py b/connector.py index d10a6ab..a0b7149 100755 --- a/connector.py +++ b/connector.py @@ -620,7 +620,13 @@ def proxy_chunk_remote_to_local(remote, local, data = None): else: remote.dec.feed(data) while True: - data = remote.dec.read() + try: + data = remote.dec.read() + except (WebSocketDecoder.MaskingError, ValueError), e: + log(u"WebSocket decode error from remote: %s" % repr(str(e))) + remote.close() + local.close() + return False if data is None: break elif not data:

1 0