March 2012 - tor-commits - lists.torproject.org

[metrics-web/master] Add metrics-lib's descriptor.jar to build.xml.
by karsten＠torproject.org 30 Mar '12

30 Mar '12

commit f9f41b984963006b082fec70eaa46fb1e8b1ab47 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Mar 30 09:38:08 2012 +0200 Add metrics-lib's descriptor.jar to build.xml. --- build.xml | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/build.xml b/build.xml index 8f57532..24715e9 100644 --- a/build.xml +++ b/build.xml @@ -15,6 +15,7 @@ <pathelement location="lib/commons-codec-1.4.jar"/> <pathelement location="lib/postgresql-8.4-702.jdbc3.jar"/> <pathelement location="lib/junit-4.8.2.jar"/> + <pathelement location="lib/descriptor.jar"/> </path> <target name="init">

1 0

[metrics-lib/master] Add metrics-web to the list of software using this lib.
by karsten＠torproject.org 30 Mar '12

30 Mar '12

commit db33202e446c8d35b5f31a22fb47c665a13a4c8f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Mar 30 09:40:08 2012 +0200 Add metrics-web to the list of software using this lib. --- README | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/README b/README index 93c2cee..292cab6 100644 --- a/README +++ b/README @@ -34,4 +34,5 @@ the list below (software name, repository, contact person): - DocTor, https://git.torproject.org/doctor.git, karsten - Onionoo, https://git.torproject.org/onionoo.git, karsten + - metrics-web, https://git.torproject.org/metrics-web.git, karsten

1 0

[flashproxy/js] Give remote hosts 2 seconds to make their WebSocket request.
by dcf＠torproject.org 30 Mar '12

30 Mar '12

commit d469ec913900395011245fb53b7a0a1bf139d1da Author: David Fifield <david(a)bamsoftware.com> Date: Thu Mar 29 19:26:03 2012 -0700 Give remote hosts 2 seconds to make their WebSocket request. --- connector.py | 26 ++++++++++++++++++++++++-- 1 files changed, 24 insertions(+), 2 deletions(-) diff --git a/connector.py b/connector.py index 07f0f3e..908f988 100755 --- a/connector.py +++ b/connector.py @@ -358,6 +358,9 @@ def format_peername(s): except socket.error, e: return "<unconnected>" +# How long to wait for a WebSocket request on the remote socket. It is limited +# to avoid Slowloris-like attacks. +WEBSOCKET_REQUEST_TIMEOUT = 2.0 class WebSocketRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def __init__(self, request_text, fd): @@ -601,6 +604,17 @@ def match_proxies(): if remote.buf: local.send_chunk(remote.buf) +class TimeoutSocket(object): + def __init__(self, fd): + self.fd = fd + self.birthday = time.time() + + def age(self): + return time.time() - self.birthday + + def __getattr__(self, name): + return getattr(self.fd, name) + class RemoteSocket(object): def __init__(self, fd, protocols): self.fd = fd @@ -635,12 +649,12 @@ class LocalSocket(object): def main(): while True: rset = [remote_s, local_s] + websocket_pending + socks_pending + locals + remotes - rset, _, _ = select.select(rset, [], []) + rset, _, _ = select.select(rset, [], [], WEBSOCKET_REQUEST_TIMEOUT) for fd in rset: if fd == remote_s: remote_c, addr = fd.accept() log(u"Remote connection from %s." % format_addr(addr)) - websocket_pending.append(remote_c) + websocket_pending.append(TimeoutSocket(remote_c)) elif fd == local_s: local_c, addr = fd.accept() log(u"Local connection from %s." % format_addr(addr)) @@ -687,6 +701,14 @@ def main(): locals.remove(fd) report_pending() match_proxies() + while websocket_pending: + pending = websocket_pending[0] + if pending.age() < WEBSOCKET_REQUEST_TIMEOUT: + break + log(u"Expired remote connection from %s." % format_peername(pending)) + pending.close() + websocket_pending.pop(0) + report_pending() if __name__ == "__main__": opts, args = getopt.gnu_getopt(sys.argv[1:], "hl:", ["daemon", "help", "log=", "pidfile="])

1 0

[flashproxy/js] Remove crossdomain policy sending from connector.
by dcf＠torproject.org 30 Mar '12

30 Mar '12

commit 15d91390d63ae0fccf4fa0538886b35bb47addbf Author: David Fifield <david(a)bamsoftware.com> Date: Sat Mar 24 23:42:11 2012 -0700 Remove crossdomain policy sending from connector. --- connector.py | 63 ++++----------------------------------------------------- 1 files changed, 5 insertions(+), 58 deletions(-) diff --git a/connector.py b/connector.py index 68f1ce4..85fc4ad 100755 --- a/connector.py +++ b/connector.py @@ -180,22 +180,11 @@ def format_peername(s): except socket.error, e: return "<unconnected>" -# How long to wait for a crossdomain policy request before deciding that this is -# a normal socket. In 160 crossdomain connections to the public facilitator, the -# mean time before receiving a crossdomain request is only 0.05625 s. But that -# was measured from log entries with a one-second resolution, so give a healthy -# margin above that. -CROSSDOMAIN_TIMEOUT = 0.5 - # Local socket, accepting SOCKS requests from localhost local_s = listen_socket(options.local_addr) -# Remote socket, accepting both crossdomain policy requests and remote proxy -# connections. +# Remote socket, accepting remote WebSocket connections from proxies. remote_s = listen_socket(options.remote_addr) -# Sockets that may be crossdomain policy requests or may be normal remote -# connections. -crossdomain_pending = [] # Remote connection sockets. remotes = [] # New local sockets waiting to finish their SOCKS negotiation. @@ -208,31 +197,6 @@ local_for = {} remote_for = {} -def handle_policy_request(fd): - """Returns True iff the socket is still open and usable (wasn't a - crossdomain request and wasn't closed.""" - log(u"handle_policy_request") - try: - addr = fd.getpeername() - data = fd.recv(100) - except socket.error, e: - log(u"Socket error from crossdomain-pending: %s" % repr(str(e))) - return False - if data == "<policy-file-request/>\0": - log(u"Sending crossdomain policy to %s." % format_addr(addr)) - fd.sendall(""" -<cross-domain-policy> -<allow-access-from domain="*" to-ports="%s"/> -</cross-domain-policy> -\0""" % xml.sax.saxutils.escape(str(options.remote_addr[1]))) - return False - elif data == "": - log(u"No data from %s." % format_addr(addr)) - return False - else: - fd.buf += data - return True - def grab_string(s, pos): """Grab a NUL-terminated string from the given string, starting at the given offset. Return (pos, str) tuple, or (pos, None) on error.""" @@ -377,27 +341,19 @@ register() def main(): while True: - rset = [remote_s, local_s] + crossdomain_pending + socks_pending + remote_for.keys() + local_for.keys() + locals + remotes - rset, _, _ = select.select(rset, [], [], CROSSDOMAIN_TIMEOUT) + rset = [remote_s, local_s] + socks_pending + remote_for.keys() + local_for.keys() + locals + remotes + rset, _, _ = select.select(rset, [], []) for fd in rset: if fd == remote_s: remote_c, addr = fd.accept() log(u"Remote connection from %s." % format_addr(addr)) - crossdomain_pending.append(BufferSocket(remote_c)) + remotes.append(fd) + handle_remote_connection(fd) elif fd == local_s: local_c, addr = fd.accept() log(u"Local connection from %s." % format_addr(addr)) socks_pending.append(local_c) register() - elif fd in crossdomain_pending: - log(u"Data from crossdomain-pending %s." % format_addr(addr)) - if handle_policy_request(fd): - remotes.append(fd) - handle_remote_connection(fd) - else: - fd.close() - crossdomain_pending.remove(fd) - report_pending() elif fd in socks_pending: log(u"SOCKS request from %s." % format_addr(addr)) if handle_socks_request(fd): @@ -428,15 +384,6 @@ def main(): remotes.remove(fd) report_pending() match_proxies() - while crossdomain_pending: - pending = crossdomain_pending[0] - if not pending.is_expired(CROSSDOMAIN_TIMEOUT): - break - log(u"Expired pending crossdomain from %s." % format_peername(pending)) - crossdomain_pending.pop(0) - remotes.append(pending) - handle_remote_connection(pending) - report_pending() try: main()

1 0

[flashproxy/js] Add ServerTransportPlugin line for websocket transport.
by dcf＠torproject.org 30 Mar '12

30 Mar '12

commit eea30d38701d66e308c3d0ce3023bcb69f72186a Author: David Fifield <david(a)bamsoftware.com> Date: Sun Mar 25 10:51:23 2012 -0700 Add ServerTransportPlugin line for websocket transport. --- README | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/README b/README index d362ed7..46776d5 100644 --- a/README +++ b/README @@ -125,6 +125,8 @@ The relay runs with a WebSocket proxy in external proxy mode. This command proxies WebSocket on port 9901 to a local ORPort on 9001. ./websockify :9901 127.0.0.1:9001 +Add this line to torrc to make the relay use the external proxy: + ServerTransportPlugin websocket proxy 127.0.0.1:9901 == How to put a flash proxy badge on a web page.

1 0

[flashproxy/js] websockify command example.
by dcf＠torproject.org 30 Mar '12

30 Mar '12

commit b5048c334224bd67b89689df7892bc4efc21eb83 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Mar 24 22:22:59 2012 -0700 websockify command example. --- README | 10 +++++++++- 1 files changed, 9 insertions(+), 1 deletions(-) diff --git a/README b/README index cd15ba7..d362ed7 100644 --- a/README +++ b/README @@ -118,6 +118,14 @@ If tor apparently hangs here: wait a few minutes. It can take a while to download relay descriptors. +== How to run a relay + +The relay runs with a WebSocket proxy in external proxy mode. + git clone git://github.com/kanaka/websockify.git +This command proxies WebSocket on port 9901 to a local ORPort on 9001. + ./websockify :9901 127.0.0.1:9001 + + == How to put a flash proxy badge on a web page. Paste in this HTML where you want the badge to appear: @@ -129,7 +137,7 @@ Paste in this HTML where you want the badge to appear: See design.txt for some more technical information. -=== Running a relay or facilitator +=== Running a facilitator On the facilitator, run $ ./facilitator.py -r <relay-ip>

1 0

[flashproxy/js] Remove more Flash stuff.
by dcf＠torproject.org 30 Mar '12

30 Mar '12

commit 5cdb99b25b101624fd1135a9a4e204cfffe737f0 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Mar 24 22:18:56 2012 -0700 Remove more Flash stuff. --- README | 26 +------------------------- 1 files changed, 1 insertions(+), 25 deletions(-) diff --git a/README b/README index ced9527..cd15ba7 100644 --- a/README +++ b/README @@ -9,8 +9,7 @@ There are five main parts. Our terminology for each part is in quotes. 1. The Tor "client," running on someone's localhost. 2. A "connector," which waits for connections from the flash proxy and the Tor client, and joins them together. -3. A "flash proxy," running in someone's web browser. This piece is - called swfcat because it is like a netcat implemented in Flash. +3. A "flash proxy," an application running in someone's web browser. 4. A "facilitator," a pseudo-HTTP server that keeps a list of clients that want a connection, and hands them out to proxies. 5. A Tor "relay," which is just a normal Tor relay. @@ -130,29 +129,6 @@ Paste in this HTML where you want the badge to appear: See design.txt for some more technical information. -=== Building swfcat.swf - -Download the Flex SDK. You only need the "Open Source Flex SDK," not the -"Adobe Flex SDK." - http://opensource.adobe.com/wiki/display/flexsdk/Flex+SDK -Put its bin directory in your PATH. The important executable is mxmlc. - -To build, run - $ make -Copy the resulting swfcat.swf file to a web server. - -You can pass swfcat.swf the parameter debug=1 to see lots of debugging -information. - -Adobe makes a debug version of Flash player that raises a window on -errors instead of silently failing, and exposes some logging functions. - http://www.adobe.com/support/flashplayer/downloads.html - -This is a good tutorial on ActionScript programming with the Flex tools, -with sample code: - http://www.senocular.com/flash/tutorials/as3withmxmlc/ - http://www.senocular.com/flash/tutorials/as3withmxmlc/AS3Flex2b3StarterFile… - === Running a relay or facilitator On the facilitator, run

1 0

[flashproxy/js] Make debug output use pre not div.
by dcf＠torproject.org 30 Mar '12

30 Mar '12

commit 6928215914947d6f722b580c313180dde8565da6 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Mar 24 22:33:43 2012 -0700 Make debug output use pre not div. --- embed.html | 1 + flashproxy.js | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/embed.html b/embed.html index 591f492..c88827b 100644 --- a/embed.html +++ b/embed.html @@ -9,6 +9,7 @@ body { } #flashproxy-badge.debug { position: absolute; + margin: 0; left: 0; top: 0; width: 100%; diff --git a/flashproxy.js b/flashproxy.js index 43c2342..32497e6 100644 --- a/flashproxy.js +++ b/flashproxy.js @@ -91,7 +91,7 @@ function format_addr(addr) function FlashProxy() { - this.debug_div = document.createElement("div"); + this.debug_div = document.createElement("pre"); this.debug_div.className = "debug"; this.badge_elem = this.debug_div; @@ -103,8 +103,7 @@ function FlashProxy() /* http://www.w3.org/TR/cssom-view/#element-scrolling-members */ at_bottom = (this.debug_div.scrollTop + this.debug_div.clientHeight == this.debug_div.scrollHeight); - this.debug_div.appendChild(document.createTextNode(s)); - this.debug_div.appendChild(document.createElement("br")); + this.debug_div.appendChild(document.createTextNode(s + "\n")); if (at_bottom) this.debug_div.scrollTop = this.debug_div.scrollHeight; }

1 0

[flashproxy/js] Do the __name__ == "__main__" trick in connector.py.
by dcf＠torproject.org 30 Mar '12

30 Mar '12

commit c6e3c10ef6e73ca9fd6fe730699e1f2161114df3 Author: David Fifield <david(a)bamsoftware.com> Date: Sun Mar 25 10:53:23 2012 -0700 Do the __name__ == "__main__" trick in connector.py. --- connector.py | 139 +++++++++++++++++++++++++++++----------------------------- 1 files changed, 70 insertions(+), 69 deletions(-) diff --git a/connector.py b/connector.py index 85fc4ad..f84f8a2 100755 --- a/connector.py +++ b/connector.py @@ -115,40 +115,6 @@ def format_addr(addr): else: return u"%s:%d" % (host, port) -opts, args = getopt.gnu_getopt(sys.argv[1:], "f:hl:", ["daemon", "facilitator=", "help", "log=", "pidfile="]) -for o, a in opts: - if o == "--daemon": - options.daemonize = True - elif o == "-f" or o == "--facilitator": - options.facilitator_addr = parse_addr_spec(a, None, DEFAULT_FACILITATOR_PORT) - elif o == "-h" or o == "--help": - usage() - sys.exit() - elif o == "-l" or o == "--log": - options.log_filename = a - elif o == "--pidfile": - options.pid_filename = a - -if len(args) == 0: - options.local_addr = (DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT) - options.remote_addr = (DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT) -elif len(args) == 1: - options.local_addr = parse_addr_spec(args[0], DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT) - options.remote_addr = (DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT) -elif len(args) == 2: - options.local_addr = parse_addr_spec(args[0], DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT) - options.remote_addr = parse_addr_spec(args[1], DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT) -else: - usage(sys.stderr) - sys.exit(1) - -if options.log_filename: - options.log_file = open(options.log_filename, "a") - # Send error tracebacks to the log. - sys.stderr = options.log_file -else: - options.log_file = sys.stdout - class BufferSocket(object): """A socket containing a time of creation and a buffer of data received. The @@ -180,22 +146,6 @@ def format_peername(s): except socket.error, e: return "<unconnected>" -# Local socket, accepting SOCKS requests from localhost -local_s = listen_socket(options.local_addr) -# Remote socket, accepting remote WebSocket connections from proxies. -remote_s = listen_socket(options.remote_addr) - -# Remote connection sockets. -remotes = [] -# New local sockets waiting to finish their SOCKS negotiation. -socks_pending = [] -# Local Tor sockets, after SOCKS negotiation. -locals = [] - -# Bidirectional mapping between local sockets and remote sockets. -local_for = {} -remote_for = {} - def grab_string(s, pos): """Grab a NUL-terminated string from the given string, starting at the given @@ -327,18 +277,6 @@ def match_proxies(): remote_for[local.fd] = remote.fd local_for[remote.fd] = local.fd -if options.daemonize: - log(u"Daemonizing.") - pid = os.fork() - if pid != 0: - if options.pid_filename: - f = open(options.pid_filename, "w") - print >> f, pid - f.close() - sys.exit(0) - -register() - def main(): while True: rset = [remote_s, local_s] + socks_pending + remote_for.keys() + local_for.keys() + locals + remotes @@ -347,8 +285,9 @@ def main(): if fd == remote_s: remote_c, addr = fd.accept() log(u"Remote connection from %s." % format_addr(addr)) - remotes.append(fd) - handle_remote_connection(fd) + remotes.append(BufferSocket(remote_c)) + handle_remote_connection(remote_c) + report_pending() elif fd == local_s: local_c, addr = fd.accept() log(u"Local connection from %s." % format_addr(addr)) @@ -385,8 +324,70 @@ def main(): report_pending() match_proxies() -try: - main() -except Exception: - exc = traceback.format_exc() - log("".join(exc)) +if __name__ == "__main__": + opts, args = getopt.gnu_getopt(sys.argv[1:], "f:hl:", ["daemon", "facilitator=", "help", "log=", "pidfile="]) + for o, a in opts: + if o == "--daemon": + options.daemonize = True + elif o == "-f" or o == "--facilitator": + options.facilitator_addr = parse_addr_spec(a, None, DEFAULT_FACILITATOR_PORT) + elif o == "-h" or o == "--help": + usage() + sys.exit() + elif o == "-l" or o == "--log": + options.log_filename = a + elif o == "--pidfile": + options.pid_filename = a + + if len(args) == 0: + options.local_addr = (DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT) + options.remote_addr = (DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT) + elif len(args) == 1: + options.local_addr = parse_addr_spec(args[0], DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT) + options.remote_addr = (DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT) + elif len(args) == 2: + options.local_addr = parse_addr_spec(args[0], DEFAULT_LOCAL_ADDRESS, DEFAULT_LOCAL_PORT) + options.remote_addr = parse_addr_spec(args[1], DEFAULT_REMOTE_ADDRESS, DEFAULT_REMOTE_PORT) + else: + usage(sys.stderr) + sys.exit(1) + + if options.log_filename: + options.log_file = open(options.log_filename, "a") + # Send error tracebacks to the log. + sys.stderr = options.log_file + else: + options.log_file = sys.stdout + + # Local socket, accepting SOCKS requests from localhost + local_s = listen_socket(options.local_addr) + # Remote socket, accepting remote WebSocket connections from proxies. + remote_s = listen_socket(options.remote_addr) + + # Remote connection sockets. + remotes = [] + # New local sockets waiting to finish their SOCKS negotiation. + socks_pending = [] + # Local Tor sockets, after SOCKS negotiation. + locals = [] + + # Bidirectional mapping between local sockets and remote sockets. + local_for = {} + remote_for = {} + + register() + + if options.daemonize: + log(u"Daemonizing.") + pid = os.fork() + if pid != 0: + if options.pid_filename: + f = open(options.pid_filename, "w") + print >> f, pid + f.close() + sys.exit(0) + try: + main() + except Exception: + exc = traceback.format_exc() + log("".join(exc))

1 0

[flashproxy/js] Add WebSocketDecoder and tests.
by dcf＠torproject.org 30 Mar '12

30 Mar '12

commit 4726aa424262be3af244a57fe274805da6f26842 Author: David Fifield <david(a)bamsoftware.com> Date: Tue Mar 27 22:59:31 2012 -0700 Add WebSocketDecoder and tests. --- Makefile | 1 + connector-test.py | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++++ connector.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+), 0 deletions(-) diff --git a/Makefile b/Makefile index 0fb0c79..61c3901 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ clean: rm -f $(TARGETS) test: + ./connector-test.py ./flashproxy-test.js .PHONY: all clean test diff --git a/connector-test.py b/connector-test.py new file mode 100755 index 0000000..c0479bd --- /dev/null +++ b/connector-test.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +from connector import WebSocketDecoder + +def read_frames(dec): + frames = [] + while True: + frame = dec.read_frame() + if frame is None: + break + frames.append((frame.fin, frame.opcode, frame.payload)) + return frames + +def read_messages(dec): + messages = [] + while True: + message = dec.read_message() + if message is None: + break + messages.append((message.opcode, message.payload)) + return messages + +class TestWebSocketDecoder(unittest.TestCase): + def test_rfc(self): + """Test samples from RFC 6455 section 5.7.""" + TESTS = [ + ("\x81\x05\x48\x65\x6c\x6c\x6f", False, + [(True, 1, "Hello")], + [(1, u"Hello")]), + ("\x81\x85\x37\xfa\x21\x3d\x7f\x9f\x4d\x51\x58", True, + [(True, 1, "Hello")], + [(1, u"Hello")]), + ("\x01\x03\x48\x65\x6c\x80\x02\x6c\x6f", False, + [(False, 1, "Hel"), (True, 0, "lo")], + [(1, u"Hello")]), + ("\x89\x05\x48\x65\x6c\x6c\x6f", False, + [(True, 9, "Hello")], + [(9, u"Hello")]), + ("\x8a\x85\x37\xfa\x21\x3d\x7f\x9f\x4d\x51\x58", True, + [(True, 10, "Hello")], + [(10, u"Hello")]), + ("\x82\x7e\x01\x00" + "\x00" * 256, False, + [(True, 2, "\x00" * 256)], + [(2, "\x00" * 256)]), + ("\x82\x7f\x00\x00\x00\x00\x00\x01\x00\x00" + "\x00" * 65536, False, + [(True, 2, "\x00" * 65536)], + [(2, "\x00" * 65536)]), + ] + for data, use_mask, expected_frames, expected_messages in TESTS: + dec = WebSocketDecoder(use_mask = use_mask) + dec.feed(data) + actual_frames = read_frames(dec) + self.assertEqual(actual_frames, expected_frames) + + dec = WebSocketDecoder(use_mask = use_mask) + dec.feed(data) + actual_messages = read_messages(dec) + self.assertEqual(actual_messages, expected_messages) + + dec = WebSocketDecoder(use_mask = not use_mask) + dec.feed(data) + self.assertRaises(WebSocketDecoder.MaskingError, dec.read_frame) + + def test_empty_feed(self): + """Test that the decoder can handle a zero-byte feed.""" + dec = WebSocketDecoder() + self.assertIsNone(dec.read_frame()) + dec.feed("") + self.assertIsNone(dec.read_frame()) + dec.feed("\x81\x05H") + self.assertIsNone(dec.read_frame()) + dec.feed("ello") + self.assertEqual(read_frames(dec), [(True, 1, u"Hello")]) + + def test_empty_frame(self): + """Test that a frame may contain a zero-byte payload.""" + dec = WebSocketDecoder() + dec.feed("\x81\x00") + self.assertEqual(read_frames(dec), [(True, 1, u"")]) + dec.feed("\x82\x00") + self.assertEqual(read_frames(dec), [(True, 2, "")]) + + def test_empty_message(self): + """Test that a message may have a zero-byte payload.""" + dec = WebSocketDecoder() + dec.feed("\x01\x00\x00\x00\x80\x00") + self.assertEqual(read_messages(dec), [(1, u"")]) + dec.feed("\x02\x00\x00\x00\x80\x00") + self.assertEqual(read_messages(dec), [(2, "")]) + + def test_interleaved_control(self): + """Test that control messages interleaved with fragmented messages are + returned.""" + dec = WebSocketDecoder() + dec.feed("\x89\x04PING\x01\x03Hel\x8a\x04PONG\x80\x02lo\x89\x04PING") + self.assertEqual(read_messages(dec), [(9, "PING"), (10, "PONG"), (1, u"Hello"), (9, "PING")]) + + def test_fragmented_control(self): + """Test that illegal fragmented control messages cause an error.""" + dec = WebSocketDecoder() + dec.feed("\x09\x04PING") + self.assertRaises(ValueError, dec.read_message) + + def test_zero_opcode(self): + """Test that it is an error for the first frame in a message to have an + opcode of 0.""" + dec = WebSocketDecoder() + dec.feed("\x80\x05Hello") + self.assertRaises(ValueError, dec.read_message) + dec = WebSocketDecoder() + dec.feed("\x00\x05Hello") + self.assertRaises(ValueError, dec.read_message) + + def test_nonzero_opcode(self): + """Test that every frame after the first must have a zero opcode.""" + dec = WebSocketDecoder() + dec.feed("\x01\x01H\x01\x02el\x80\x02lo") + self.assertRaises(ValueError, dec.read_message) + dec = WebSocketDecoder() + dec.feed("\x01\x01H\x00\x02el\x01\x02lo") + self.assertRaises(ValueError, dec.read_message) + + def test_utf8(self): + """Test that text frames (opcode 1) are decoded from UTF-8.""" + text = u"Hello World or Καλημέρα κόσμε or こんにちは世界 or \U0001f639" + utf8_text = text.encode("utf-8") + dec = WebSocketDecoder() + dec.feed("\x81" + chr(len(utf8_text)) + utf8_text) + self.assertEqual(read_messages(dec), [(1, text)]) + + def test_wrong_utf8(self): + """Test that failed UTF-8 decoding causes an error.""" + TESTS = [ + "\xc0\x41", # Non-shortest form. + "\xc2", # Unfinished sequence. + ] + for test in TESTS: + dec = WebSocketDecoder() + dec.feed("\x81" + chr(len(test)) + test) + self.assertRaises(ValueError, dec.read_message) + + def test_overly_large_payload(self): + """Test that large payloads are rejected.""" + dec = WebSocketDecoder() + dec.feed("\x82\x7f\x00\x00\x00\x00\x01\x00\x00\x00") + self.assertRaises(ValueError, dec.read_frame) + +if __name__ == "__main__": + unittest.main() diff --git a/connector.py b/connector.py index f84f8a2..39d89e8 100755 --- a/connector.py +++ b/connector.py @@ -130,6 +130,159 @@ class BufferSocket(object): def is_expired(self, timeout): return time.time() - self.birthday > timeout + +class WebSocketFrame(object): + def __init__(self): + self.fin = False + self.opcode = None + self.payload = None + + def is_control(self): + return (self.opcode & 0x08) != 0 + +class WebSocketMessage(object): + def __init__(self): + self.opcode = None + self.payload = None + + def is_control(self): + return (self.opcode & 0x08) != 0 + +class WebSocketDecoder(object): + """RFC 6455 section 5 is about the WebSocket framing format.""" + # Raise an exception rather than buffer anything arger than this. + MAX_MESSAGE_LENGTH = 1024 * 1024 + + class MaskingError(ValueError): + pass + + def __init__(self, use_mask = False): + """use_mask should be True for server-to-client sockets, and False for + client-to-server sockets.""" + self.use_mask = use_mask + + # Per-frame state. + self.buf = "" + + # Per-message state. + self.message_buf = "" + self.message_opcode = None + + def feed(self, data): + self.buf += data + + @staticmethod + def mask(payload, mask_key): + result = [] + for i, c in enumerate(payload): + mc = chr(ord(payload[i]) ^ ord(mask_key[i%4])) + result.append(mc) + return "".join(result) + + def read_frame(self): + """Read a frame from the internal buffer, if one is available. Returns a + WebSocketFrame object, or None if there are no complete frames to + read.""" + # RFC 6255 section 5.2. + if len(self.buf) < 2: + return None + offset = 0 + b0, b1 = struct.unpack_from(">BB", self.buf, offset) + offset += 2 + fin = (b0 & 0x80) != 0 + opcode = b0 & 0x0f + frame_masked = (b1 & 0x80) != 0 + payload_len = b1 & 0x7f + + if payload_len == 126: + if len(self.buf) < offset + 2: + return None + payload_len, = struct.unpack_from(">H", self.buf, offset) + offset += 2 + elif payload_len == 127: + if len(self.buf) < offset + 8: + return None + payload_len, = struct.unpack_from(">Q", self.buf, offset) + offset += 8 + + if frame_masked: + if not self.use_mask: + # "A client MUST close a connection if it detects a masked + # frame." + raise self.MaskingError("Got masked payload from server") + if len(self.buf) < offset + 4: + return None + mask_key = self.buf[offset:offset+4] + offset += 4 + else: + if self.use_mask: + # "The server MUST close the connection upon receiving a frame + # that is not masked." + raise self.MaskingError("Got unmasked payload from client") + mask_key = "\x00\x00\x00\x00" + + if payload_len > self.MAX_MESSAGE_LENGTH: + raise ValueError("Refusing to buffer payload of %d bytes" % payload_len) + + if len(self.buf) < offset + payload_len: + return None + payload = WebSocketDecoder.mask(self.buf[offset:offset+payload_len], mask_key) + self.buf = self.buf[offset+payload_len:] + + frame = WebSocketFrame() + frame.fin = fin + frame.opcode = opcode + frame.payload = payload + + return frame + + def read_message(self): + """Read a complete message. If the opcode is 1, the payload is decoded + from a UTF-8 binary string to a unicode string. If a control frame is + read while another fragmented message is in progress, the control frame + is returned as a new message immediately. Returns None if there is no + complete frame to be read.""" + # RFC 6455 section 5.4 is about fragmentation. + while True: + frame = self.read_frame() + if frame is None: + return None + # "Control frames (see Section 5.5) MAY be injected in the middle of + # a fragmented message. Control frames themselves MUST NOT be + # fragmented. + if frame.is_control(): + if not frame.fin: + raise ValueError("Control frame (opcode %d) has FIN bit clear" % frame.opcode) + message = WebSocketMessage() + message.opcode = frame.opcode + message.payload = frame.payload + return message + + if self.message_opcode is None: + if frame.opcode == 0: + raise ValueError("First frame has opcode 0") + self.message_opcode = frame.opcode + else: + if frame.opcode != 0: + raise ValueError("Non-first frame has nonzero opcode %d" % frame.opcode) + self.message_buf += frame.payload + + if frame.fin: + break + message = WebSocketMessage() + message.opcode = self.message_opcode + message.payload = self.message_buf + self.postprocess_message(message) + self.message_opcode = None + self.message_buf = "" + + return message + + def postprocess_message(self, message): + if message.opcode == 1: + message.payload = message.payload.decode("utf-8") + return message + def listen_socket(addr): """Return a nonblocking socket listening on the given address.""" addrinfo = socket.getaddrinfo(addr[0], addr[1], 0, socket.SOCK_STREAM, socket.IPPROTO_TCP)[0]

1 0