January 2018 - tor-commits - lists.torproject.org

[bridgedb/develop] Add more unittests for legacy code in Bridges.py.
by isis＠torproject.org 23 Jan '18

23 Jan '18

commit 8aedf0b81874d828c85470a5ef36dbce8324d291 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 22 03:18:06 2017 +0000 Add more unittests for legacy code in Bridges.py. --- bridgedb/test/test_Bridges.py | 95 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/bridgedb/test/test_Bridges.py b/bridgedb/test/test_Bridges.py index 092b033..5991952 100644 --- a/bridgedb/test/test_Bridges.py +++ b/bridgedb/test/test_Bridges.py @@ -14,6 +14,7 @@ from __future__ import print_function import copy +import io import ipaddr import logging @@ -23,9 +24,9 @@ from bridgedb import Bridges from bridgedb.test import util # For additional logger output for debugging, comment out the following: -#logging.disable(50) +logging.disable(50) # and then uncomment the following line: -Bridges.logging.getLogger().setLevel(10) +#Bridges.logging.getLogger().setLevel(10) class BridgeRingTests(unittest.TestCase): @@ -34,6 +35,11 @@ class BridgeRingTests(unittest.TestCase): def setUp(self): self.ring = Bridges.BridgeRing('fake-hmac-key') + def addRandomBridges(self): + bridges = copy.deepcopy(util.generateFakeBridges()) + + [self.ring.insert(bridge) for bridge in bridges] + def addBridgesFromSameSubnet(self): bridges = copy.deepcopy(util.generateFakeBridges()) subnet = "5.5.%d.%d" @@ -62,3 +68,88 @@ class BridgeRingTests(unittest.TestCase): # Since they're all in the same /16, we should only get one # bridge back: self.assertEqual(len(bridges), 1) + + def test_filterDistinctSubnets_random_bridges(self): + """Even after filtering, in a normal case we should get the amount of + bridges we asked for. However, we should always get at least one. + """ + self.addRandomBridges() + + chosen = self.ring.bridges.keys()[:3] + bridges = self.ring.filterDistinctSubnets(chosen) + + self.assertGreaterEqual(len(bridges), 1) + + def test_clear(self): + """Clear should get rid of all the inserted bridges.""" + self.addRandomBridges() + self.assertGreater(len(self.ring), 0) + self.ring.clear() + self.assertEqual(len(self.ring), 0) + + def test_getBridges_filterBySubnet(self): + """We should still get the number of bridges we asked for, even when + filtering by distinct subnets. + """ + self.addRandomBridges() + bridges = self.ring.getBridges('a' * Bridges.DIGEST_LEN, N=3, filterBySubnet=True) + self.assertEqual(len(bridges), 3) + + def test_dumpAssignments(self): + """This should dump the bridges to the file.""" + self.addRandomBridges() + + f = io.StringIO() + + self.ring.dumpAssignments(f) + + f.flush() + f.seek(0) + + data = f.read() + first = self.ring.bridges.values()[0].fingerprint + + # The first bridge's fingerprint should be within the data somewhere + self.assertIn(first, data) + + +class FixedBridgeSplitterTests(unittest.TestCase): + """Unittests for :class:`bridgedb.Bridges.FixedBridgeSplitter`.""" + + def setUp(self): + self.rings = [Bridges.BridgeRing('fake-hmac-key-1'), + Bridges.BridgeRing('fake-hmac-key-2')] + self.splitter = Bridges.FixedBridgeSplitter('fake-hmac-key', self.rings) + + def addRandomBridges(self): + bridges = copy.deepcopy(util.generateFakeBridges()) + + [self.splitter.insert(bridge) for bridge in bridges] + + def test_insert(self): + self.addRandomBridges() + self.assertGreater(len(self.splitter), 0) + + def test_clear(self): + """Clear should get rid of all the inserted bridges.""" + self.addRandomBridges() + self.assertGreater(len(self.splitter), 0) + self.splitter.clear() + self.assertEqual(len(self.splitter), 0) + + def test_dumpAssignments(self): + """This should dump the bridges to the file.""" + self.addRandomBridges() + + f = io.StringIO() + + self.splitter.dumpAssignments(f) + + f.flush() + f.seek(0) + + data = f.read() + first = self.splitter.rings[0].bridges.values()[0].fingerprint + + # The first bridge's fingerprint should be within the data somewhere + self.assertIn(first, data)

1 0

[bridgedb/develop] Create CHANGELOG entry for 0.6.3.
by isis＠torproject.org 23 Jan '18

23 Jan '18

commit bc61afb1ec44f402a86c670d99e749164b5f3f3e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Jan 23 21:40:33 2018 +0000 Create CHANGELOG entry for 0.6.3. --- CHANGELOG | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 82cf769..9ffeac1 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,27 @@ +Changes in version 0.6.3 - 2018-01-23 + + * FIXES #24432 https://bugs.torproject.org/24432 + The production moat server had issues related to redirecting to + resources properly, which are now fixed. + + * FIXES #24701 https://bugs.torproject.org/24701 + Adds a special surprise for the special someone who has been + automatedly requesting bridges not through driving a browser, but + through a script which is so thoroughly stupid that it doesn't + even send the URL parameters for the CAPTCHA challenge and solution. + Their script will now be delayed for quite some time and then + rickrolled. Mess with the best, die like the rest. + + * FIXES #24704 https://bugs.torproject.org/24704 + Bridges returned to a single request are now filtered such that + there will never be two bridges from the same IPv4 /16 or IPv6 /64. + +And includes the following general changes: + + * ADDS unittests for the legacy code in bridgedb/Bridges.py, + bringing the total test coverage above 90% for the first time. + + Changes in version 0.6.2 - 2017-12-20 * FIXES #24636 https://bugs.torproject.org/24636

1 0

[bridgedb/develop] Merge branch 'release/0.6.3'
by isis＠torproject.org 23 Jan '18

23 Jan '18

commit d49e90455c1fd6d66d057b0ba6e6ee4f23129b9f Merge: 2ae288d bc61afb Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Jan 23 21:41:55 2018 +0000 Merge branch 'release/0.6.3' CHANGELOG | 44 ++++++++ bridgedb/Bridges.py | 52 ++++++++-- bridgedb/distributors/email/distributor.py | 2 +- bridgedb/distributors/https/distributor.py | 2 +- bridgedb/distributors/https/server.py | 43 ++++++-- bridgedb/distributors/moat/server.py | 1 + bridgedb/test/test_Bridges.py | 155 +++++++++++++++++++++++++++++ bridgedb/test/test_https_server.py | 11 +- scripts/externalize-pt-client | 12 +++ scripts/test-moat | 49 +++++++-- 10 files changed, 337 insertions(+), 34 deletions(-)

1 0

[bridgedb/master] Merge branch 'fix/24704' into develop
by isis＠torproject.org 23 Jan '18

23 Jan '18

commit 94327eb3eca56785f865483d1ccdf4aa6e03752e Merge: c6c9f47 8aedf0b Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Jan 23 02:02:09 2018 +0000 Merge branch 'fix/24704' into develop bridgedb/test/test_Bridges.py | 95 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-)

1 0

[bridgedb/master] Merge branch 'fix/24432' into develop
by isis＠torproject.org 23 Jan '18

23 Jan '18

commit 3a5dce31dc43f1ea7292e67372b57385ab5a4788 Merge: 94327eb 19c111c Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Jan 23 21:05:21 2018 +0000 Merge branch 'fix/24432' into develop bridgedb/distributors/moat/server.py | 1 + scripts/externalize-pt-client | 12 +++++++++ scripts/test-moat | 49 ++++++++++++++++++++++++++++-------- 3 files changed, 52 insertions(+), 10 deletions(-)

1 0

[bridgedb/master] Add David's script for running an external meek client.
by isis＠torproject.org 23 Jan '18

23 Jan '18

commit 2783933bf9935b6a7ab4ce97386887865a45589a Author: David Fifield <david(a)bamsoftware.com> Date: Tue Nov 28 02:05:58 2017 +0000 Add David's script for running an external meek client. --- scripts/externalize-pt-client | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/externalize-pt-client b/scripts/externalize-pt-client new file mode 100755 index 0000000..bbe01c6 --- /dev/null +++ b/scripts/externalize-pt-client @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +CMDLINE="${@:?need a meek-client command line}" +TRANSPORTS=meek +# This could be controlled by a command-line arg. +PORT=10000 +TOR_PT_MANAGED_TRANSPORT_VER=1 \ + TOR_PT_CLIENT_TRANSPORTS="$TRANSPORTS" $CMDLINE \ + | sed -n -u -e '/^CMETHOD /{s/^.*127\.0\.0\.1://;p}' | while read pt_port; do + echo "forwarding port $PORT -> $pt_port" + socat -v -v TCP-LISTEN:$PORT,fork,reuseaddr TCP-CONNECT:127.0.0.1:$pt_port & +done

1 0

[bridgedb/master] Create CHANGELOG entry for 0.6.3.
by isis＠torproject.org 23 Jan '18

23 Jan '18

commit bc61afb1ec44f402a86c670d99e749164b5f3f3e Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Jan 23 21:40:33 2018 +0000 Create CHANGELOG entry for 0.6.3. --- CHANGELOG | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 82cf769..9ffeac1 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,27 @@ +Changes in version 0.6.3 - 2018-01-23 + + * FIXES #24432 https://bugs.torproject.org/24432 + The production moat server had issues related to redirecting to + resources properly, which are now fixed. + + * FIXES #24701 https://bugs.torproject.org/24701 + Adds a special surprise for the special someone who has been + automatedly requesting bridges not through driving a browser, but + through a script which is so thoroughly stupid that it doesn't + even send the URL parameters for the CAPTCHA challenge and solution. + Their script will now be delayed for quite some time and then + rickrolled. Mess with the best, die like the rest. + + * FIXES #24704 https://bugs.torproject.org/24704 + Bridges returned to a single request are now filtered such that + there will never be two bridges from the same IPv4 /16 or IPv6 /64. + +And includes the following general changes: + + * ADDS unittests for the legacy code in bridgedb/Bridges.py, + bringing the total test coverage above 90% for the first time. + + Changes in version 0.6.2 - 2017-12-20 * FIXES #24636 https://bugs.torproject.org/24636

1 0

[bridgedb/master] Merge branch 'fix/24704' into develop
by isis＠torproject.org 23 Jan '18

23 Jan '18

commit 637e1fc2a4e09ec8ad5a2442db2298d3666897eb Merge: 1096bf4 ad26965 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 22 02:41:46 2017 +0000 Merge branch 'fix/24704' into develop bridgedb/Bridges.py | 52 ++++++++++++++++++++---- bridgedb/distributors/email/distributor.py | 2 +- bridgedb/distributors/https/distributor.py | 2 +- bridgedb/test/test_Bridges.py | 64 ++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 9 deletions(-)

1 0

[bridgedb/master] Add CHANGELOG entries for 0.6.2.
by isis＠torproject.org 23 Jan '18

23 Jan '18

commit 1096bf4d9ed1cbded7ce8ad2a183338f969b1166 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 20 23:22:31 2017 +0000 Add CHANGELOG entries for 0.6.2. --- CHANGELOG | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index effc405..82cf769 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,23 @@ +Changes in version 0.6.2 - 2017-12-20 + + * FIXES #24636 https://bugs.torproject.org/24636 + The moat API specification included an extra response type which + could be sent if there was no overlap between transports the + client supported and those which the server supported. This has + been removed from the specification, which now describes the + behaviour moat has always exhibited: if there is no overlap, the + server responds with a CAPTCHA image response which includes the + list of transports it does support. + + * FIXES #24637 https://bugs.torproject.org/24637 + The moat server did not respond correctly with the specified JSON + API error type when there were no bridges available. It now + responds correctly with a 404 error whose details describe why the + request could not be fulfilled. The moat server also now logs + messages if there were not the configured + MOAT_BRIDGES_PER_RESPONSE number of bridges available. + + Changes in version 0.6.1 - 2017-12-13 * ADDS a shell script, scripts/test-moat, for testing either a

1 0

[bridgedb/master] Delay and then redirect malicious requests.
by isis＠torproject.org 23 Jan '18

23 Jan '18

commit 0f03fb65518353e1409e08def7201ea17fe1318f Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Dec 22 01:45:52 2017 +0000 Delay and then redirect malicious requests. * FIXES #24701: https://bugs.torproject.org/24701 --- bridgedb/distributors/https/server.py | 43 +++++++++++++++++++++++++++-------- bridgedb/test/test_https_server.py | 11 ++++----- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index d4771a6..8c50bc1 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -37,7 +37,9 @@ import mako.exceptions from mako.template import Template from mako.lookup import TemplateLookup +from twisted.internet import defer from twisted.internet import reactor +from twisted.internet import task from twisted.internet.error import CannotListenError from twisted.web import resource from twisted.web import static @@ -138,6 +140,20 @@ def replaceErrorPage(request, error, template_name=None, html=True): return rendered +def redirectMaliciousRequest(request): + '''Redirect the client to a "daring work of art" which "in true + post-modern form, […] tends to raise more questions than answers." + ''' + logging.debug("Redirecting %s to a daring work of art..." % getClientIP(request)) + request.write(redirectTo(base64.b64decode("aHR0cDovLzJnaXJsczFjdXAuY2Ev"), request)) + request.finish() + return request + + +class MaliciousRequest(Exception): + """Raised when we received a possibly malicious request.""" + + class CSPResource(resource.Resource): """A resource which adds a ``'Content-Security-Policy:'`` header. @@ -411,9 +427,9 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): challenge = request.args['captcha_challenge_field'][0] response = request.args['captcha_response_field'][0] except Exception as error: - logging.debug(("Client CAPTCHA solution to HTTPS distributor server" - "didn't include correct HTTP arguments: %s" % error)) - return redirectTo(type(b'')(request.URLPath()), request) + raise MaliciousRequest( + ("Client CAPTCHA solution to HTTPS distributor server " + "didn't include correct HTTP arguments: %s" % error)) return (challenge, response) def checkSolution(self, request): @@ -477,12 +493,21 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): self.setCSPHeader(request) request.setHeader("Content-Type", "text/html; charset=utf-8") - if self.checkSolution(request) is True: - try: - rendered = self.resource.render(request) - except Exception as err: - rendered = replaceErrorPage(request, err) - return rendered + try: + if self.checkSolution(request) is True: + return self.resource.render(request) + except ValueError as err: + logging.debug(err.message) + except MaliciousRequest as err: + logging.debug(err.message) + # Make them wait a bit, then redirect them to a "daring + # work of art" as pennance for their sins. + d = task.deferLater(reactor, 1, lambda: request) + d.addCallback(redirectMaliciousRequest) + return NOT_DONE_YET + except Exception as err: + logging.debug(err.message) + return replaceErrorPage(request, err) logging.debug("Client failed a CAPTCHA; returning redirect to %s" % request.uri) diff --git a/bridgedb/test/test_https_server.py b/bridgedb/test/test_https_server.py index 13ec20e..ba555e8 100644 --- a/bridgedb/test/test_https_server.py +++ b/bridgedb/test/test_https_server.py @@ -373,18 +373,17 @@ class GimpCaptchaProtectedResourceTests(unittest.TestCase): self.assertEqual(response, expectedResponse) def test_extractClientSolution_missing_arguments(self): - """A solution with missing arguments (the solution field) should - return a very agressive redirect to the originally requested, - CAPTCHA-protected page. + """A solution with missing arguments (the solution/challenge fields) + should raise a MaliciousRequest exception. """ expectedChallenge = '23232323232323232323' self.request.method = b'POST' self.request.addArg('captcha_challenge_field', expectedChallenge) - response = self.captchaResource.extractClientSolution(self.request) - - self.assertIn("click here", response) + self.assertRaises(server.MaliciousRequest, + self.captchaResource.extractClientSolution, + self.request) def test_checkSolution(self): """checkSolution() should return False is the solution is invalid."""

1 0