June 2015 - tor-commits - lists.torproject.org

[bridgedb/develop] Minor cleanup for PluggableTransport._runChecks() changes from ad7b5f9a.
by isis＠torproject.org 04 Jun '15

04 Jun '15

commit c4013bc8a5ba0c48d8f752bdffd67d96a01c84d7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Jun 4 02:32:57 2015 +0000 Minor cleanup for PluggableTransport._runChecks() changes from ad7b5f9a. --- lib/bridgedb/bridges.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 4e9ecd1..91b62fa 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -372,19 +372,16 @@ class PluggableTransport(BridgeAddressBase): ("Cannot create PluggableTransport with arguments type: %s") % type(self.arguments)) - for k, v in self.arguments.items(): - kv = ''.join((k, v)) + for item in self.arguments.items(): + kv = ''.join(item) if not isascii_noncontrol(kv): raise MalformedPluggableTransport( ("Cannot create PluggableTransport with non-ASCII or " - "control characters in arguments: %r=%r") - % (k, v)) + "control characters in arguments: %r=%r") % item) if '"' in kv or '\\' in kv: raise MalformedPluggableTransport( ("Cannot create PluggableTransport with double quotes or " - "backslashes in arguments: %r=%r") - % (k, v)) - pass + "backslashes in arguments: %r=%r") % item) if not self._checkArguments(): raise MalformedPluggableTransport(

1 0

[bridgedb/develop] Minor cleanup for htmlify_string() from commits faf48983 and ccb3b8d1.
by isis＠torproject.org 04 Jun '15

04 Jun '15

commit 0b62526335ed532fee5b2b1b3cba4ea04424333a Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Jun 4 02:58:01 2015 +0000 Minor cleanup for htmlify_string() from commits faf48983 and ccb3b8d1. First, we don't need the bridgedb.util.htmlify_string() function, since Mako provides syntax for "piping" content about to be written to the page through filters [0] and it even provides a HTML-escaping function (by default, it uses markupsafe.escape). Second, we do actually need to remove control characters (at the very least, but since technically we're parsing unicode… removing all escape/control characters isn't actually feasible) from bridge lines. * REMOVE bridgedb.util.htmlify_string(). * CHANGE lib/bridgedb/templates/bridges.html to use Mako's builtin text filtering syntax and functions for replacing HTML token characters with their corresponding HTML special characters. * ADD bridgedb.util.replaceControlChars(). * ADD unittests to ensure that everything is escaped properly. [0]: http://docs.makotemplates.org/en/latest/filtering.html --- lib/bridgedb/HTTPServer.py | 12 ++-- lib/bridgedb/templates/bridges.html | 8 ++- lib/bridgedb/test/test_HTTPServer.py | 109 ++++++++++++++++++++++++++++++++-- lib/bridgedb/util.py | 50 +++++++++------- 4 files changed, 145 insertions(+), 34 deletions(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index c969908..beff4bd 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -56,7 +56,7 @@ from bridgedb.qrcodes import generateQR from bridgedb.safelog import logSafely from bridgedb.schedule import Unscheduled from bridgedb.schedule import ScheduledInterval -from bridgedb.util import htmlify_string +from bridgedb.util import replaceControlChars TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), 'templates') @@ -739,12 +739,11 @@ class WebResourceBridges(resource.Resource): self.nBridgesToGive, countryCode, bridgeFilterRules=rules) - bridgeLines = "".join("%s\n" % b.getConfigLine( + bridgeLines = [replaceControlChars(b.getConfigLine( includeFingerprint=self.includeFingerprints, addressClass=addressClass, transport=transport, - request=bridgedb.Dist.uniformMap(ip) - ) for b in bridges) + request=bridgedb.Dist.uniformMap(ip))) for b in bridges] answer = self.renderAnswer(request, bridgeLines, format) return answer @@ -772,7 +771,7 @@ class WebResourceBridges(resource.Resource): if format == 'plain': request.setHeader("Content-Type", "text/plain") try: - rendered = bytes(bridgeLines) + rendered = bytes('\n'.join(bridgeLines)) except Exception as err: rendered = replaceErrorPage(err) else: @@ -790,8 +789,7 @@ class WebResourceBridges(resource.Resource): rtl=rtl, lang=langs[0], answer=bridgeLines, - qrcode=qrcode, - htmlify_string=htmlify_string) + qrcode=qrcode) except Exception as err: rendered = replaceErrorPage(err) diff --git a/lib/bridgedb/templates/bridges.html b/lib/bridgedb/templates/bridges.html index 0ecf1d4..b5ac544 100644 --- a/lib/bridgedb/templates/bridges.html +++ b/lib/bridgedb/templates/bridges.html @@ -1,8 +1,7 @@ ## -*- coding: utf-8 -*- <%inherit file="base.html"/> -<%page args="strings, rtl=False, lang='en', answer=0, qrcode=0, - htmlify_string=None, **kwargs"/> +<%page args="strings, rtl=False, lang='en', answer=0, qrcode=0, **kwargs"/> </div> </div> @@ -67,7 +66,10 @@ <div class="row" id="bridgesrow1"> <div class="col col-lg-12"> <div class="bridge-lines" id="bridgelines"> -${htmlify_string(answer)} +## See http://docs.makotemplates.org/en/latest/filtering.html +% for bridgeline in answer: +${bridgeline | h,trim}<br /> +% endfor </div> </div> </div> diff --git a/lib/bridgedb/test/test_HTTPServer.py b/lib/bridgedb/test/test_HTTPServer.py index 68616bf..d597c55 100644 --- a/lib/bridgedb/test/test_HTTPServer.py +++ b/lib/bridgedb/test/test_HTTPServer.py @@ -422,6 +422,8 @@ class DummyBridge(object): """A mock :class:`bridgedb.Bridges.Bridge` which only supports a mocked ``getConfigLine`` method.""" + ptArgs = {} + def _randORPort(self): return random.randint(9001, 9999) def _randPTPort(self): return random.randint(6001, 6666) def _returnFour(self): return random.randint(2**24, 2**32-1) @@ -457,16 +459,32 @@ class DummyBridge(object): line.append(str(transport)) line.append("%s:%s" % (self.ip, self.orport)) if includeFingerprint is True: line.append(self.fingerprint) + if self.ptArgs: + line.append(','.join(['='.join(x) for x in self.ptArgs.items()])) bridgeLine = " ".join([item for item in line]) #print "Created config line: %r" % bridgeLine return bridgeLine +class DummyMaliciousBridge(DummyBridge): + """A mock :class:`bridgedb.Bridges.Bridge` which only supports a mocked + ``getConfigLine`` method and which maliciously insert an additional fake + bridgeline and some javascript into its PT arguments. + """ + ptArgs = { + "eww": "\rBridge 1.2.3.4:1234", + "bad": "\nBridge 6.6.6.6:6666 0123456789abcdef0123456789abcdef01234567", + "evil": "<script>alert('fuuuu');</script>", + } + + class DummyIPBasedDistributor(object): """A mocked :class:`bridgedb.Dist.IPBasedDistributor` which is used to test :class:`bridgedb.HTTPServer.WebResourceBridges. """ + _bridge_class = DummyBridge + def _dumbAreaMapper(ip): return ip def __init__(self, areaMapper=None, nClusters=None, key=None, @@ -486,7 +504,7 @@ class DummyIPBasedDistributor(object): """Needed because it's called in :meth:`WebResourceBridges.getBridgesForIP`. """ - return [DummyBridge() for _ in xrange(N)] + return [self._bridge_class() for _ in xrange(N)] class DummyRequest(requesthelper.DummyRequest): @@ -520,11 +538,19 @@ class WebResourceBridgesTests(unittest.TestCase): self.dist = DummyIPBasedDistributor() self.sched = ScheduledInterval(1, 'hour') self.nBridgesPerRequest = 2 + + def useBenignBridges(self): + self.dist._bridge_class = DummyBridge self.bridgesResource = HTTPServer.WebResourceBridges( - self.dist, self.sched, N=2, - #useForwardedHeader=True, + self.dist, self.sched, N=self.nBridgesPerRequest, includeFingerprints=True) + self.root.putChild(self.pagename, self.bridgesResource) + def useMaliciousBridges(self): + self.dist._bridge_class = DummyMaliciousBridge + self.bridgesResource = HTTPServer.WebResourceBridges( + self.dist, self.sched, N=self.nBridgesPerRequest, + includeFingerprints=True) self.root.putChild(self.pagename, self.bridgesResource) def parseBridgesFromHTMLPage(self, page): @@ -550,8 +576,76 @@ class WebResourceBridgesTests(unittest.TestCase): return bridges + def test_render_GET_malicious_newlines(self): + """Test rendering a request when the some of the bridges returned have + malicious (HTML, Javascript, etc., in their) PT arguments. + """ + self.useMaliciousBridges() + + request = DummyRequest([self.pagename]) + request.method = b'GET' + request.getClientIP = lambda: '1.1.1.1' + + page = self.bridgesResource.render(request) + self.assertTrue( + 'bad=Bridge 6.6.6.6:6666 0123456789abcdef0123456789abcdef01234567' in str(page), + "Newlines in bridge lines should be removed.") + + def test_render_GET_malicious_returnchar(self): + """Test rendering a request when the some of the bridges returned have + malicious (HTML, Javascript, etc., in their) PT arguments. + """ + self.useMaliciousBridges() + + request = DummyRequest([self.pagename]) + request.method = b'GET' + request.getClientIP = lambda: '1.1.1.1' + + page = self.bridgesResource.render(request) + self.assertTrue( + 'eww=Bridge 1.2.3.4:1234' in str(page), + "Return characters in bridge lines should be removed.") + + def test_render_GET_malicious_javascript(self): + """Test rendering a request when the some of the bridges returned have + malicious (HTML, Javascript, etc., in their) PT arguments. + """ + self.useMaliciousBridges() + + request = DummyRequest([self.pagename]) + request.method = b'GET' + request.getClientIP = lambda: '1.1.1.1' + + page = self.bridgesResource.render(request) + self.assertTrue( + "evil=<script>alert('fuuuu');</script>" in str(page), + ("The characters &, <, >, ', and \" in bridge lines should be " + "replaced with their corresponding HTML special characters.")) + + def test_renderAnswer_GET_textplain_malicious(self): + """If the request format specifies 'plain', we should return content + with mimetype 'text/plain' and ASCII control characters replaced. + """ + self.useMaliciousBridges() + + request = DummyRequest([self.pagename]) + request.args.update({'format': ['plain']}) + request.getClientIP = lambda: '4.4.4.4' + request.method = b'GET' + + page = self.bridgesResource.render(request) + self.assertTrue("html" not in str(page)) + self.assertTrue( + 'eww=Bridge 1.2.3.4:1234' in str(page), + "Return characters in bridge lines should be removed.") + self.assertTrue( + 'bad=Bridge 6.6.6.6:6666' in str(page), + "Newlines in bridge lines should be removed.") + def test_render_GET_vanilla(self): """Test rendering a request for normal, vanilla bridges.""" + self.useBenignBridges() + request = DummyRequest([self.pagename]) request.method = b'GET' request.getClientIP = lambda: '1.1.1.1' @@ -577,6 +671,8 @@ class WebResourceBridgesTests(unittest.TestCase): """The client's IP address should be obtainable from the 'X-Forwarded-For' header in the request. """ + self.useBenignBridges() + self.bridgesResource.useForwardedHeader = True request = DummyRequest([self.pagename]) request.method = b'GET' @@ -594,6 +690,8 @@ class WebResourceBridgesTests(unittest.TestCase): def test_render_GET_RTLlang(self): """Test rendering a request for plain bridges in Arabic.""" + self.useBenignBridges() + request = DummyRequest([b"bridges?transport=obfs3"]) request.method = b'GET' request.getClientIP = lambda: '3.3.3.3' @@ -614,6 +712,8 @@ class WebResourceBridgesTests(unittest.TestCase): def test_render_GET_RTLlang_obfs3(self): """Test rendering a request for obfs3 bridges in Farsi.""" + self.useBenignBridges() + request = DummyRequest([b"bridges?transport=obfs3"]) request.method = b'GET' request.getClientIP = lambda: '3.3.3.3' @@ -643,11 +743,12 @@ class WebResourceBridgesTests(unittest.TestCase): self.assertGreater(int(port), 0) self.assertLessEqual(int(port), 65535) - def test_renderAnswer_textplain(self): """If the request format specifies 'plain', we should return content with mimetype 'text/plain'. """ + self.useBenignBridges() + request = DummyRequest([self.pagename]) request.args.update({'format': ['plain']}) request.getClientIP = lambda: '4.4.4.4' diff --git a/lib/bridgedb/util.py b/lib/bridgedb/util.py index 580f31f..7f906d7 100644 --- a/lib/bridgedb/util.py +++ b/lib/bridgedb/util.py @@ -180,26 +180,6 @@ def levenshteinDistance(s1, s2, len1=None, len2=None, memo[key] = distance return distance -htmlify_string_map = { - '<': '<', - '>': '>', - '&': '&', - '"': '"', - "'": ''', - '\n': '<br/>' - } -def htmlify_string(s): - """Encode HTML special characters, and newlines, in s. - - >>> htmlify_string('<script>alert("badthink");</script>') - '<script>alert("badthink");</script>' - >>> htmlify_string('bridge 1\nbridge 2') - 'bridge 1<br/>bridge 2' - - :param str s: The string to encode. - """ - return ''.join(map((lambda ch: htmlify_string_map.get(ch, ch)), s)) - def isascii(s): """Return True if there are no non-ASCII characters in s, False otherwise. @@ -235,6 +215,36 @@ def isascii_noncontrol(s): """ return all(map((lambda ch: 32 <= ord(ch) < 127), s)) +def replaceControlChars(text, replacement=None, encoding="utf-8"): + """Remove ASCII control characters [0-31, 92, 127]. + + >>> replaceControlChars('foo\n bar\\ baz\r \t\0quux\n') + 'foo bar baz quux' + >>> replaceControlChars("\bI wonder if I'm outside the quotes now") + "I wonder if I'm outside the quotes now" + + :param str text: Some text to remove ASCII control characters from. + :param int replacement: If given, the **replacement** should be an integer + representing the decimal representation of the byte to replace + occurences of ASCII control characters with. For example, if they + should be replaced with the character ``'a'``, then ``97`` should be + used as the **replacement**, because ``ord('a') == 97``. + :param str encoding: The encoding of the **text**. + :rtype: str + :returns: The sanitized **text**. + """ + escaped = bytearray() + + for byte in bytearray(text, encoding): + if byte in range(0, 32) + [92, 127]: + if replacement: + byte = replacement + else: + continue + escaped += bytearray([byte]) + + return str(escaped) + class JustifiedLogFormatter(logging.Formatter): """A logging formatter which pretty prints thread and calling function

1 0

[bridgedb/develop] Revert "Remove clear abuses of str.strip() and str.rstrip()"
by isis＠torproject.org 04 Jun '15

04 Jun '15

commit 9a4cf49b975fd219bac868884bd573f705bf629e Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed May 6 20:22:14 2015 +0000 Revert "Remove clear abuses of str.strip() and str.rstrip()" This reverts commit 4e773f346f2b442a1726af94b7b2da231b73c1d6. --- lib/bridgedb/Bridges.py | 3 +-- lib/bridgedb/bridges.py | 6 ++---- lib/bridgedb/test/test_Main.py | 3 +-- lib/bridgedb/test/test_Tests.py | 3 +-- lib/bridgedb/test/test_bridgedb.py | 3 +-- lib/bridgedb/test/test_captcha.py | 7 +++---- lib/bridgedb/test/test_configure.py | 3 +-- lib/bridgedb/test/test_crypto.py | 3 +-- lib/bridgedb/test/test_email_autoresponder.py | 3 +-- lib/bridgedb/test/test_https.py | 3 +-- lib/bridgedb/test/test_smtp.py | 3 +-- lib/bridgedb/util.py | 9 --------- 12 files changed, 14 insertions(+), 35 deletions(-) diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py index eb39d47..58a81cd 100644 --- a/lib/bridgedb/Bridges.py +++ b/lib/bridgedb/Bridges.py @@ -29,7 +29,6 @@ from bridgedb.parse.fingerprint import toHex from bridgedb.parse.fingerprint import fromHex from bridgedb.parse.fingerprint import isValidFingerprint from bridgedb.safelog import logSafely -from bridgedb.util import remove_prefix try: from cStringIO import StringIO @@ -651,7 +650,7 @@ class FilteredBridgeSplitter(BridgeHolder): subringName = [self.distributorName] for filterName in filterNames: if filterName != 'filterAssignBridgesToRing': - subringName.append(remove_prefix(filterName, 'filterBridgesBy')) + subringName.append(filterName.strip('filterBridgesBy')) subringName = '-'.join([x for x in subringName]) subring.setName(subringName) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 6ea7d3f..2922fa1 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -38,8 +38,6 @@ from bridgedb.parse.fingerprint import isValidFingerprint from bridgedb.parse.fingerprint import toHex from bridgedb.parse.fingerprint import fromHex from bridgedb.parse.nickname import isValidRouterNickname -from bridgedb.util import remove_prefix -from bridgedb.util import remove_suffix class PluggableTransportUnavailable(Exception): @@ -1499,8 +1497,8 @@ class Bridge(BridgeBackwardsCompatibility): sequence = asn1.DerSequence() key = self.signingKey - key = remove_prefix(key, TOR_SIGNING_KEY_HEADER) - key = remove_suffix(key, TOR_SIGNING_KEY_FOOTER) + key = key.strip(TOR_SIGNING_KEY_HEADER) + key = key.strip(TOR_SIGNING_KEY_FOOTER) key = key.replace('\n', '') key = base64.b64decode(key) diff --git a/lib/bridgedb/test/test_Main.py b/lib/bridgedb/test/test_Main.py index 6c603fb..5ae4025 100644 --- a/lib/bridgedb/test/test_Main.py +++ b/lib/bridgedb/test/test_Main.py @@ -29,14 +29,13 @@ from twisted.trial import unittest from bridgedb import Main from bridgedb.Bridges import BridgeHolder from bridgedb.parse.options import parseOptions -from bridgedb.util import remove_suffix logging.getLogger().disabled = True HERE = os.getcwd() -TOPDIR = remove_suffix(HERE, '_trial_temp') +TOPDIR = HERE.rstrip('_trial_temp') CI_RUNDIR = os.path.join(TOPDIR, 'run') # A networkstatus descriptor with two invalid ORAddress (127.0.0.1 and ::1) diff --git a/lib/bridgedb/test/test_Tests.py b/lib/bridgedb/test/test_Tests.py index 01fc1ca..84c3f8b 100644 --- a/lib/bridgedb/test/test_Tests.py +++ b/lib/bridgedb/test/test_Tests.py @@ -29,7 +29,6 @@ from twisted.trial import unittest from bridgedb.test import legacy_Tests as Tests from bridgedb.test import deprecated -from bridgedb.util import remove_suffix warnings.filterwarnings('ignore', module="bridgedb\.test\.legacy_Tests") @@ -181,7 +180,7 @@ class DynamicTestCaseMeta(type): origFunc = origName.split('.')[-2:] origName = '_'.join(origFunc) if origName.endswith('_py'): # this happens with doctests - origName = remove_suffix(origName, '_py') + origName = origName.strip('_py') methName = str(methodPrefix + origName).replace('.', '_') meth = testMethodFactory(test, methName) logging.debug("Set %s.%s=%r" % (cls.__name__, methName, meth)) diff --git a/lib/bridgedb/test/test_bridgedb.py b/lib/bridgedb/test/test_bridgedb.py index 301309a..70c71f0 100644 --- a/lib/bridgedb/test/test_bridgedb.py +++ b/lib/bridgedb/test/test_bridgedb.py @@ -23,7 +23,6 @@ from twisted.trial.unittest import SkipTest from bridgedb.test.util import processExists from bridgedb.test.util import getBridgeDBPID -from bridgedb.util import remove_suffix class BridgeDBCliTest(unittest.TestCase): @@ -31,7 +30,7 @@ class BridgeDBCliTest(unittest.TestCase): def setUp(self): here = os.getcwd() - topdir = remove_suffix(here, '_trial_temp') + topdir = here.rstrip('_trial_temp') self.rundir = os.path.join(topdir, 'run') self.pidfile = os.path.join(self.rundir, 'bridgedb.pid') self.pid = getBridgeDBPID(self.pidfile) diff --git a/lib/bridgedb/test/test_captcha.py b/lib/bridgedb/test/test_captcha.py index 4db476d..1000477 100644 --- a/lib/bridgedb/test/test_captcha.py +++ b/lib/bridgedb/test/test_captcha.py @@ -23,7 +23,6 @@ from zope.interface import providedBy from bridgedb import captcha from bridgedb import crypto -from bridgedb.util import remove_suffix class CaptchaTests(unittest.TestCase): @@ -98,7 +97,7 @@ class GimpCaptchaTests(unittest.TestCase): def setUp(self): here = os.getcwd() - self.topDir = remove_suffix(here, '_trial_temp') + self.topDir = here.rstrip('_trial_temp') self.cacheDir = os.path.join(self.topDir, 'captchas') self.badCacheDir = os.path.join(here, 'capt') @@ -132,7 +131,7 @@ class GimpCaptchaTests(unittest.TestCase): """GimpCaptcha with bad cacheDir should raise GimpCaptchaError.""" self.assertRaises(captcha.GimpCaptchaError, captcha.GimpCaptcha, self.publik, self.sekrit, self.hmacKey, - remove_suffix(self.cacheDir, 'chas')) + self.cacheDir.rstrip('chas')) def test_init(self): """Test that __init__ correctly initialised all the values.""" @@ -239,7 +238,7 @@ class GimpCaptchaTests(unittest.TestCase): c = captcha.GimpCaptcha(self.publik, self.sekrit, self.hmacKey, self.cacheDir) image, challenge = c.get() - challengeBadB64 = remove_suffix(challenge, '==') + "\x42\x42\x42" + challengeBadB64 = challenge.rstrip('==') + "\x42\x42\x42" self.assertEquals( c.check(challenge, c.answer, c.secretKey, c.hmacKey), True) diff --git a/lib/bridgedb/test/test_configure.py b/lib/bridgedb/test/test_configure.py index c70e7de..ebac95a 100644 --- a/lib/bridgedb/test/test_configure.py +++ b/lib/bridgedb/test/test_configure.py @@ -16,7 +16,6 @@ import os from twisted.trial import unittest from bridgedb import configure -from bridgedb.util import remove_suffix class ConfigureTests(unittest.TestCase): @@ -25,7 +24,7 @@ class ConfigureTests(unittest.TestCase): def setUp(self): """Find the config file in the top directory of this repo.""" here = os.getcwd() - topdir = remove_suffix(here, '_trial_temp') + topdir = here.rstrip('_trial_temp') self.configFilename = os.path.join(topdir, 'bridgedb.conf') def test_loadConfig_with_file(self): diff --git a/lib/bridgedb/test/test_crypto.py b/lib/bridgedb/test/test_crypto.py index a10dde9..3264ace 100644 --- a/lib/bridgedb/test/test_crypto.py +++ b/lib/bridgedb/test/test_crypto.py @@ -35,7 +35,6 @@ from bridgedb import txrecaptcha from bridgedb.persistent import Conf from bridgedb.test.util import fileCheckDecorator from bridgedb.test.email_helpers import _createConfig -from bridgedb.util import remove_suffix logging.disable(50) @@ -101,7 +100,7 @@ class InitializeGnuPGTests(unittest.TestCase): :returns: The full path to the new gnupg home directory. """ here = os.getcwd() - topDir = remove_suffix(here, '_trial_temp') + topDir = here.rstrip('_trial_temp') gnupghome = os.path.join(topDir, '.gnupg') gnupghomeNew = os.path.join(here, '.gnupg') diff --git a/lib/bridgedb/test/test_email_autoresponder.py b/lib/bridgedb/test/test_email_autoresponder.py index 4c36be4..2895802 100644 --- a/lib/bridgedb/test/test_email_autoresponder.py +++ b/lib/bridgedb/test/test_email_autoresponder.py @@ -29,7 +29,6 @@ from bridgedb.Dist import TooSoonEmail from bridgedb.test.email_helpers import _createConfig from bridgedb.test.email_helpers import _createMailServerContext from bridgedb.test.email_helpers import DummyEmailDistributorWithState -from bridgedb.util import remove_suffix class CreateResponseBodyTests(unittest.TestCase): @@ -37,7 +36,7 @@ class CreateResponseBodyTests(unittest.TestCase): def _moveGPGTestKeyfile(self): here = os.getcwd() - topDir = remove_suffix(here, '_trial_temp') + topDir = here.rstrip('_trial_temp') self.gpgFile = os.path.join(topDir, '.gnupg', 'TESTING.subkeys.sec') self.gpgMoved = os.path.join(here, 'TESTING.subkeys.sec') shutil.copy(self.gpgFile, self.gpgMoved) diff --git a/lib/bridgedb/test/test_https.py b/lib/bridgedb/test/test_https.py index fb88abe..1e0c778 100644 --- a/lib/bridgedb/test/test_https.py +++ b/lib/bridgedb/test/test_https.py @@ -37,7 +37,6 @@ from twisted.trial.unittest import SkipTest from bridgedb.test.util import processExists from bridgedb.test.util import getBridgeDBPID -from bridgedb.util import remove_suffix HTTP_ROOT = 'http://127.0.0.1:6788' CAPTCHA_RESPONSE = 'Tvx74Pmy' @@ -46,7 +45,7 @@ CAPTCHA_RESPONSE = 'Tvx74Pmy' class HTTPTests(unittest.TestCase): def setUp(self): here = os.getcwd() - topdir = remove_suffix(here, '_trial_temp') + topdir = here.rstrip('_trial_temp') self.rundir = os.path.join(topdir, 'run') self.pidfile = os.path.join(self.rundir, 'bridgedb.pid') self.pid = getBridgeDBPID(self.pidfile) diff --git a/lib/bridgedb/test/test_smtp.py b/lib/bridgedb/test/test_smtp.py index b4cf8f9..1cbafba 100644 --- a/lib/bridgedb/test/test_smtp.py +++ b/lib/bridgedb/test/test_smtp.py @@ -17,7 +17,6 @@ from twisted.trial.unittest import SkipTest from bridgedb.test.util import processExists from bridgedb.test.util import getBridgeDBPID -from bridgedb.util import remove_suffix # ------------- SMTP Client Config SMTP_DEBUG_LEVEL = 0 # set to 1 to see SMTP message exchange @@ -119,7 +118,7 @@ class SMTPTests(unittest.TestCase): running. ''' here = os.getcwd() - topdir = remove_suffix(here, '_trial_temp') + topdir = here.rstrip('_trial_temp') self.rundir = os.path.join(topdir, 'run') self.pidfile = os.path.join(self.rundir, 'bridgedb.pid') self.pid = getBridgeDBPID(self.pidfile) diff --git a/lib/bridgedb/util.py b/lib/bridgedb/util.py index e6ecbd1..f3d87bd 100644 --- a/lib/bridgedb/util.py +++ b/lib/bridgedb/util.py @@ -302,12 +302,3 @@ class mixin: expect mixin classes to end in ``'mixin'``. """ __metaclass__ = abc.ABCMeta - -def remove_prefix(s, f): - assert s.startswith(f) - return s[len(f):] - -def remove_suffix(s, f): - assert s.endswith(f) - return s[:-len(f)] -

1 0

[bridgedb/develop] Revert "Also forbid bogus characters in PT fingerprint and address"
by isis＠torproject.org 04 Jun '15

04 Jun '15

commit 2254b859abf24f437376b94c6c4907f91a04d957 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed May 6 20:41:31 2015 +0000 Revert "Also forbid bogus characters in PT fingerprint and address" This reverts commit 44e31908090a0e70dfdd82020a5b89c80e04a6d2. In the @property setters for b.bridges.BridgeAddressBase.fingerprint and b.bridges.BridgeAddressBase.address, the functions b.p.fingerprints.isValidFingerprint() and b.p.addr.isValidIP() respectively determine whether the attribute will be set. My tests show that all of {non-ASCII characters, control characters, double quotes, backslashes} will not make it through either of those validation functions. --- lib/bridgedb/bridges.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 01bbdd0..4e9ecd1 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -351,10 +351,6 @@ class PluggableTransport(BridgeAddressBase): characters or double quotes or backslashes, in keys or in values. - 3. The :data:`fingerprint` and :data:`address` do not - contain non-ASCII or control characters or double quotes - or backslashes. - :raises MalformedPluggableTransport: if any of the above checks fails. """ if not self.fingerprint: @@ -390,28 +386,6 @@ class PluggableTransport(BridgeAddressBase): % (k, v)) pass - if not isascii_noncontrol(self.fingerprint): - raise MalformedPluggableTransport( - ("Cannot create PluggableTransport with non-ASCII or " - "control characters in fingerprint: %r") - % self.fingerprint) - if '"' in self.fingerprint or '\\' in self.fingerprint: - raise MalformedPluggableTransport( - ("Cannot create PluggableTransport with double quotes or " - "backslashes in fingerprint: %r") - % self.fingerprint) - - if not isascii_noncontrol(self.address): - raise MalformedPluggableTransport( - ("Cannot create PluggableTransport with non-ASCII or " - "control characters in address: %r") - % self.address) - if '"' in self.address or '\\' in self.address: - raise MalformedPluggableTransport( - ("Cannot create PluggableTransport with double quotes or " - "backslashes in address: %r") - % self.address) - if not self._checkArguments(): raise MalformedPluggableTransport( ("Can't use %s transport with missing arguments. Arguments: "

1 0

[bridgedb/develop] Merge remote-tracking branch 'rr/2015-04-30-03-security-fixes' into develop
by isis＠torproject.org 04 Jun '15

04 Jun '15

commit 4fb9962bf80c5693937e83079b3083220431b11d Merge: 9a4cf49 44e3190 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed May 6 20:25:45 2015 +0000 Merge remote-tracking branch 'rr/2015-04-30-03-security-fixes' into develop lib/bridgedb/HTTPServer.py | 4 ++- lib/bridgedb/bridges.py | 45 ++++++++++++++++++++++++++++ lib/bridgedb/templates/bridges.html | 5 ++-- lib/bridgedb/util.py | 55 +++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 3 deletions(-)

1 0

[bridgedb/develop] Also forbid bogus characters in PT fingerprint and address
by isis＠torproject.org 04 Jun '15

04 Jun '15

commit 44e31908090a0e70dfdd82020a5b89c80e04a6d2 Author: Robert Ransom <rransom.8774(a)gmail.com> Date: Thu Apr 30 10:37:47 2015 -0700 Also forbid bogus characters in PT fingerprint and address --- lib/bridgedb/bridges.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index b799ca3..61cf503 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -356,6 +356,10 @@ class PluggableTransport(BridgeAddressBase): characters or double quotes or backslashes, in keys or in values. + 3. The :data:`fingerprint` and :data:`address` do not + contain non-ASCII or control characters or double quotes + or backslashes. + :raises MalformedPluggableTransport: if any of the above checks fails. """ if not self.fingerprint: @@ -391,6 +395,28 @@ class PluggableTransport(BridgeAddressBase): % (k, v)) pass + if not isascii_noncontrol(self.fingerprint): + raise MalformedPluggableTransport( + ("Cannot create PluggableTransport with non-ASCII or " + "control characters in fingerprint: %r") + % self.fingerprint) + if '"' in self.fingerprint or '\\' in self.fingerprint: + raise MalformedPluggableTransport( + ("Cannot create PluggableTransport with double quotes or " + "backslashes in fingerprint: %r") + % self.fingerprint) + + if not isascii_noncontrol(self.address): + raise MalformedPluggableTransport( + ("Cannot create PluggableTransport with non-ASCII or " + "control characters in address: %r") + % self.address) + if '"' in self.address or '\\' in self.address: + raise MalformedPluggableTransport( + ("Cannot create PluggableTransport with double quotes or " + "backslashes in address: %r") + % self.address) + if not self._checkArguments(): raise MalformedPluggableTransport( ("Can't use %s transport with missing arguments. Arguments: "

1 0

[bridgedb/develop] HTML-encode bridge lines properly
by isis＠torproject.org 04 Jun '15

04 Jun '15

commit ccb3b8d17ddf228c903a3abaabf80ba179dadb3a Author: Robert Ransom <rransom.8774(a)gmail.com> Date: Thu Apr 30 05:21:50 2015 -0700 HTML-encode bridge lines properly --- lib/bridgedb/HTTPServer.py | 4 +++- lib/bridgedb/templates/bridges.html | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 2e0398c..47ea298 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -56,6 +56,7 @@ from bridgedb.qrcodes import generateQR from bridgedb.safelog import logSafely from bridgedb.schedule import Unscheduled from bridgedb.schedule import ScheduledInterval +from bridgedb.util import htmlify_string TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), 'templates') @@ -786,7 +787,8 @@ class WebResourceBridges(resource.Resource): rtl=rtl, lang=langs[0], answer=bridgeLines, - qrcode=qrcode) + qrcode=qrcode, + htmlify_string=htmlify_string) except Exception as err: rendered = replaceErrorPage(err) diff --git a/lib/bridgedb/templates/bridges.html b/lib/bridgedb/templates/bridges.html index 8048919..0ecf1d4 100644 --- a/lib/bridgedb/templates/bridges.html +++ b/lib/bridgedb/templates/bridges.html @@ -1,7 +1,8 @@ ## -*- coding: utf-8 -*- <%inherit file="base.html"/> -<%page args="strings, rtl=False, lang='en', answer=0, qrcode=0, **kwargs"/> +<%page args="strings, rtl=False, lang='en', answer=0, qrcode=0, + htmlify_string=None, **kwargs"/> </div> </div> @@ -66,7 +67,7 @@ <div class="row" id="bridgesrow1"> <div class="col col-lg-12"> <div class="bridge-lines" id="bridgelines"> -${answer.replace("\n", "<br />")} +${htmlify_string(answer)} </div> </div> </div>

1 0

[bridgedb/develop] Documentation; import bridgedb.parse modules into Sphinx conf.py.
by isis＠torproject.org 04 Jun '15

04 Jun '15

commit e6e7ea12af85f94813d95603043c02294da8eaf0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Jun 3 01:00:11 2015 +0000 Documentation; import bridgedb.parse modules into Sphinx conf.py. --- doc/sphinx/source/conf.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index 2f01b4c..5cd485f 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -44,6 +44,14 @@ import bridgedb.Filters import bridgedb.geo import bridgedb.HTTPServer import bridgedb.Main +import bridgedb.parse +import bridgedb.parse.addr +import bridgedb.parse.descriptors +import bridgedb.parse.fingerprint +import bridgedb.parse.headers +import bridgedb.parse.nickname +import bridgedb.parse.options +import bridgedb.parse.versions import bridgedb.persistent import bridgedb.proxy import bridgedb.qrcodes

1 0

[bridgedb/develop] Forbid non-ASCII and control chars, double quotes and backslashes in PT args
by isis＠torproject.org 04 Jun '15

04 Jun '15

commit ad7b5f9ab3354aef3d69db33933ab1fcebf16bd1 Author: Robert Ransom <rransom.8774(a)gmail.com> Date: Thu Apr 30 08:41:08 2015 -0700 Forbid non-ASCII and control chars, double quotes and backslashes in PT args --- lib/bridgedb/bridges.py | 19 +++++++++++++++++++ lib/bridgedb/util.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index e2b2ff5..b799ca3 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -38,6 +38,7 @@ from bridgedb.parse.fingerprint import isValidFingerprint from bridgedb.parse.fingerprint import toHex from bridgedb.parse.fingerprint import fromHex from bridgedb.parse.nickname import isValidRouterNickname +from bridgedb.util import isascii_noncontrol class PluggableTransportUnavailable(Exception): @@ -351,6 +352,10 @@ class PluggableTransport(BridgeAddressBase): 2. The :data:`arguments` is a dictionary. + 3. The :data:`arguments` do not contain non-ASCII or control + characters or double quotes or backslashes, in keys or + in values. + :raises MalformedPluggableTransport: if any of the above checks fails. """ if not self.fingerprint: @@ -372,6 +377,20 @@ class PluggableTransport(BridgeAddressBase): ("Cannot create PluggableTransport with arguments type: %s") % type(self.arguments)) + for k, v in self.arguments.items(): + kv = ''.join((k, v)) + if not isascii_noncontrol(kv): + raise MalformedPluggableTransport( + ("Cannot create PluggableTransport with non-ASCII or " + "control characters in arguments: %r=%r") + % (k, v)) + if '"' in kv or '\\' in kv: + raise MalformedPluggableTransport( + ("Cannot create PluggableTransport with double quotes or " + "backslashes in arguments: %r=%r") + % (k, v)) + pass + if not self._checkArguments(): raise MalformedPluggableTransport( ("Can't use %s transport with missing arguments. Arguments: " diff --git a/lib/bridgedb/util.py b/lib/bridgedb/util.py index 8034120..580f31f 100644 --- a/lib/bridgedb/util.py +++ b/lib/bridgedb/util.py @@ -200,6 +200,41 @@ def htmlify_string(s): """ return ''.join(map((lambda ch: htmlify_string_map.get(ch, ch)), s)) +def isascii(s): + """Return True if there are no non-ASCII characters in s, False otherwise. + + Note that this function differs from the str.is* methods in that + it returns True for the empty string, rather than False. + + >>> isascii('\x80') + False + >>> isascii('foo\tbar\rbaz\n') + True + >>> isascii('foo bar') + True + + :param str s: The string to check for non-ASCII characters. + """ + return all(map((lambda ch: ord(ch) < 128), s)) + +def isascii_noncontrol(s): + """Return True if there are no non-ASCII or control characters in + s, False otherwise. + + Note that this function differs from the str.is* methods in that + it returns True for the empty string, rather than False. + + >>> isascii_noncontrol('\x80') + False + >>> isascii_noncontrol('foo\tbar\rbaz\n') + False + >>> isascii_noncontrol('foo bar') + True + + :param str s: The string to check for non-ASCII or control characters. + """ + return all(map((lambda ch: 32 <= ord(ch) < 127), s)) + class JustifiedLogFormatter(logging.Formatter): """A logging formatter which pretty prints thread and calling function

1 0

[bridgedb/develop] Fix comment typo
by isis＠torproject.org 04 Jun '15

04 Jun '15

commit 14ebe12b221700644a0e492207bfa184305fdc3c Author: Robert Ransom <rransom.8774(a)gmail.com> Date: Sat Mar 28 16:56:37 2015 -0700 Fix comment typo --- lib/bridgedb/bridges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/bridges.py b/lib/bridgedb/bridges.py index 93b9fe9..e2b2ff5 100644 --- a/lib/bridgedb/bridges.py +++ b/lib/bridgedb/bridges.py @@ -321,7 +321,7 @@ class PluggableTransport(BridgeAddressBase): """ argDict = {} - # PT argumentss are comma-separated in the extrainfo + # PT arguments are comma-separated in the extrainfo # descriptors. While there *shouldn't* be anything after them that was # separated by a space (and hence would wind up being in a different # item in `arguments`), if there was we'll join it to the rest of the

1 0