commit d604bc5a47b646d154d45deeaba6a2cf6f1e8424 Author: Damian Johnson <atagar@torproject.org> Date: Thu Jan 16 17:40:04 2020 -0800 Fix https_server tests Ooph, this was more difficult than it should've been. Isis relied on catch-alls for errors she found difficult, requiring me to repeatedly bypass them to troubleshoot. test_https_server.py finally passes under python 3. Test results changed as follows... before: FAILED (skips=109, failures=19, errors=221, successes=632) after: FAILED (skips=114, failures=14, errors=198, successes=655) --- bridgedb/distributors/email/templates.py | 1 + bridgedb/distributors/https/server.py | 22 +++++++++++----------- bridgedb/test/test_https_server.py | 28 ++++++++++++++++++++++++---- bridgedb/util.py | 17 ++++++++--------- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/bridgedb/distributors/email/templates.py b/bridgedb/distributors/email/templates.py index 85dd105..188f052 100644 --- a/bridgedb/distributors/email/templates.py +++ b/bridgedb/distributors/email/templates.py @@ -59,6 +59,7 @@ def addCommands(template): def addGreeting(template, clientName=None, welcome=False): greeting = "" + clientName = clientName.decode('utf-8') if isinstance(clientName, bytes) else clientName if not clientName: greeting = template.gettext(strings.EMAIL_MISC_TEXT[7]) diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index aea19da..b33065f 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -135,15 +135,15 @@ def replaceErrorPage(request, error, template_name=None, html=True): errorMessage = _("Sorry! Something went wrong with your request.") if not html: - return errorMessage.encode('utf-8') + return errorMessage try: rendered = resource500.render(request) except Exception as err: logging.exception(err) - rendered = errorMessage.encode('utf-8') + rendered = errorMessage - return rendered + return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered def redirectMaliciousRequest(request): @@ -343,7 +343,7 @@ class ErrorResource(CSPResource): except Exception as err: rendered = replaceErrorPage(request, err, html=False) - return rendered + return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered render_POST = render_GET @@ -392,7 +392,7 @@ class TranslatedTemplateResource(CustomErrorHandlingResource, CSPResource): except Exception as err: # pragma: no cover rendered = replaceErrorPage(request, err) request.setHeader("Content-Type", "text/html; charset=utf-8") - return rendered + return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered render_POST = render_GET @@ -509,7 +509,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): langs = translations.getLocaleFromHTTPRequest(request) rtl = translations.usingRTLLang(langs) # TODO: this does not work for versions of IE < 8.0 - imgstr = 'data:image/jpeg;base64,%s' % base64.b64encode(image) + imgstr = b'data:image/jpeg;base64,%s' % base64.b64encode(image.encode('utf-8')) template = lookup.get_template('captcha.html') rendered = template.render(strings, getSortedLangList(), @@ -522,7 +522,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): rendered = replaceErrorPage(request, err, 'captcha.html') request.setHeader("Content-Type", "text/html; charset=utf-8") - return rendered + return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered def render_POST(self, request): """Process a client's CAPTCHA solution. @@ -730,7 +730,7 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource): rendered = redirectTo(request.uri, request) try: - request.write(rendered) + request.write(rendered.encode('utf-8') if isinstance(rendered, str) else rendered) request.finish() except Exception as err: # pragma: no cover logging.exception(err) @@ -919,7 +919,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): logging.exception(err) response = self.renderAnswer(request) - return response + return response.decode('utf-8') if isinstance(response, bytes) else response def getClientIP(self, request): """Get the client's IP address from the ``'X-Forwarded-For:'`` @@ -1048,7 +1048,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): except Exception as err: rendered = replaceErrorPage(request, err) - return rendered + return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered def addWebServer(config, distributor): @@ -1135,7 +1135,7 @@ def addWebServer(config, distributor): if config.HTTPS_ROTATION_PERIOD: count, period = config.HTTPS_ROTATION_PERIOD.split() - sched = ScheduledInterval(count, period) + sched = ScheduledInterval(int(count), period) else: sched = Unscheduled() diff --git a/bridgedb/test/test_https_server.py b/bridgedb/test/test_https_server.py index 3c799ba..a25a99d 100644 --- a/bridgedb/test/test_https_server.py +++ b/bridgedb/test/test_https_server.py @@ -27,7 +27,7 @@ from twisted.trial import unittest from twisted.web.resource import Resource from twisted.web.test import requesthelper -from bridgedb import translations +from bridgedb import _langs, translations from bridgedb.distributors.https import server from bridgedb.schedule import ScheduledInterval @@ -218,6 +218,10 @@ class IndexResourceTests(unittest.TestCase): def test_IndexResource_render_GET_lang_ta(self): """renderGet() with ?lang=ta should return the index page in Tamil.""" + + if 'ta' not in _langs.get_langs(): + self.skipTest("'ta' language unsupported") + request = DummyRequest([self.pagename]) request.method = b'GET' request.addArg('lang', 'ta') @@ -243,6 +247,10 @@ class HowtoResourceTests(unittest.TestCase): def test_HowtoResource_render_GET_lang_ru(self): """renderGet() with ?lang=ru should return the howto page in Russian.""" + + if 'ru' not in _langs.get_langs(): + self.skipTest("'ru' language unsupported") + request = DummyRequest([self.pagename]) request.method = b'GET' request.addArg('lang', 'ru') @@ -522,7 +530,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): def testCB(request): """Check the ``Request`` returned from ``_renderDeferred``.""" self.assertIsInstance(request, DummyRequest) - soup = BeautifulSoup(b''.join(request.written)).find('meta')['http-equiv'] + soup = BeautifulSoup(b''.join(request.written)).find(b'meta')['http-equiv'] self.assertEqual(soup, 'refresh') d = task.deferLater(reactor, 0, lambda x: x, (False, self.request)) @@ -541,7 +549,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): """Check the ``Request`` returned from ``_renderDeferred``.""" self.assertIsInstance(request, DummyRequest) html = b''.join(request.written) - self.assertSubstring('Uh oh, spaghettios!', html) + self.assertSubstring(b'Uh oh, spaghettios!', html) d = task.deferLater(reactor, 0, lambda x: x, (True, self.request)) d.addCallback(self.captchaResource._renderDeferred) @@ -668,7 +676,7 @@ class BridgesResourceTests(unittest.TestCase): # The bridge lines are contained in a <div class='bridges'> tag: soup = BeautifulSoup(page) well = soup.find('div', {'class': 'bridge-lines'}) - content = well.renderContents().strip() + content = well.renderContents().decode('utf-8').strip() lines = content.splitlines() bridges = [] @@ -794,6 +802,10 @@ class BridgesResourceTests(unittest.TestCase): def test_render_GET_RTLlang(self): """Test rendering a request for plain bridges in Arabic.""" + + if 'ar' not in _langs.get_langs(): + self.skipTest("'ar' language unsupported") + self.useBenignBridges() request = DummyRequest([b"bridges?transport=obfs3"]) @@ -816,6 +828,10 @@ class BridgesResourceTests(unittest.TestCase): def test_render_GET_RTLlang_obfs3(self): """Test rendering a request for obfs3 bridges in Farsi.""" + + if 'fa' not in _langs.get_langs(): + self.skipTest("'ar' language unsupported") + self.useBenignBridges() request = DummyRequest([b"bridges?transport=obfs3"]) @@ -915,6 +931,10 @@ class OptionsResourceTests(unittest.TestCase): def test_render_GET_RTLlang(self): """Test rendering a request for obfs3 bridges in Hebrew.""" + + if 'he' not in _langs.get_langs(): + self.skipTest("'ar' language unsupported") + request = DummyRequest(["bridges?transport=obfs3"]) request.method = b'GET' request.getClientIP = lambda: '3.3.3.3' diff --git a/bridgedb/util.py b/bridgedb/util.py index 7b68831..db8e749 100644 --- a/bridgedb/util.py +++ b/bridgedb/util.py @@ -18,6 +18,7 @@ import logging import logging.config import logging.handlers import os +import re import time from twisted.python import components @@ -260,17 +261,15 @@ def replaceControlChars(text, replacement=None, encoding="utf-8"): :rtype: str :returns: The sanitized **text**. """ - escaped = bytearray() - for byte in bytearray(text, encoding): - if byte in list(range(0, 32)) + [92, 127]: - if replacement: - byte = replacement - else: - continue - escaped += bytearray([byte]) + if replacement is None: + replacement = '' + + # the following replaces characters 0-31, 92, and 127 + + text = text.decode(encoding) if isinstance(text, bytes) else text + return re.sub(r'[\x00-\x1f\x5c\x7f]', '', text) - return str(escaped) def registerAdapter(adapter, adapted, interface): """Register a Zope interface adapter for global use.