commit d604bc5a47b646d154d45deeaba6a2cf6f1e8424
Author: Damian Johnson <atagar(a)torproject.org>
Date: Thu Jan 16 17:40:04 2020 -0800
Fix https_server tests
Ooph, this was more difficult than it should've been. Isis relied on catch-alls
for errors she found difficult, requiring me to repeatedly bypass them to
troubleshoot.
test_https_server.py finally passes under python 3.
Test results changed as follows...
before: FAILED (skips=109, failures=19, errors=221, successes=632)
after: FAILED (skips=114, failures=14, errors=198, successes=655)
---
bridgedb/distributors/email/templates.py | 1 +
bridgedb/distributors/https/server.py | 22 +++++++++++-----------
bridgedb/test/test_https_server.py | 28 ++++++++++++++++++++++++----
bridgedb/util.py | 17 ++++++++---------
4 files changed, 44 insertions(+), 24 deletions(-)
diff --git a/bridgedb/distributors/email/templates.py b/bridgedb/distributors/email/templates.py
index 85dd105..188f052 100644
--- a/bridgedb/distributors/email/templates.py
+++ b/bridgedb/distributors/email/templates.py
@@ -59,6 +59,7 @@ def addCommands(template):
def addGreeting(template, clientName=None, welcome=False):
greeting = ""
+ clientName = clientName.decode('utf-8') if isinstance(clientName, bytes) else clientName
if not clientName:
greeting = template.gettext(strings.EMAIL_MISC_TEXT[7])
diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py
index aea19da..b33065f 100644
--- a/bridgedb/distributors/https/server.py
+++ b/bridgedb/distributors/https/server.py
@@ -135,15 +135,15 @@ def replaceErrorPage(request, error, template_name=None, html=True):
errorMessage = _("Sorry! Something went wrong with your request.")
if not html:
- return errorMessage.encode('utf-8')
+ return errorMessage
try:
rendered = resource500.render(request)
except Exception as err:
logging.exception(err)
- rendered = errorMessage.encode('utf-8')
+ rendered = errorMessage
- return rendered
+ return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered
def redirectMaliciousRequest(request):
@@ -343,7 +343,7 @@ class ErrorResource(CSPResource):
except Exception as err:
rendered = replaceErrorPage(request, err, html=False)
- return rendered
+ return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered
render_POST = render_GET
@@ -392,7 +392,7 @@ class TranslatedTemplateResource(CustomErrorHandlingResource, CSPResource):
except Exception as err: # pragma: no cover
rendered = replaceErrorPage(request, err)
request.setHeader("Content-Type", "text/html; charset=utf-8")
- return rendered
+ return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered
render_POST = render_GET
@@ -509,7 +509,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource):
langs = translations.getLocaleFromHTTPRequest(request)
rtl = translations.usingRTLLang(langs)
# TODO: this does not work for versions of IE < 8.0
- imgstr = 'data:image/jpeg;base64,%s' % base64.b64encode(image)
+ imgstr = b'data:image/jpeg;base64,%s' % base64.b64encode(image.encode('utf-8'))
template = lookup.get_template('captcha.html')
rendered = template.render(strings,
getSortedLangList(),
@@ -522,7 +522,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource):
rendered = replaceErrorPage(request, err, 'captcha.html')
request.setHeader("Content-Type", "text/html; charset=utf-8")
- return rendered
+ return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered
def render_POST(self, request):
"""Process a client's CAPTCHA solution.
@@ -730,7 +730,7 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource):
rendered = redirectTo(request.uri, request)
try:
- request.write(rendered)
+ request.write(rendered.encode('utf-8') if isinstance(rendered, str) else rendered)
request.finish()
except Exception as err: # pragma: no cover
logging.exception(err)
@@ -919,7 +919,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource):
logging.exception(err)
response = self.renderAnswer(request)
- return response
+ return response.decode('utf-8') if isinstance(response, bytes) else response
def getClientIP(self, request):
"""Get the client's IP address from the ``'X-Forwarded-For:'``
@@ -1048,7 +1048,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource):
except Exception as err:
rendered = replaceErrorPage(request, err)
- return rendered
+ return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered
def addWebServer(config, distributor):
@@ -1135,7 +1135,7 @@ def addWebServer(config, distributor):
if config.HTTPS_ROTATION_PERIOD:
count, period = config.HTTPS_ROTATION_PERIOD.split()
- sched = ScheduledInterval(count, period)
+ sched = ScheduledInterval(int(count), period)
else:
sched = Unscheduled()
diff --git a/bridgedb/test/test_https_server.py b/bridgedb/test/test_https_server.py
index 3c799ba..a25a99d 100644
--- a/bridgedb/test/test_https_server.py
+++ b/bridgedb/test/test_https_server.py
@@ -27,7 +27,7 @@ from twisted.trial import unittest
from twisted.web.resource import Resource
from twisted.web.test import requesthelper
-from bridgedb import translations
+from bridgedb import _langs, translations
from bridgedb.distributors.https import server
from bridgedb.schedule import ScheduledInterval
@@ -218,6 +218,10 @@ class IndexResourceTests(unittest.TestCase):
def test_IndexResource_render_GET_lang_ta(self):
"""renderGet() with ?lang=ta should return the index page in Tamil."""
+
+ if 'ta' not in _langs.get_langs():
+ self.skipTest("'ta' language unsupported")
+
request = DummyRequest([self.pagename])
request.method = b'GET'
request.addArg('lang', 'ta')
@@ -243,6 +247,10 @@ class HowtoResourceTests(unittest.TestCase):
def test_HowtoResource_render_GET_lang_ru(self):
"""renderGet() with ?lang=ru should return the howto page in Russian."""
+
+ if 'ru' not in _langs.get_langs():
+ self.skipTest("'ru' language unsupported")
+
request = DummyRequest([self.pagename])
request.method = b'GET'
request.addArg('lang', 'ru')
@@ -522,7 +530,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase):
def testCB(request):
"""Check the ``Request`` returned from ``_renderDeferred``."""
self.assertIsInstance(request, DummyRequest)
- soup = BeautifulSoup(b''.join(request.written)).find('meta')['http-equiv']
+ soup = BeautifulSoup(b''.join(request.written)).find(b'meta')['http-equiv']
self.assertEqual(soup, 'refresh')
d = task.deferLater(reactor, 0, lambda x: x, (False, self.request))
@@ -541,7 +549,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase):
"""Check the ``Request`` returned from ``_renderDeferred``."""
self.assertIsInstance(request, DummyRequest)
html = b''.join(request.written)
- self.assertSubstring('Uh oh, spaghettios!', html)
+ self.assertSubstring(b'Uh oh, spaghettios!', html)
d = task.deferLater(reactor, 0, lambda x: x, (True, self.request))
d.addCallback(self.captchaResource._renderDeferred)
@@ -668,7 +676,7 @@ class BridgesResourceTests(unittest.TestCase):
# The bridge lines are contained in a <div class='bridges'> tag:
soup = BeautifulSoup(page)
well = soup.find('div', {'class': 'bridge-lines'})
- content = well.renderContents().strip()
+ content = well.renderContents().decode('utf-8').strip()
lines = content.splitlines()
bridges = []
@@ -794,6 +802,10 @@ class BridgesResourceTests(unittest.TestCase):
def test_render_GET_RTLlang(self):
"""Test rendering a request for plain bridges in Arabic."""
+
+ if 'ar' not in _langs.get_langs():
+ self.skipTest("'ar' language unsupported")
+
self.useBenignBridges()
request = DummyRequest([b"bridges?transport=obfs3"])
@@ -816,6 +828,10 @@ class BridgesResourceTests(unittest.TestCase):
def test_render_GET_RTLlang_obfs3(self):
"""Test rendering a request for obfs3 bridges in Farsi."""
+
+ if 'fa' not in _langs.get_langs():
+ self.skipTest("'ar' language unsupported")
+
self.useBenignBridges()
request = DummyRequest([b"bridges?transport=obfs3"])
@@ -915,6 +931,10 @@ class OptionsResourceTests(unittest.TestCase):
def test_render_GET_RTLlang(self):
"""Test rendering a request for obfs3 bridges in Hebrew."""
+
+ if 'he' not in _langs.get_langs():
+ self.skipTest("'ar' language unsupported")
+
request = DummyRequest(["bridges?transport=obfs3"])
request.method = b'GET'
request.getClientIP = lambda: '3.3.3.3'
diff --git a/bridgedb/util.py b/bridgedb/util.py
index 7b68831..db8e749 100644
--- a/bridgedb/util.py
+++ b/bridgedb/util.py
@@ -18,6 +18,7 @@ import logging
import logging.config
import logging.handlers
import os
+import re
import time
from twisted.python import components
@@ -260,17 +261,15 @@ def replaceControlChars(text, replacement=None, encoding="utf-8"):
:rtype: str
:returns: The sanitized **text**.
"""
- escaped = bytearray()
- for byte in bytearray(text, encoding):
- if byte in list(range(0, 32)) + [92, 127]:
- if replacement:
- byte = replacement
- else:
- continue
- escaped += bytearray([byte])
+ if replacement is None:
+ replacement = ''
+
+ # the following replaces characters 0-31, 92, and 127
+
+ text = text.decode(encoding) if isinstance(text, bytes) else text
+ return re.sub(r'[\x00-\x1f\x5c\x7f]', '', text)
- return str(escaped)
def registerAdapter(adapter, adapted, interface):
"""Register a Zope interface adapter for global use.
