tor-commits February 2020

tor-commits@lists.torproject.org

22 participants
1454 discussions

[bridgedb/develop] Fix txcaptcha tests
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit e2af8e34664c1ed91ccbe2b5b93e4b2b99f9270c Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 13 18:15:20 2020 -0800 Fix txcaptcha tests The _encodeIfNecessary() helper didn't make sense. It claimed to encode unicode into unicode whereas what it really did was simply normalizing inputs to bytes. I tested urlencode() with a mixture of unicode/bytes and it normalizes its inputs so this helper doesn't do anything. This fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_txrecaptcha.py", line 274, in test_encodeIfNecessary self.assertIsInstance(origString, unicode) builtins.NameError: name 'unicode' is not defined Test results changed as follows... before: FAILED (skips=109, failures=18, errors=386, successes=468) after: FAILED (skips=109, failures=18, errors=383, successes=470) --- bridgedb/test/test_txrecaptcha.py | 11 +---------- bridgedb/txrecaptcha.py | 21 ++++++++------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/bridgedb/test/test_txrecaptcha.py b/bridgedb/test/test_txrecaptcha.py index fd7365f..db37506 100644 --- a/bridgedb/test/test_txrecaptcha.py +++ b/bridgedb/test/test_txrecaptcha.py @@ -142,7 +142,7 @@ class BodyProducerTests(unittest.TestCase): def setUp(self): """Setup the tests.""" - self.content = 'Line 1\r\nLine 2\r\n' + self.content = b'Line 1\r\nLine 2\r\n' self.producer = txrecaptcha._BodyProducer(self.content) def test_interface(self): @@ -265,12 +265,3 @@ class MiscTests(unittest.TestCase): result = txrecaptcha._ebRequest(fail) self.assertIsInstance(result, txrecaptcha.RecaptchaResponse) self.assertRegexpMatches(result.error_code, msg) - - def test_encodeIfNecessary(self): - """:func:`txrecapcha._encodeIfNecessary` should convert unicode objects - into strings. - """ - origString = 'abc' - self.assertIsInstance(origString, unicode) - newString = txrecaptcha._encodeIfNecessary(origString) - self.assertIsInstance(newString, str) diff --git a/bridgedb/txrecaptcha.py b/bridgedb/txrecaptcha.py index 6308af8..7292360 100644 --- a/bridgedb/txrecaptcha.py +++ b/bridgedb/txrecaptcha.py @@ -46,8 +46,8 @@ from zope.interface import implementer from bridgedb.crypto import SSLVerifyingContextFactory #: This was taken from :data:`recaptcha.client.captcha.API_SSL_SERVER`. -API_SSL_SERVER = API_SERVER = "https://www.google.com/recaptcha/api" -API_SSL_VERIFY_URL = "%s/verify" % API_SSL_SERVER +API_SSL_SERVER = API_SERVER = b"https://www.google.com/recaptcha/api" +API_SSL_VERIFY_URL = b"%s/verify" % API_SSL_SERVER #: (:class:`OpenSSL.crypto.X509`) Only trust certificate for the reCAPTCHA #: :data:`API_SSL_SERVER` which were signed by the Google Internet Authority CA. @@ -253,12 +253,6 @@ def _ebRequest(fail): error = fail.getErrorMessage() or "possible problem in _ebRequest()" return RecaptchaResponse(is_valid=False, error_code=error) -def _encodeIfNecessary(string): - """Encode unicode objects in utf-8 if necessary.""" - if isinstance(string, unicode): - return string.encode('utf-8') - return string - def submit(recaptcha_challenge_field, recaptcha_response_field, private_key, remoteip, agent=_agent): """Submits a reCaptcha request for verification. This function is a patched @@ -292,13 +286,14 @@ def submit(recaptcha_challenge_field, recaptcha_response_field, return d params = urllib.parse.urlencode({ - 'privatekey': _encodeIfNecessary(private_key), - 'remoteip': _encodeIfNecessary(remoteip), - 'challenge': _encodeIfNecessary(recaptcha_challenge_field), - 'response': _encodeIfNecessary(recaptcha_response_field)}) + 'privatekey': private_key, + 'remoteip': remoteip, + 'challenge': recaptcha_challenge_field, + 'response': recaptcha_response_field, + }).encode('utf-8') body = _BodyProducer(params) headers = Headers({"Content-type": ["application/x-www-form-urlencoded"], "User-agent": ["reCAPTCHA Python"]}) - d = agent.request('POST', API_SSL_VERIFY_URL, headers, body) + d = agent.request(b'POST', API_SSL_VERIFY_URL, headers, body) d.addCallbacks(_cbRequest, _ebRequest) return d

1 0

[bridgedb/develop] Fix GOOGLE_INTERNET_AUTHORITY_CA_CERT constant initialization
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit f1a1dfe3b66f1857fcb6ce2f0291c88a2d6a7a72 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 13 18:05:22 2020 -0800 Fix GOOGLE_INTERNET_AUTHORITY_CA_CERT constant initialization Byte strings have always been initialized via b'msg'. Calling bytes() for this constant never made sense - in python 2.x it was a no-op, and in python 3.x it errors... Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/trial/runner.py", line 823, in loadByName return self.suiteFactory([self.findByName(name, recurse=recurse)]) File "/usr/local/lib/python3.5/dist-packages/twisted/trial/runner.py", line 702, in findByName __import__(name) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_txrecaptcha.py", line 31, in <module> from bridgedb import txrecaptcha File "/home/atagar/Desktop/tor/bridgedb/bridgedb/txrecaptcha.py", line 71, in <module> -----END CERTIFICATE-----""")) builtins.TypeError: string argument without an encoding This changes the test results as follows... before: FAILED (skips=106, failures=17, errors=256, successes=425) after: FAILED (skips=109, failures=18, errors=386, successes=468) --- bridgedb/txrecaptcha.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridgedb/txrecaptcha.py b/bridgedb/txrecaptcha.py index 885b40d..26ead43 100644 --- a/bridgedb/txrecaptcha.py +++ b/bridgedb/txrecaptcha.py @@ -51,7 +51,7 @@ API_SSL_VERIFY_URL = "%s/verify" % API_SSL_SERVER #: (:class:`OpenSSL.crypto.X509`) Only trust certificate for the reCAPTCHA #: :data:`API_SSL_SERVER` which were signed by the Google Internet Authority CA. -GOOGLE_INTERNET_AUTHORITY_CA_CERT = load_certificate(FILETYPE_PEM, bytes("""\ +GOOGLE_INTERNET_AUTHORITY_CA_CERT = load_certificate(FILETYPE_PEM, b"""\ -----BEGIN CERTIFICATE----- MIICsDCCAhmgAwIBAgIDFXfhMA0GCSqGSIb3DQEBBQUAME4xCzAJBgNVBAYTAlVT MRAwDgYDVQQKEwdFcXVpZmF4MS0wKwYDVQQLEyRFcXVpZmF4IFNlY3VyZSBDZXJ0 @@ -68,7 +68,7 @@ Y3Jscy9zZWN1cmVjYS5jcmwwDQYJKoZIhvcNAQEFBQADgYEAvprjecFG+iJsxzEF ZUNgujFQodUovxOWZshcnDW7fZ7mTlk3zpeVJrGPZzhaDhvuJjIfKqHweFB7gwB+ ARlIjNvrPq86fpVg0NOTawALkSqOUMl3MynBQO+spR7EHcRbADQ/JemfTEh2Ycfl vZqhEFBfurZkX0eTANq98ZvVfpg= ------END CERTIFICATE-----""")) +-----END CERTIFICATE-----""") # `t.w.client.HTTPConnectionPool` isn't available in Twisted-12.0.0 # (see ticket #11219: https://bugs.torproject.org/11219):

1 0

[bridgedb/develop] Drop unicode argument from gettext's install
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 61e82530f027463b164f9181ab3de1db08bd6668 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 13 16:10:54 2020 -0800 Drop unicode argument from gettext's install Python 3.x is unicode by default and gettext doesn't have this argument any longer so simply dropping it... https://docs.python.org/3/library/gettext.html#gettext.NullTranslations.ins… This fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_translations.py", line 63, in test_getLocaleFromHTTPRequest_withLangParam_AcceptLanguage parsed = translations.getLocaleFromHTTPRequest(request) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/translations.py", line 90, in getLocaleFromHTTPRequest installTranslations(langs) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/translations.py", line 129, in installTranslations language.install(unicode=True) builtins.TypeError: install() got an unexpected keyword argument 'unicode' Test results changed as follows... before: FAILED (skips=106, failures=16, errors=266, successes=416) after: FAILED (skips=106, failures=17, errors=258, successes=423) --- bridgedb/translations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgedb/translations.py b/bridgedb/translations.py index f2b06e9..10c27fc 100644 --- a/bridgedb/translations.py +++ b/bridgedb/translations.py @@ -126,7 +126,7 @@ def installTranslations(langs): except IOError as error: logging.error(str(error)) - language.install(unicode=True) + language.install() return language def usingRTLLang(langs):

1 0

[bridgedb/develop] Fix urlencode call
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 7cc865b7899ad9389d4748ce1ed388282df5995c Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 13 18:10:21 2020 -0800 Fix urlencode call Oops, my bad on this one. Earlier I ported from urllib2 to urllib, but the urlencode function is on urllib.parse instead... https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode This fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_txrecaptcha.py", line 205, in test_submit_returnsDeferred self.ip) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/txrecaptcha.py", line 294, in submit params = urllib.urlencode({ builtins.AttributeError: module 'urllib' has no attribute 'urlencode' Test results unchanged... before: FAILED (skips=109, failures=18, errors=386, successes=468) after: FAILED (skips=109, failures=18, errors=386, successes=468) --- bridgedb/txrecaptcha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgedb/txrecaptcha.py b/bridgedb/txrecaptcha.py index 26ead43..6308af8 100644 --- a/bridgedb/txrecaptcha.py +++ b/bridgedb/txrecaptcha.py @@ -291,7 +291,7 @@ def submit(recaptcha_challenge_field, recaptcha_response_field, d.errback(failure.Failure(ValueError('incorrect-captcha-sol'))) return d - params = urllib.urlencode({ + params = urllib.parse.urlencode({ 'privatekey': _encodeIfNecessary(private_key), 'remoteip': _encodeIfNecessary(remoteip), 'challenge': _encodeIfNecessary(recaptcha_challenge_field),

1 0

[bridgedb/develop] Normalize getHMAC inputs
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 7ac4b8ea8605470371d78ba6e3c58731c9549c57 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 14 14:29:42 2020 -0800 Normalize getHMAC inputs Calculating the hmac requires bytes. This fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_https_distributor.py", line 152, in test_HTTPSDistributor_prepopulateRings_without_proxies dist = distributor.HTTPSDistributor(3, self.key) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/https/distributor.py", line 93, in __init__ key2 = getHMAC(key, "Assign-Bridges-To-Rings") File "/home/atagar/Desktop/tor/bridgedb/bridgedb/crypto.py", line 193, in getHMAC h = hmac.new(key, value, digestmod=DIGESTMOD) File "/usr/lib/python3.5/hmac.py", line 144, in new return HMAC(key, msg, digestmod) File "/usr/lib/python3.5/hmac.py", line 42, in __init__ raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) builtins.TypeError: key: expected bytes or bytearray, but got 'str' Test results changed as follows... before: FAILED (skips=109, failures=22, errors=318, successes=531) after: FAILED (skips=109, failures=21, errors=283, successes=567) --- bridgedb/crypto.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bridgedb/crypto.py b/bridgedb/crypto.py index b39c659..bd3d783 100644 --- a/bridgedb/crypto.py +++ b/bridgedb/crypto.py @@ -190,6 +190,12 @@ def getKey(filename): def getHMAC(key, value): """Return the HMAC of **value** using the **key**.""" + + # normalize inputs to be bytes + + key = key.encode('utf-8') if isinstance(key, str) else key + value = value.encode('utf-8') if isinstance(value, str) else value + h = hmac.new(key, value, digestmod=DIGESTMOD) return h.digest() @@ -200,14 +206,19 @@ def getHMACFunc(key, hex=True): :rtype: callable :returns: A function which can be uses to generate HMACs. """ + + key = key.encode('utf-8') if isinstance(key, str) else key h = hmac.new(key, digestmod=DIGESTMOD) + def hmac_fn(value): + value = value.encode('utf-8') if isinstance(value, str) else value h_tmp = h.copy() h_tmp.update(value) if hex: return h_tmp.hexdigest() else: return h_tmp.digest() + return hmac_fn def removePKCS1Padding(message):

1 0

[bridgedb/develop] Drop unicode argument from gettext.install()
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 0ac0e46a2afaa67cbf6719770e4fff592f148496 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 13 19:02:30 2020 -0800 Drop unicode argument from gettext.install() Oops, I removed this deprecated argument from gettext.translation's install method but I missed that there were similar calls to this function as well. This changes the test results as follows... before: FAILED (skips=109, failures=18, errors=383, successes=470) after: FAILED (skips=109, failures=24, errors=364, successes=483) --- bridgedb/distributors/https/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index db7e2fe..d1c058c 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -373,7 +373,7 @@ class TranslatedTemplateResource(CustomErrorHandlingResource, CSPResource): """Create a new :api:`Resource <twisted.web.resource.Resource>` for a Mako-templated webpage. """ - gettext.install("bridgedb", unicode=True) + gettext.install("bridgedb") CSPResource.__init__(self) self.template = template @@ -888,7 +888,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): :param bool includeFingerprints: Do we include the bridge's fingerprint in the response? """ - gettext.install("bridgedb", unicode=True) + gettext.install("bridgedb") CSPResource.__init__(self) self.distributor = distributor self.schedule = schedule

1 0

[bridgedb/develop] Replace rfc822 with email.message
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 1138daa8e1202d1649a70c7b238ac7ebcade23a8 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 10 16:13:01 2020 -0800 Replace rfc822 with email.message Hardest part of this migration yet. Python 2's deprecated rfc8222 module was removed in python 3. Unfortunately Isis used a catch-all here, covering the fact that this code has always been a bit broken (due to overriding our 'addr' import with a local variable)... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_email_autoresponder.py", line 353, in test_SMTPAutoresponder_getMailFrom_givemebridges_at_seriously recipient = str(self.responder.getMailFrom()) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/email/autoresponder.py", line 535, in getMailFrom raise addr.BadEmail(allRecipients) builtins.AttributeError: 'str' object has no attribute 'BadEmail' Finally the single test that exercises this code path works... before: FAILED (skips=109, failures=24, errors=225, successes=623) after: FAILED (skips=109, failures=23, errors=225, successes=624) --- bridgedb/distributors/email/autoresponder.py | 19 ++++++++++--------- bridgedb/distributors/email/server.py | 12 ++++-------- bridgedb/test/test_email_dkim.py | 4 ++-- bridgedb/test/test_email_server.py | 6 +++--- 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/bridgedb/distributors/email/autoresponder.py b/bridgedb/distributors/email/autoresponder.py index 1d2b618..3711eae 100644 --- a/bridgedb/distributors/email/autoresponder.py +++ b/bridgedb/distributors/email/autoresponder.py @@ -39,6 +39,7 @@ Functionality for autoresponding to incoming emails. from __future__ import unicode_literals from __future__ import print_function +import email import io import logging import time @@ -439,8 +440,8 @@ class SMTPAutoresponder(smtp.SMTPClient): if not body: return # The client was already warned. - messageID = self.incoming.message.getheader("Message-ID", None) - subject = self.incoming.message.getheader("Subject", None) + messageID = self.incoming.message.get("Message-ID", None) + subject = self.incoming.message.get("Subject", None) response = generateResponse(recipient, client, body, subject, messageID, self.incoming.context.gpgSignFunc) @@ -461,13 +462,13 @@ class SMTPAutoresponder(smtp.SMTPClient): """ clients = [] addrHeader = None - try: fromAddr = self.incoming.message.getaddr("From")[1] + try: fromAddr = email.utils.parseaddr(self.incoming.message.get("From"))[1] except (IndexError, TypeError, AttributeError): pass else: addrHeader = fromAddr if not addrHeader: logging.warn("No From header on incoming mail.") - try: senderHeader = self.incoming.message.getaddr("Sender")[1] + try: senderHeader = email.utils.parseaddr(self.incoming.message.get("Sender"))[1] except (IndexError, TypeError, AttributeError): pass else: addrHeader = senderHeader if not addrHeader: @@ -509,10 +510,10 @@ class SMTPAutoresponder(smtp.SMTPClient): try: ourAddress = smtp.Address(self.incoming.context.fromAddr) - allRecipients = self.incoming.message.getaddrlist("To") + allRecipients = self.incoming.message.get_all("To") - for _, addr in allRecipients: - recipient = smtp.Address(addr) + for address in allRecipients: + recipient = smtp.Address(address) if not ourAddress.domain in recipient.domain: logging.debug(("Not our domain (%s) or subdomain, skipping" " email address: %s") @@ -525,11 +526,11 @@ class SMTPAutoresponder(smtp.SMTPClient): " email address: %s") % str(recipient)) continue # Only check the username before the first '+': - beforePlus = recipient.local.split('+', 1)[0] + beforePlus = recipient.local.split(b'+', 1)[0] if beforePlus == ourAddress.local: ours = str(recipient) if not ours: - raise addr.BadEmail(allRecipients) + raise addr.BadEmail('No email address accepted, please see log', allRecipients) except Exception as error: logging.error(("Couldn't find our email address in incoming email " diff --git a/bridgedb/distributors/email/server.py b/bridgedb/distributors/email/server.py index 0f96812..8a2fbea 100644 --- a/bridgedb/distributors/email/server.py +++ b/bridgedb/distributors/email/server.py @@ -49,10 +49,10 @@ Servers which interface with clients and distribute bridges over SMTP. from __future__ import unicode_literals +import email.message import logging import io import socket -import rfc822 from twisted.internet import defer from twisted.internet import reactor @@ -222,7 +222,7 @@ class SMTPMessage(object): if self.nBytes > self.context.maximumSize: self.ignoring = True else: - self.lines.append(line) + self.lines.append(line.decode('utf-8') if isinstance(line, bytes) else line) if not safelog.safe_logging: try: ln = line.rstrip("\r\n").encode('utf-8', 'replace') @@ -251,12 +251,8 @@ class SMTPMessage(object): :rtype: :api:`twisted.mail.smtp.rfc822.Message` :returns: A ``Message`` comprised of all lines received thus far. """ - rawMessage = io.StringIO() - for line in self.lines: - line = line.decode('utf-8') if isinstance(line, bytes) else line - rawMessage.writelines(line + '\n') - rawMessage.seek(0) - return rfc822.Message(rawMessage) + + return email.message_from_string('\n'.join(self.lines)) @implementer(smtp.IMessageDelivery) diff --git a/bridgedb/test/test_email_dkim.py b/bridgedb/test/test_email_dkim.py index e8fe3f3..2330a71 100644 --- a/bridgedb/test/test_email_dkim.py +++ b/bridgedb/test/test_email_dkim.py @@ -11,8 +11,8 @@ """Unittests for the :mod:`bridgedb.distributors.email.dkim` module.""" +import email.message import io -import rfc822 from twisted.trial import unittest @@ -47,7 +47,7 @@ get bridges def _createMessage(self, messageString): """Create an ``email.message.Message`` from a string.""" - messageIO = io.StringIO(messageString if isinstance(messageString, str) else messageString.decode('utf-8')) + messageIO = io.StringIO(unicode(messageString)) return rfc822.Message(messageIO) def test_checkDKIM_good(self): diff --git a/bridgedb/test/test_email_server.py b/bridgedb/test/test_email_server.py index bc331f4..081a0e4 100644 --- a/bridgedb/test/test_email_server.py +++ b/bridgedb/test/test_email_server.py @@ -13,10 +13,10 @@ from __future__ import print_function +import email.message import socket import string import types -import rfc822 from twisted.python import log from twisted.internet import defer @@ -95,10 +95,10 @@ class SMTPMessageTests(unittest.TestCase): defer.Deferred) def test_SMTPMessage_getIncomingMessage(self): - """``getIncomingMessage`` should return a ``rfc822.Message``.""" + """``getIncomingMessage`` should return a ``email.message.Message``.""" self.message.lineReceived(self.line) self.assertIsInstance(self.message.getIncomingMessage(), - rfc822.Message) + email.message.Message) class SMTPIncomingDeliveryTests(unittest.TestCase):

1 0

[bridgedb/develop] Fix most autoresponder tests
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 5aa0adcbb2725011ecd7fd11eb94d0bc2c30ab42 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 14 16:12:15 2020 -0800 Fix most autoresponder tests Low hanging fruit to get all the autoresponder tests aside from 'getMailFrom_plus_address' passing. This last test involves API differences between the rfc822 and email modules. Errors this resolve are the usual byte/unicode differences... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_email_autoresponder.py", line 256, in test_EmailResponse_write response.write(self.body) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/email/autoresponder.py", line 273, in write self.mailfile.write(line.encode('utf8')) builtins.TypeError: string argument expected, got 'bytes' Indexing into an iterable... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_email_autoresponder.py", line 50, in setUp self.ctx = _createMailServerContext(self.config) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/email_helpers.py", line 119, in _createMailServerContext context = MailServerContext(config, distributor, Unscheduled()) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/email/server.py", line 142, in __init__ self.gpg, self.gpgSignFunc = initializeGnuPG(config) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/crypto.py", line 318, in initializeGnuPG for sub in primaryPK[0]['subkeys']: builtins.TypeError: 'filter' object is not subscriptable ... and double encoding... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_email_autoresponder.py", line 251, in test_EmailResponse_read_three_bytes self.assertEqual(contents, self.body[:3]) File "/usr/local/lib/python3.5/dist-packages/twisted/trial/_synctest.py", line 432, in assertEqual super(_Assertions, self).assertEqual(first, second, msg) File "/usr/lib/python3.5/unittest/case.py", line 820, in assertEqual assertion_func(first, second, msg=msg) File "/usr/lib/python3.5/unittest/case.py", line 1193, in assertMultiLineEqual self.fail(self._formatMessage(msg, standardMsg)) twisted.trial.unittest.FailTest: "b'Peo'" != 'Peo' - b'Peo' + Peo Test results changed as follows... before: FAILED (skips=109, failures=21, errors=283, successes=567) after: FAILED (skips=109, failures=24, errors=225, successes=623) --- bridgedb/crypto.py | 2 +- bridgedb/distributors/email/autoresponder.py | 24 ++++++++++++++++-------- bridgedb/distributors/email/server.py | 3 ++- bridgedb/test/test_email_autoresponder.py | 8 ++++---- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/bridgedb/crypto.py b/bridgedb/crypto.py index bd3d783..5b1912f 100644 --- a/bridgedb/crypto.py +++ b/bridgedb/crypto.py @@ -315,7 +315,7 @@ def initializeGnuPG(config): if primarySK and primaryPK: logging.info("Found GnuPG primary key with fingerprint: %s" % primary) - for sub in primaryPK[0]['subkeys']: + for sub in list(primaryPK)[0]['subkeys']: logging.info(" Subkey: %s Usage: %s" % (sub[0], sub[1].upper())) else: logging.warn("GnuPG key %s could not be found in %s!" % (primary, gpg.secring)) diff --git a/bridgedb/distributors/email/autoresponder.py b/bridgedb/distributors/email/autoresponder.py index bc1c7b9..1d2b618 100644 --- a/bridgedb/distributors/email/autoresponder.py +++ b/bridgedb/distributors/email/autoresponder.py @@ -263,6 +263,9 @@ class EmailResponse(object): :param str line: Something to append into the :data:`mailfile`. """ + + line = line.decode('utf-8') if isinstance(line, bytes) else line + if line.find('\r\n') != -1: # If **line** contains newlines, send it to :meth:`writelines` to # break it up so that we can replace them: @@ -270,7 +273,7 @@ class EmailResponse(object): self.writelines(line) else: line += self.delimiter - self.mailfile.write(line.encode('utf8')) + self.mailfile.write(line) self.mailfile.flush() def writelines(self, lines): @@ -283,7 +286,8 @@ class EmailResponse(object): :type lines: :any:`str` or :any:`list` :param lines: The lines to write to the :attr:`mailfile`. """ - if isinstance(lines, str): + if isinstance(lines, (str, bytes)): + lines = lines.decode('utf-8') if isinstance(lines, bytes) else lines lines = lines.replace('\r\n', '\n') for ln in lines.split('\n'): self.write(ln) @@ -310,20 +314,24 @@ class EmailResponse(object): :kwargs: If given, the key will become the name of the header, and the value will become the contents of that header. """ + + fromAddress = fromAddress.decode('utf-8') if isinstance(fromAddress, bytes) else fromAddress + toAddress = toAddress.decode('utf-8') if isinstance(toAddress, bytes) else toAddress + self.write("From: %s" % fromAddress) self.write("To: %s" % toAddress) if includeMessageID: - self.write("Message-ID: %s" % smtp.messageid().encode('utf-8')) + self.write("Message-ID: %s" % smtp.messageid()) if inReplyTo: - self.write("In-Reply-To: %s" % inReplyTo.encode('utf-8')) - self.write("Content-Type: %s" % contentType.encode('utf-8')) - self.write("Date: %s" % smtp.rfc822date().encode('utf-8')) + self.write("In-Reply-To: %s" % inReplyTo) + self.write("Content-Type: %s" % contentType) + self.write("Date: %s" % smtp.rfc822date().decode('utf-8')) if not subject: subject = '[no subject]' if not subject.lower().startswith('re'): subject = "Re: " + subject - self.write("Subject: %s" % subject.encode('utf-8')) + self.write("Subject: %s" % subject) if kwargs: for headerName, headerValue in kwargs.items(): @@ -331,7 +339,7 @@ class EmailResponse(object): headerName = headerName.replace(' ', '-') headerName = headerName.replace('_', '-') header = "%s: %s" % (headerName, headerValue) - self.write(header.encode('utf-8')) + self.write(header) # The first blank line designates that the headers have ended: self.write(self.delimiter) diff --git a/bridgedb/distributors/email/server.py b/bridgedb/distributors/email/server.py index c1ce56f..0f96812 100644 --- a/bridgedb/distributors/email/server.py +++ b/bridgedb/distributors/email/server.py @@ -253,7 +253,8 @@ class SMTPMessage(object): """ rawMessage = io.StringIO() for line in self.lines: - rawMessage.writelines(line.decode('utf8') + '\n') + line = line.decode('utf-8') if isinstance(line, bytes) else line + rawMessage.writelines(line + '\n') rawMessage.seek(0) return rfc822.Message(rawMessage) diff --git a/bridgedb/test/test_email_autoresponder.py b/bridgedb/test/test_email_autoresponder.py index f59e5ef..c8e9624 100644 --- a/bridgedb/test/test_email_autoresponder.py +++ b/bridgedb/test/test_email_autoresponder.py @@ -235,7 +235,7 @@ a ball of timey-wimey, wibbly-warbly... stuff.""" response = autoresponder.EmailResponse() response.write(self.body) response.rewind() - contents = str(response.read()).replace('\x00', '') + contents = response.read().replace(b'\x00', b'').decode('utf-8') # The newlines in the email body should have been replaced with # ``EmailResponse.delimiter``. delimited = self.body.replace('\n', response.delimiter) \ @@ -247,7 +247,7 @@ a ball of timey-wimey, wibbly-warbly... stuff.""" response = autoresponder.EmailResponse() response.write(self.body) response.rewind() - contents = str(response.read(3)).replace('\x00', '') + contents = response.read(3).replace(b'\x00', b'').decode('utf-8') self.assertEqual(contents, self.body[:3]) def test_EmailResponse_write(self): @@ -370,7 +370,7 @@ class SMTPAutoresponderTests(unittest.TestCase): """ self._getIncomingLines() ours = Address(self.context.fromAddr) - plus = '@'.join([ours.local + '+zh_cn', ours.domain]) + plus = '@'.join([ours.local.decode('utf-8') + '+zh_cn', ours.domain.decode('utf-8')]) self.message.lines[1] = 'To: {0}'.format(plus) self._setUpResponder() recipient = str(self.responder.getMailFrom()) @@ -383,7 +383,7 @@ class SMTPAutoresponderTests(unittest.TestCase): """ self._getIncomingLines() ours = Address(self.context.fromAddr) - plus = '@'.join(['get' + ours.local + '+zh_cn', ours.domain]) + plus = '@'.join(['get' + ours.local.decode('utf-8') + '+zh_cn', ours.domain.decode('utf-8')]) self.message.lines[1] = 'To: {0}'.format(plus) self._setUpResponder() recipient = str(self.responder.getMailFrom())

1 0

[bridgedb/develop] Fix descriptor parsing tests
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit dc07e4e34d8b5f31f015362ee239f07c0b203a44 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 13 19:07:29 2020 -0800 Fix descriptor parsing tests Mostly just simple byte normalizaiton issues. Test results changed as follows... before: FAILED (skips=109, failures=24, errors=364, successes=483) after: FAILED (skips=109, failures=23, errors=349, successes=499) --- bridgedb/parse/descriptors.py | 24 +------ bridgedb/test/test_parse_descriptors.py | 110 ++++++++++++++++---------------- 2 files changed, 56 insertions(+), 78 deletions(-) diff --git a/bridgedb/parse/descriptors.py b/bridgedb/parse/descriptors.py index ff32226..f6dee83 100644 --- a/bridgedb/parse/descriptors.py +++ b/bridgedb/parse/descriptors.py @@ -164,28 +164,6 @@ def parseServerDescriptorsFile(filename, validate=True): return list(document) -def __cmp_published__(x, y): - """A custom ``cmp()`` which sorts descriptors by published date. - - :rtype: int - :returns: Return negative if x<y, zero if x==y, positive if x>y. - """ - if x.published < y.published: - return -1 - elif x.published == y.published: - # This *shouldn't* happen. It would mean that two descriptors for - # the same router had the same timestamps, probably meaning there - # is a severely-messed up OR implementation out there. Let's log - # its fingerprint (no matter what!) so that we can look up its - # ``platform`` line in its server-descriptor and tell whoever - # wrote that code that they're probably (D)DOSing the Tor network. - logging.warn(("Duplicate descriptor with identical timestamp (%s) " - "for bridge %s with fingerprint %s !") % - (x.published, x.nickname, x.fingerprint)) - return 0 - elif x.published > y.published: - return 1 - def deduplicate(descriptors, statistics=False): """Deduplicate some descriptors, returning only the newest for each router. @@ -215,7 +193,7 @@ def deduplicate(descriptors, statistics=False): duplicates[fingerprint] = [descriptor,] for fingerprint, dupes in duplicates.items(): - dupes.sort(cmp=__cmp_published__) + dupes.sort(key=lambda x: x.published) first = dupes.pop() newest[fingerprint] = first duplicates[fingerprint] = dupes diff --git a/bridgedb/test/test_parse_descriptors.py b/bridgedb/test/test_parse_descriptors.py index 7fd7582..eb86a0e 100644 --- a/bridgedb/test/test_parse_descriptors.py +++ b/bridgedb/test/test_parse_descriptors.py @@ -36,7 +36,7 @@ else: from bridgedb.test.util import Benchmarker -BRIDGE_NETWORKSTATUS_0 = '''\ +BRIDGE_NETWORKSTATUS_0 = b'''\ r MiserLandfalls 4IsyTSCtChPhFPAnq5rD8yymlqA /GMC4lz8RXT/62v6kZNdmzSmopk 2014-11-04 06:23:22 2.215.61.223 4056 0 a [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055 s Fast Guard Running Stable Valid @@ -44,7 +44,7 @@ w Bandwidth=1678904 p reject 1-65535 ''' -BRIDGE_NETWORKSTATUS_1 = '''\ +BRIDGE_NETWORKSTATUS_1 = b'''\ r Unmentionable BgOrX0ViP5hNsK5ZvixAuPZ6EY0 NTg9NoE5ls9KjF96Dp/UdrabZ9Y 2014-11-04 12:23:37 80.44.173.87 51691 0 a [da14:7d1e:ba8e:60d0:b078:3f88:382b:5c70]:51690 s Fast Guard Running Stable Valid @@ -52,7 +52,7 @@ w Bandwidth=24361 p reject 1-65535 ''' -BRIDGE_SERVER_DESCRIPTOR = '''\ +BRIDGE_SERVER_DESCRIPTOR = b'''\ @purpose bridge router MiserLandfalls 2.215.61.223 4056 0 0 or-address [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055 @@ -86,7 +86,7 @@ P2aB/+XQfzFBA5TaWF83coDng4OGodhwHaOx10Kn7Bg= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR = b'''\ extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 published 2014-11-04 06:23:22 write-history 2014-11-04 06:23:22 (900 s) 3188736,2226176,2866176 @@ -117,7 +117,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE = b'''\ extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 published 2014-11-04 08:10:25 write-history 2014-11-04 08:10:25 (900 s) 3188736,2226176,2866176,2226176 @@ -148,7 +148,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE = b'''\ extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 published 2014-12-04 03:10:25 write-history 2014-12-04 03:10:25 (900 s) 3188736,2226176,2866176,2226176 @@ -179,7 +179,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' -BRIDGE_SERVER_DESCRIPTOR_ED25519 = '''\ +BRIDGE_SERVER_DESCRIPTOR_ED25519 = b'''\ @purpose bridge router piratepartei 80.92.79.70 80 0 0 identity-ed25519 @@ -233,7 +233,7 @@ VC4FdHgFlAkXbiqkpWtD0ojJJjLlEeXbmGILjC1Ls2I= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR_ED25519 = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR_ED25519 = b'''\ extra-info piratepartei 312D64274C29156005843EECB19C6865FA3CC10C identity-ed25519 -----BEGIN ED25519 CERT----- @@ -307,7 +307,7 @@ class ParseDescriptorsTests(unittest.TestCase): :returns: The full path to the file which was written to. """ descFilename = os.path.join(os.getcwd(), filename) - with open(descFilename, 'w') as fh: + with open(descFilename, 'wb') as fh: for desc in descriptors: fh.write(desc) fh.flush() @@ -381,8 +381,8 @@ class ParseDescriptorsTests(unittest.TestCase): raise InvalidRouterNickname. """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( - 'MiserLandfalls', - 'MiserLandfallsWaterfallsSnowfallsAvalanche') + b'MiserLandfalls', + b'MiserLandfallsWaterfallsSnowfallsAvalanche') # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', @@ -399,8 +399,8 @@ class ParseDescriptorsTests(unittest.TestCase): See also: :trac:`16616` """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( - 's Fast Guard Running Stable Valid', - 's Fast Guard Running Stable Valid HSDir') + b's Fast Guard Running Stable Valid', + b's Fast Guard Running Stable Valid HSDir') # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', @@ -420,7 +420,7 @@ class ParseDescriptorsTests(unittest.TestCase): a ValueError. """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( - '2.215.61.223', '[2837:fcd2:387b:e376:34c:1ec7:11ff:1686]') + b'2.215.61.223', b'[2837:fcd2:387b:e376:34c:1ec7:11ff:1686]') descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', unparseable) self.assertRaises(ValueError, @@ -434,9 +434,9 @@ class ParseDescriptorsTests(unittest.TestCase): expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] descFile = 'networkstatus-bridges' - with open(descFile, 'w') as fh: - fh.write('signature and stuff from the BridgeAuth would go here\n') - fh.write('some more annotations with parameters and stuff\n') + with open(descFile, 'wb') as fh: + fh.write(b'signature and stuff from the BridgeAuth would go here\n') + fh.write(b'some more annotations with parameters and stuff\n') fh.write(BRIDGE_NETWORKSTATUS_0) fh.write(BRIDGE_NETWORKSTATUS_1) fh.flush() @@ -454,9 +454,9 @@ class ParseDescriptorsTests(unittest.TestCase): expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] descFile = 'networkstatus-bridges' - with open(descFile, 'w') as fh: - fh.write('signature and stuff from the BridgeAuth would go here\n') - fh.write('some more annotations with parameters and stuff\n') + with open(descFile, 'wb') as fh: + fh.write(b'signature and stuff from the BridgeAuth would go here\n') + fh.write(b'some more annotations with parameters and stuff\n') fh.write(BRIDGE_NETWORKSTATUS_0) fh.write(BRIDGE_NETWORKSTATUS_1) fh.flush() @@ -479,7 +479,7 @@ class ParseDescriptorsTests(unittest.TestCase): """ descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseExtraInfoFiles(descFile) - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertIsInstance(bridge, RelayExtraInfoDescriptor) def test_parse_descriptors_parseExtraInfoFiles_one_file(self): @@ -488,12 +488,12 @@ class ParseDescriptorsTests(unittest.TestCase): """ descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseExtraInfoFiles(descFile) - bridge = routers.values()[0] + bridge = list(routers.values())[0] # The number of transports we parsed should be equal to the number of # 'transport' lines in the descriptor: self.assertEqual(len(bridge.transport), - BRIDGE_EXTRA_INFO_DESCRIPTOR.count('transport ')) + BRIDGE_EXTRA_INFO_DESCRIPTOR.count(b'transport ')) self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) @@ -522,7 +522,7 @@ class ParseDescriptorsTests(unittest.TestCase): "We shouldn't have any duplicate descriptors.") # We should only have the newest descriptor: - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.published, datetime.datetime.strptime("2014-11-04 08:10:25", "%Y-%m-%d %H:%M:%S"), @@ -541,7 +541,7 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertEqual(len(routers), 1, "We shouldn't have any duplicate descriptors.") - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.published, datetime.datetime.strptime("2014-11-04 08:10:25", "%Y-%m-%d %H:%M:%S"), @@ -564,7 +564,7 @@ class ParseDescriptorsTests(unittest.TestCase): "We shouldn't have any duplicate descriptors.") # We should only have the newest descriptor: - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.published, datetime.datetime.strptime("2014-12-04 03:10:25", "%Y-%m-%d %H:%M:%S"), @@ -581,10 +581,10 @@ class ParseDescriptorsTests(unittest.TestCase): descFiles = [] # The timestamp and fingerprint from BRIDGE_EXTRA_INFO_DESCRIPTOR: - timestamp = "2014-11-04 06:23:22" - Y, M, rest = timestamp.split("-") - fpr = "E08B324D20AD0A13E114F027AB9AC3F32CA696A0" - newerFpr = "E08B324D20AD0A13E114F027AB9AC3F32CA696A0" + timestamp = b"2014-11-04 06:23:22" + Y, M, rest = timestamp.split(b"-") + fpr = b"E08B324D20AD0A13E114F027AB9AC3F32CA696A0" + newerFpr = b"E08B324D20AD0A13E114F027AB9AC3F32CA696A0" total = 0 needed = b * n @@ -592,15 +592,15 @@ class ParseDescriptorsTests(unittest.TestCase): if total >= needed: break # Re-digest the fingerprint to create a "new" bridge - newerFpr = hashlib.sha1(newerFpr).hexdigest().upper() + newerFpr = hashlib.sha1(newerFpr).hexdigest().upper().encode('utf-8') # Generate n extrainfos with different timestamps: count = 0 - for year in range(1, ((n + 1)/ 12) + 2): # Start from the next year + for year in range(1, ((n + 1)// 12) + 2): # Start from the next year if count >= n: break for month in range(1, 13): if count < n: - newerTimestamp = "-".join([str(int(Y) + year), "%02d" % month, rest]) + newerTimestamp = b"-".join([str(int(Y) + year).encode('utf-8'), b"%02d" % month, rest]) newerDuplicate = BRIDGE_EXTRA_INFO_DESCRIPTOR[:].replace( fpr, newerFpr).replace( timestamp, newerTimestamp) @@ -663,10 +663,10 @@ class ParseDescriptorsTests(unittest.TestCase): """ # Give it a bad geoip-db-digest: unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( - "geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", - "geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( + b"geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", + b"geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) @@ -684,7 +684,7 @@ class ParseDescriptorsTests(unittest.TestCase): "and one was unparseable, so that should only leave one " "descriptor remaining.")) - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.fingerprint, "E08B324D20AD0A13E114F027AB9AC3F32CA696A0", @@ -703,14 +703,14 @@ class ParseDescriptorsTests(unittest.TestCase): """ # Mess up the bridge-ip-transports line: unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( - "bridge-ip-transports <OR>=8", - "bridge-ip-transports <OR>") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( + b"bridge-ip-transports <OR>=8", + b"bridge-ip-transports <OR>") parseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "ImOkWithBeingParsed 2B5DA67FBA13A6449DE625673B7AE9E3AA7DF75F") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"ImOkWithBeingParsed 2B5DA67FBA13A6449DE625673B7AE9E3AA7DF75F") descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE) @@ -750,8 +750,8 @@ class ParseDescriptorsTests(unittest.TestCase): zero parsed descriptors. """ unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - '-----END SIGNATURE-----', - '-----END SIGNATURE FOR REALZ-----') + b'-----END SIGNATURE-----', + b'-----END SIGNATURE FOR REALZ-----') # This must be a "real" file or _copyUnparseableDescriptorFile() will # raise an AttributeError saying: # '_io.BytesIO' object has no attribute 'rpartition'" @@ -766,7 +766,7 @@ class ParseDescriptorsTests(unittest.TestCase): missing the signature should return zero parsed descriptors. """ # Remove the signature - BEGIN_SIG = '-----BEGIN SIGNATURE-----' + BEGIN_SIG = b'-----BEGIN SIGNATURE-----' unparseable, _ = BRIDGE_EXTRA_INFO_DESCRIPTOR.split(BEGIN_SIG) # This must be a "real" file or _copyUnparseableDescriptorFile() will # raise an AttributeError saying: @@ -782,7 +782,7 @@ class ParseDescriptorsTests(unittest.TestCase): bad signature should raise an InvalidExtraInfoSignature exception. """ # Truncate the signature to 50 bytes - BEGIN_SIG = '-----BEGIN SIGNATURE-----' + BEGIN_SIG = b'-----BEGIN SIGNATURE-----' doc, sig = BRIDGE_EXTRA_INFO_DESCRIPTOR.split(BEGIN_SIG) unparseable = BEGIN_SIG.join([doc, sig[:50]]) # This must be a "real" file or _copyUnparseableDescriptorFile() will @@ -803,10 +803,10 @@ class ParseDescriptorsTests(unittest.TestCase): """ # Give it a bad geoip-db-digest: unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( - "geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", - "geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( + b"geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", + b"geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE) @@ -817,7 +817,7 @@ class ParseDescriptorsTests(unittest.TestCase): def test_parse_descriptors_parseExtraInfoFiles_empty_file(self): """Test parsing an empty extrainfo descriptors file.""" - routers = descriptors.parseExtraInfoFiles(io.BytesIO('')) + routers = descriptors.parseExtraInfoFiles(io.BytesIO(b'')) self.assertIsInstance(routers, dict) self.assertEqual(len(routers), 0) @@ -846,7 +846,7 @@ class ParseDescriptorsTests(unittest.TestCase): True when the new file is successfully created. """ filename = "bridge-descriptors" - with open(filename, 'w') as fh: + with open(filename, 'wb') as fh: fh.write(BRIDGE_SERVER_DESCRIPTOR) fh.flush() @@ -858,7 +858,7 @@ class ParseDescriptorsTests(unittest.TestCase): copy of the bad file with a specific filename format. """ filename = "bridge-descriptors" - with open(filename, 'w') as fh: + with open(filename, 'wb') as fh: fh.write(BRIDGE_SERVER_DESCRIPTOR) fh.flush()

1 0

[bridgedb/develop] Replace remaining bytes() calls
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 03494c59e13e08b0f153e46a8831ea664cabcbcc Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 15 16:55:59 2020 -0800 Replace remaining bytes() calls Python 3 converts unicode to bytes with its encode() method. This fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_https_server.py", line 442, in test_render_GET_missingTemplate page = self.captchaResource.render_GET(self.request) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/https/server.py", line 678, in render_GET return CaptchaProtectedResource.render_GET(self, request) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/https/server.py", line 525, in render_GET rendered = replaceErrorPage(request, err, 'captcha.html') File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/https/server.py", line 147, in replaceErrorPage rendered = bytes(errorMessage) builtins.TypeError: string argument without an encoding Test results changed as follows... before: FAILED (skips=109, failures=23, errors=225, successes=624) after: FAILED (skips=109, failures=24, errors=220, successes=628) --- bridgedb/distributors/https/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index d1c058c..aea19da 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -135,13 +135,13 @@ def replaceErrorPage(request, error, template_name=None, html=True): errorMessage = _("Sorry! Something went wrong with your request.") if not html: - return bytes(errorMessage) + return errorMessage.encode('utf-8') try: rendered = resource500.render(request) except Exception as err: logging.exception(err) - rendered = bytes(errorMessage) + rendered = errorMessage.encode('utf-8') return rendered @@ -1024,7 +1024,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): if format == 'plain': request.setHeader("Content-Type", "text/plain") try: - rendered = bytes('\n'.join(bridgeLines)) + rendered = '\n'.join(bridgeLines).encode('utf-8') except Exception as err: rendered = replaceErrorPage(request, err, html=False) else: @@ -1100,7 +1100,7 @@ def addWebServer(config, distributor): howto = HowtoResource() robots = static.File(os.path.join(TEMPLATE_DIR, 'robots.txt')) assets = static.File(os.path.join(TEMPLATE_DIR, 'assets/')) - keys = static.Data(bytes(strings.BRIDGEDB_OPENPGP_KEY), 'text/plain') + keys = static.Data(strings.BRIDGEDB_OPENPGP_KEY.encode('utf-8'), 'text/plain') csp = CSPResource(enabled=config.CSP_ENABLED, includeSelf=config.CSP_INCLUDE_SELF, reportViolations=config.CSP_REPORT_ONLY,

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits February 2020