tor-commits February 2020

tor-commits@lists.torproject.org

22 participants
1454 discussions

[bridgedb/master] Hashing requires bytes
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit cf61a322c65c533379a0bcf0f67b73cae59ec1da Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 17 12:28:36 2020 -0800 Hashing requires bytes Just another instance of bytes/unicode type conversion... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_https_distributor.py", line 153, in test_HTTPSDistributor_prepopulateRings_without_proxies [dist.insert(bridge) for bridge in self.bridges] File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_https_distributor.py", line 153, in <listcomp> [dist.insert(bridge) for bridge in self.bridges] File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/https/distributor.py", line 278, in insert self.hashring.insert(bridge) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/Bridges.py", line 606, in insert logging.debug("Inserting %s into hashring..." % bridge) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/bridges.py", line 1041, in __str__ fingerprint = hashlib.sha1(fingerprint).hexdigest().upper() builtins.TypeError: Unicode-objects must be encoded before hashing Test results changed as follows... before: FAILED (skips=114, failures=14, errors=157, successes=700) after: FAILED (skips=114, failures=14, errors=149, successes=708) --- bridgedb/bridges.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgedb/bridges.py b/bridgedb/bridges.py index a611c47..827a156 100644 --- a/bridgedb/bridges.py +++ b/bridgedb/bridges.py @@ -1037,7 +1037,7 @@ class Bridge(BridgeBackwardsCompatibility): if safelog.safe_logging: prefix = '$$' if fingerprint: - fingerprint = hashlib.sha1(fingerprint).hexdigest().upper() + fingerprint = hashlib.sha1(fingerprint.encode('utf-8')).hexdigest().upper() if not fingerprint: fingerprint = '0' * 40

1 0

[bridgedb/master] None type cannot be compared to integers
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit e71c76a87ff509ea2d19ffaec06ae6e7d8a7a7c7 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 17 13:40:11 2020 -0800 None type cannot be compared to integers Python changed its behavior with regard to how integers are compared with None... Python 2: >>> bool(not None > 0) True Python3: >>> bool(not None > 0) Traceback (most recent call last): File "<stdin>", line 1, in <module> TypeError: unorderable types: NoneType() > int() This fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_schedule.py", line 110, in test_ScheduledInterval_init_noargs sched = self.sched() File "/home/atagar/Desktop/tor/bridgedb/bridgedb/schedule.py", line 187, in __init__ self._setIntervalCount(count) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/schedule.py", line 208, in _setIntervalCount % (self.__class__.__name__, count)) bridgedb.schedule.UnknownInterval: ScheduledInterval.intervalCount: None ist not an integer. Test results changed as follows... before: FAILED (skips=114, failures=15, errors=117, successes=739) after: FAILED (skips=114, failures=15, errors=116, successes=740) --- bridgedb/schedule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgedb/schedule.py b/bridgedb/schedule.py index e962c39..8a7ca0f 100644 --- a/bridgedb/schedule.py +++ b/bridgedb/schedule.py @@ -200,7 +200,7 @@ class ScheduledInterval(Unscheduled): :raises UnknownInterval: if the specified **count** was invalid. """ try: - if not count > 0: + if count is None or not count > 0: count = 1 count = int(count) except (TypeError, ValueError):

1 0

[bridgedb/master] Hash bytes within Storage.py
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit f76eb054d4dcad4a8677ea163054b99161cceb1c Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 17 13:32:51 2020 -0800 Hash bytes within Storage.py Another hashing byte/unicode fix... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_email_distributor.py", line 192, in test_EmailDistributor_cleanDatabase self.assertEqual(len(dist.getBridges(bridgeRequest, 1)), 3) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/email/distributor.py", line 154, in getBridges wasWarned = db.getWarnedEmail(bridgeRequest.client) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/Storage.py", line 264, in getWarnedEmail addr = hashlib.sha1(addr).hexdigest() builtins.TypeError: Unicode-objects must be encoded before hashing Test results changed as follows... before: FAILED (skips=114, failures=15, errors=122, successes=734) after: FAILED (skips=114, failures=15, errors=117, successes=739) --- bridgedb/Storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bridgedb/Storage.py b/bridgedb/Storage.py index 1f81052..cfd60bb 100644 --- a/bridgedb/Storage.py +++ b/bridgedb/Storage.py @@ -211,7 +211,7 @@ class Database(object): cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,)) def getEmailTime(self, addr): - addr = hashlib.sha1(addr).hexdigest() + addr = hashlib.sha1(addr.encode('utf-8')).hexdigest() cur = self._cur cur.execute("SELECT when_mailed FROM EmailedBridges WHERE email = ?", (addr,)) v = cur.fetchone() @@ -220,7 +220,7 @@ class Database(object): return strToTime(v[0]) def setEmailTime(self, addr, whenMailed): - addr = hashlib.sha1(addr).hexdigest() + addr = hashlib.sha1(addr.encode('utf-8')).hexdigest() cur = self._cur t = timeToStr(whenMailed) cur.execute("INSERT OR REPLACE INTO EmailedBridges " @@ -261,7 +261,7 @@ class Database(object): (distributor, hex_key)) def getWarnedEmail(self, addr): - addr = hashlib.sha1(addr).hexdigest() + addr = hashlib.sha1(addr.encode('utf-8')).hexdigest() cur = self._cur cur.execute("SELECT * FROM WarnedEmails WHERE email = ?", (addr,)) v = cur.fetchone() @@ -270,7 +270,7 @@ class Database(object): return True def setWarnedEmail(self, addr, warned=True, whenWarned=time.time()): - addr = hashlib.sha1(addr).hexdigest() + addr = hashlib.sha1(addr.encode('utf-8')).hexdigest() t = timeToStr(whenWarned) cur = self._cur if warned == True:

1 0

[bridgedb/master] Fix persistence tests
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit e893c41565345720997a6195a498e37a6a455039 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 14 13:08:03 2020 -0800 Fix persistence tests Nothing too interesting. Few files needed to be opened in bytes mode and a deprecated os function replaced... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_persistent.py", line 53, in setUp fakeArgs = ['-c', os.path.join(os.getcwdu(), '..', 'bridgedb.conf')] builtins.AttributeError: module 'os' has no attribute 'getcwdu' Test results changed as follows... before: FAILED (skips=109, failures=24, errors=349, successes=498) after: FAILED (skips=109, failures=22, errors=318, successes=531) --- bridgedb/parse/options.py | 2 +- bridgedb/persistent.py | 11 ++++++++--- bridgedb/test/test_persistent.py | 2 +- bridgedb/test/test_persistentSaveAndLoad.py | 6 +++--- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/bridgedb/parse/options.py b/bridgedb/parse/options.py index 0eeb79e..c32ada3 100644 --- a/bridgedb/parse/options.py +++ b/bridgedb/parse/options.py @@ -206,7 +206,7 @@ class BaseOptions(usage.Options): if rundir is not None: gRundir = os.path.abspath(os.path.expanduser(rundir)) else: - gRundir = os.getcwdu() + gRundir = os.getcwd() setRundir(gRundir) if not os.path.isdir(gRundir): # pragma: no cover diff --git a/bridgedb/persistent.py b/bridgedb/persistent.py index 89d3dd4..3d52ec5 100644 --- a/bridgedb/persistent.py +++ b/bridgedb/persistent.py @@ -128,6 +128,11 @@ class State(jelly.Jellyable): :param string statefile: The filename of the statefile. """ + + if filename is None: + self._statefile = None + return + filename = os.path.abspath(os.path.expanduser(filename)) logging.debug("Setting statefile to '%s'" % filename) self._statefile = filename @@ -173,8 +178,8 @@ class State(jelly.Jellyable): err = '' try: - if isinstance(statefile, str): - fh = open(statefile, 'r') + if isinstance(statefile, (str, bytes)): + fh = open(statefile, 'rb') elif not statefile.closed: fh = statefile except (IOError, OSError) as error: # pragma: no cover @@ -209,7 +214,7 @@ class State(jelly.Jellyable): fh = None try: - fh = open(statefile, 'w') + fh = open(statefile, 'wb') except (IOError, OSError) as error: # pragma: no cover logging.warn("Error writing state file to '%s': %s" % (statefile, error)) diff --git a/bridgedb/test/test_persistent.py b/bridgedb/test/test_persistent.py index 269cc86..21aa67f 100644 --- a/bridgedb/test/test_persistent.py +++ b/bridgedb/test/test_persistent.py @@ -50,7 +50,7 @@ class StateTest(unittest.TestCase): exec(compiled, configuration) config = persistent.Conf(**configuration) - fakeArgs = ['-c', os.path.join(os.getcwdu(), '..', 'bridgedb.conf')] + fakeArgs = ['-c', os.path.join(os.getcwd(), '..', 'bridgedb.conf')] options = MainOptions() options.parseOptions(fakeArgs) diff --git a/bridgedb/test/test_persistentSaveAndLoad.py b/bridgedb/test/test_persistentSaveAndLoad.py index d01a536..bcdcdd7 100644 --- a/bridgedb/test/test_persistentSaveAndLoad.py +++ b/bridgedb/test/test_persistentSaveAndLoad.py @@ -58,8 +58,8 @@ class StateSaveAndLoadTests(unittest.TestCase): self.assertNotIdentical(self.state, loadedState) self.assertNotEqual(self.state, loadedState) - self.assertEqual(self.state.__dict__.keys().sort(), - loadedState.__dict__.keys().sort()) + self.assertEqual(list(self.state.__dict__.keys()).sort(), + list(loadedState.__dict__.keys()).sort()) def savedStateAssertions(self, savedStatefile=None): self.assertTrue(os.path.isfile(str(self.state.statefile))) @@ -148,7 +148,7 @@ class StateSaveAndLoadTests(unittest.TestCase): self.state.load, 'quux.state') def test_load_with_statefile_opened(self): - fh = open('quux.state', 'w+') + fh = open('quux.state', 'wb+') self.assertRaises(persistent.MissingState, self.state.load, fh) fh.close()

1 0

[bridgedb/master] Refactor log messages.
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit b170b03a3a7909a93b764c8c8c69ec4b843d0c2e Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Jan 14 09:06:02 2020 +0100 Refactor log messages. These log messages were misleading because they were called from an incorrect place in the code. --- bridgedb/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridgedb/main.py b/bridgedb/main.py index 7c2df6d..7776750 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -423,17 +423,17 @@ def run(options, reactor=reactor): logging.info("Reloading decoy bridges...") antibot.loadDecoyBridges(config.DECOY_BRIDGES_FILE) - logging.info("Reparsing bridge descriptors...") (hashring, emailDistributorTmp, ipDistributorTmp, moatDistributorTmp) = createBridgeRings(cfg, proxies, key) - logging.info("Bridges loaded: %d" % len(hashring)) # Initialize our DB. bridgedb.Storage.initializeDBLock() bridgedb.Storage.setDBFilename(cfg.DB_FILE + ".sqlite") + logging.info("Reparsing bridge descriptors...") load(state, hashring, clear=False) + logging.info("Bridges loaded: %d" % len(hashring)) if emailDistributorTmp is not None: emailDistributorTmp.prepopulateRings() # create default rings

1 0

[bridgedb/master] Convert generators to lists
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 75f876ee89226977d358c2dfc42b931ac94f3430 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 17 12:20:30 2020 -0800 Convert generators to lists Python 3 changes several builtins to provide generators rather than lists. This is more efficient, but means we need to convert to a list to be indexable. There will likely be more instances down the road, but fixing all the instances that are immediately evedant in the test output. This fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_proxy.py", line 355, in test_ProxySet_addProxies_tuple_individual_tags self.assertEquals(len(self.proxyList), len(self.proxies) + len(extraProxies)) builtins.TypeError: object of type 'zip' has no len() Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_parse_headers.py", line 47, in test_addNonLocalizedVariant langs = headers.parseAcceptLanguage(header) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/parse/headers.py", line 70, in parseAcceptLanguage placement = langs.index(insertAfter[0]) + 1 builtins.TypeError: 'filter' object is not subscriptable Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_bridges.py", line 895, in setUp self._parseAllDescriptorFiles() File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_bridges.py", line 852, in _parseAllDescriptorFiles self._extrainfoFile).values()[0] builtins.TypeError: 'dict_values' object does not support indexing Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_bridges.py", line 315, in test_integration_getConfigLine_scramblesuit_withFingerprint transport='scramblesuit') File "/home/atagar/Desktop/tor/bridgedb/bridgedb/bridges.py", line 847, in getConfigLine bridgeLine = self.getBridgeLine(bridgeRequest, includeFingerprint) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/bridges.py", line 1362, in getBridgeLine pt = self._getTransportForRequest(bridgeRequest) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/bridges.py", line 1196, in _getTransportForRequest return transports[position % len(unblocked)] builtins.TypeError: 'filter' object is not subscriptable Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_Bridges.py", line 78, in test_filterDistinctSubnets_random_bridges chosen = self.ring.bridges.keys()[:3] builtins.TypeError: 'dict_keys' object is not subscriptable Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_Bridges.py", line 65, in test_filterDistinctSubnets chosen = self.ring.bridges.keys()[:10] builtins.TypeError: 'dict_keys' object is not subscriptable Test results changed as follows... before: FAILED (skips=114, failures=13, errors=191, successes=663) after: FAILED (skips=114, failures=14, errors=158, successes=695) --- bridgedb/bridges.py | 2 +- bridgedb/parse/headers.py | 4 ++-- bridgedb/test/test_Bridges.py | 4 ++-- bridgedb/test/test_bridges.py | 8 ++++---- bridgedb/test/test_proxy.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bridgedb/bridges.py b/bridgedb/bridges.py index ca1330b..a611c47 100644 --- a/bridgedb/bridges.py +++ b/bridgedb/bridges.py @@ -1177,7 +1177,7 @@ class Bridge(BridgeBackwardsCompatibility): # their ``methodname`` matches the requested transport: transports = filter(lambda pt: pt.methodname == desired, self.transports) # Filter again for whichever of IPv4 or IPv6 was requested: - transports = filter(lambda pt: pt.address.version == ipVersion, transports) + transports = list(filter(lambda pt: pt.address.version == ipVersion, transports)) if not transports: raise PluggableTransportUnavailable( diff --git a/bridgedb/parse/headers.py b/bridgedb/parse/headers.py index 5c3597a..fa8cbf2 100644 --- a/bridgedb/parse/headers.py +++ b/bridgedb/parse/headers.py @@ -64,8 +64,8 @@ def parseAcceptLanguage(header): for only in langsOnly: if only not in langs: # Add the fallback after the other languages like it: - insertAfter = filter(lambda x: x.startswith(only), - [x for x in langs]) + insertAfter = list(filter(lambda x: x.startswith(only), + [x for x in langs])) if insertAfter: placement = langs.index(insertAfter[0]) + 1 langs.insert(placement, only) diff --git a/bridgedb/test/test_Bridges.py b/bridgedb/test/test_Bridges.py index 5991952..19ab09c 100644 --- a/bridgedb/test/test_Bridges.py +++ b/bridgedb/test/test_Bridges.py @@ -62,7 +62,7 @@ class BridgeRingTests(unittest.TestCase): """ self.addBridgesFromSameSubnet() - chosen = self.ring.bridges.keys()[:10] + chosen = list(self.ring.bridges.keys())[:10] bridges = self.ring.filterDistinctSubnets(chosen) # Since they're all in the same /16, we should only get one @@ -75,7 +75,7 @@ class BridgeRingTests(unittest.TestCase): """ self.addRandomBridges() - chosen = self.ring.bridges.keys()[:3] + chosen = list(self.ring.bridges.keys())[:3] bridges = self.ring.filterDistinctSubnets(chosen) self.assertGreaterEqual(len(bridges), 1) diff --git a/bridgedb/test/test_bridges.py b/bridgedb/test/test_bridges.py index f2c2876..cf0ee23 100644 --- a/bridgedb/test/test_bridges.py +++ b/bridgedb/test/test_bridges.py @@ -848,10 +848,10 @@ class BridgeTests(unittest.TestCase): self._networkstatusFile)[0] self.serverdescriptor = descriptors.parseServerDescriptorsFile( self._serverDescriptorFile)[0] - self.extrainfo = descriptors.parseExtraInfoFiles( - self._extrainfoFile).values()[0] - self.extrainfoNew = descriptors.parseExtraInfoFiles( - self._extrainfoNewFile).values()[0] + self.extrainfo = list(descriptors.parseExtraInfoFiles( + self._extrainfoFile).values())[0] + self.extrainfoNew = list(descriptors.parseExtraInfoFiles( + self._extrainfoNewFile).values())[0] def _writeNetworkstatus(self, networkstatus): with open(self._networkstatusFile, 'w') as fh: diff --git a/bridgedb/test/test_proxy.py b/bridgedb/test/test_proxy.py index 74290bc..69b2b99 100644 --- a/bridgedb/test/test_proxy.py +++ b/bridgedb/test/test_proxy.py @@ -350,7 +350,7 @@ class ProxySetUnittests(unittest.TestCase): whatever tags we want. """ tags = ['foo', 'bar', 'baz'] - extraProxies = zip(self.moarProxies, tags) + extraProxies = list(zip(self.moarProxies, tags)) self.proxyList.addProxies(extraProxies) self.assertEquals(len(self.proxyList), len(self.proxies) + len(extraProxies)) self.assertIn(extraProxies[0][0], self.proxyList)

1 0

[bridgedb/master] Fix most autoresponder tests
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 5aa0adcbb2725011ecd7fd11eb94d0bc2c30ab42 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 14 16:12:15 2020 -0800 Fix most autoresponder tests Low hanging fruit to get all the autoresponder tests aside from 'getMailFrom_plus_address' passing. This last test involves API differences between the rfc822 and email modules. Errors this resolve are the usual byte/unicode differences... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_email_autoresponder.py", line 256, in test_EmailResponse_write response.write(self.body) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/email/autoresponder.py", line 273, in write self.mailfile.write(line.encode('utf8')) builtins.TypeError: string argument expected, got 'bytes' Indexing into an iterable... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_email_autoresponder.py", line 50, in setUp self.ctx = _createMailServerContext(self.config) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/email_helpers.py", line 119, in _createMailServerContext context = MailServerContext(config, distributor, Unscheduled()) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/email/server.py", line 142, in __init__ self.gpg, self.gpgSignFunc = initializeGnuPG(config) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/crypto.py", line 318, in initializeGnuPG for sub in primaryPK[0]['subkeys']: builtins.TypeError: 'filter' object is not subscriptable ... and double encoding... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_email_autoresponder.py", line 251, in test_EmailResponse_read_three_bytes self.assertEqual(contents, self.body[:3]) File "/usr/local/lib/python3.5/dist-packages/twisted/trial/_synctest.py", line 432, in assertEqual super(_Assertions, self).assertEqual(first, second, msg) File "/usr/lib/python3.5/unittest/case.py", line 820, in assertEqual assertion_func(first, second, msg=msg) File "/usr/lib/python3.5/unittest/case.py", line 1193, in assertMultiLineEqual self.fail(self._formatMessage(msg, standardMsg)) twisted.trial.unittest.FailTest: "b'Peo'" != 'Peo' - b'Peo' + Peo Test results changed as follows... before: FAILED (skips=109, failures=21, errors=283, successes=567) after: FAILED (skips=109, failures=24, errors=225, successes=623) --- bridgedb/crypto.py | 2 +- bridgedb/distributors/email/autoresponder.py | 24 ++++++++++++++++-------- bridgedb/distributors/email/server.py | 3 ++- bridgedb/test/test_email_autoresponder.py | 8 ++++---- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/bridgedb/crypto.py b/bridgedb/crypto.py index bd3d783..5b1912f 100644 --- a/bridgedb/crypto.py +++ b/bridgedb/crypto.py @@ -315,7 +315,7 @@ def initializeGnuPG(config): if primarySK and primaryPK: logging.info("Found GnuPG primary key with fingerprint: %s" % primary) - for sub in primaryPK[0]['subkeys']: + for sub in list(primaryPK)[0]['subkeys']: logging.info(" Subkey: %s Usage: %s" % (sub[0], sub[1].upper())) else: logging.warn("GnuPG key %s could not be found in %s!" % (primary, gpg.secring)) diff --git a/bridgedb/distributors/email/autoresponder.py b/bridgedb/distributors/email/autoresponder.py index bc1c7b9..1d2b618 100644 --- a/bridgedb/distributors/email/autoresponder.py +++ b/bridgedb/distributors/email/autoresponder.py @@ -263,6 +263,9 @@ class EmailResponse(object): :param str line: Something to append into the :data:`mailfile`. """ + + line = line.decode('utf-8') if isinstance(line, bytes) else line + if line.find('\r\n') != -1: # If **line** contains newlines, send it to :meth:`writelines` to # break it up so that we can replace them: @@ -270,7 +273,7 @@ class EmailResponse(object): self.writelines(line) else: line += self.delimiter - self.mailfile.write(line.encode('utf8')) + self.mailfile.write(line) self.mailfile.flush() def writelines(self, lines): @@ -283,7 +286,8 @@ class EmailResponse(object): :type lines: :any:`str` or :any:`list` :param lines: The lines to write to the :attr:`mailfile`. """ - if isinstance(lines, str): + if isinstance(lines, (str, bytes)): + lines = lines.decode('utf-8') if isinstance(lines, bytes) else lines lines = lines.replace('\r\n', '\n') for ln in lines.split('\n'): self.write(ln) @@ -310,20 +314,24 @@ class EmailResponse(object): :kwargs: If given, the key will become the name of the header, and the value will become the contents of that header. """ + + fromAddress = fromAddress.decode('utf-8') if isinstance(fromAddress, bytes) else fromAddress + toAddress = toAddress.decode('utf-8') if isinstance(toAddress, bytes) else toAddress + self.write("From: %s" % fromAddress) self.write("To: %s" % toAddress) if includeMessageID: - self.write("Message-ID: %s" % smtp.messageid().encode('utf-8')) + self.write("Message-ID: %s" % smtp.messageid()) if inReplyTo: - self.write("In-Reply-To: %s" % inReplyTo.encode('utf-8')) - self.write("Content-Type: %s" % contentType.encode('utf-8')) - self.write("Date: %s" % smtp.rfc822date().encode('utf-8')) + self.write("In-Reply-To: %s" % inReplyTo) + self.write("Content-Type: %s" % contentType) + self.write("Date: %s" % smtp.rfc822date().decode('utf-8')) if not subject: subject = '[no subject]' if not subject.lower().startswith('re'): subject = "Re: " + subject - self.write("Subject: %s" % subject.encode('utf-8')) + self.write("Subject: %s" % subject) if kwargs: for headerName, headerValue in kwargs.items(): @@ -331,7 +339,7 @@ class EmailResponse(object): headerName = headerName.replace(' ', '-') headerName = headerName.replace('_', '-') header = "%s: %s" % (headerName, headerValue) - self.write(header.encode('utf-8')) + self.write(header) # The first blank line designates that the headers have ended: self.write(self.delimiter) diff --git a/bridgedb/distributors/email/server.py b/bridgedb/distributors/email/server.py index c1ce56f..0f96812 100644 --- a/bridgedb/distributors/email/server.py +++ b/bridgedb/distributors/email/server.py @@ -253,7 +253,8 @@ class SMTPMessage(object): """ rawMessage = io.StringIO() for line in self.lines: - rawMessage.writelines(line.decode('utf8') + '\n') + line = line.decode('utf-8') if isinstance(line, bytes) else line + rawMessage.writelines(line + '\n') rawMessage.seek(0) return rfc822.Message(rawMessage) diff --git a/bridgedb/test/test_email_autoresponder.py b/bridgedb/test/test_email_autoresponder.py index f59e5ef..c8e9624 100644 --- a/bridgedb/test/test_email_autoresponder.py +++ b/bridgedb/test/test_email_autoresponder.py @@ -235,7 +235,7 @@ a ball of timey-wimey, wibbly-warbly... stuff.""" response = autoresponder.EmailResponse() response.write(self.body) response.rewind() - contents = str(response.read()).replace('\x00', '') + contents = response.read().replace(b'\x00', b'').decode('utf-8') # The newlines in the email body should have been replaced with # ``EmailResponse.delimiter``. delimited = self.body.replace('\n', response.delimiter) \ @@ -247,7 +247,7 @@ a ball of timey-wimey, wibbly-warbly... stuff.""" response = autoresponder.EmailResponse() response.write(self.body) response.rewind() - contents = str(response.read(3)).replace('\x00', '') + contents = response.read(3).replace(b'\x00', b'').decode('utf-8') self.assertEqual(contents, self.body[:3]) def test_EmailResponse_write(self): @@ -370,7 +370,7 @@ class SMTPAutoresponderTests(unittest.TestCase): """ self._getIncomingLines() ours = Address(self.context.fromAddr) - plus = '@'.join([ours.local + '+zh_cn', ours.domain]) + plus = '@'.join([ours.local.decode('utf-8') + '+zh_cn', ours.domain.decode('utf-8')]) self.message.lines[1] = 'To: {0}'.format(plus) self._setUpResponder() recipient = str(self.responder.getMailFrom()) @@ -383,7 +383,7 @@ class SMTPAutoresponderTests(unittest.TestCase): """ self._getIncomingLines() ours = Address(self.context.fromAddr) - plus = '@'.join(['get' + ours.local + '+zh_cn', ours.domain]) + plus = '@'.join(['get' + ours.local.decode('utf-8') + '+zh_cn', ours.domain.decode('utf-8')]) self.message.lines[1] = 'To: {0}'.format(plus) self._setUpResponder() recipient = str(self.responder.getMailFrom())

1 0

[bridgedb/master] Fix https_server tests
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit d604bc5a47b646d154d45deeaba6a2cf6f1e8424 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jan 16 17:40:04 2020 -0800 Fix https_server tests Ooph, this was more difficult than it should've been. Isis relied on catch-alls for errors she found difficult, requiring me to repeatedly bypass them to troubleshoot. test_https_server.py finally passes under python 3. Test results changed as follows... before: FAILED (skips=109, failures=19, errors=221, successes=632) after: FAILED (skips=114, failures=14, errors=198, successes=655) --- bridgedb/distributors/email/templates.py | 1 + bridgedb/distributors/https/server.py | 22 +++++++++++----------- bridgedb/test/test_https_server.py | 28 ++++++++++++++++++++++++---- bridgedb/util.py | 17 ++++++++--------- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/bridgedb/distributors/email/templates.py b/bridgedb/distributors/email/templates.py index 85dd105..188f052 100644 --- a/bridgedb/distributors/email/templates.py +++ b/bridgedb/distributors/email/templates.py @@ -59,6 +59,7 @@ def addCommands(template): def addGreeting(template, clientName=None, welcome=False): greeting = "" + clientName = clientName.decode('utf-8') if isinstance(clientName, bytes) else clientName if not clientName: greeting = template.gettext(strings.EMAIL_MISC_TEXT[7]) diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index aea19da..b33065f 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -135,15 +135,15 @@ def replaceErrorPage(request, error, template_name=None, html=True): errorMessage = _("Sorry! Something went wrong with your request.") if not html: - return errorMessage.encode('utf-8') + return errorMessage try: rendered = resource500.render(request) except Exception as err: logging.exception(err) - rendered = errorMessage.encode('utf-8') + rendered = errorMessage - return rendered + return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered def redirectMaliciousRequest(request): @@ -343,7 +343,7 @@ class ErrorResource(CSPResource): except Exception as err: rendered = replaceErrorPage(request, err, html=False) - return rendered + return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered render_POST = render_GET @@ -392,7 +392,7 @@ class TranslatedTemplateResource(CustomErrorHandlingResource, CSPResource): except Exception as err: # pragma: no cover rendered = replaceErrorPage(request, err) request.setHeader("Content-Type", "text/html; charset=utf-8") - return rendered + return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered render_POST = render_GET @@ -509,7 +509,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): langs = translations.getLocaleFromHTTPRequest(request) rtl = translations.usingRTLLang(langs) # TODO: this does not work for versions of IE < 8.0 - imgstr = 'data:image/jpeg;base64,%s' % base64.b64encode(image) + imgstr = b'data:image/jpeg;base64,%s' % base64.b64encode(image.encode('utf-8')) template = lookup.get_template('captcha.html') rendered = template.render(strings, getSortedLangList(), @@ -522,7 +522,7 @@ class CaptchaProtectedResource(CustomErrorHandlingResource, CSPResource): rendered = replaceErrorPage(request, err, 'captcha.html') request.setHeader("Content-Type", "text/html; charset=utf-8") - return rendered + return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered def render_POST(self, request): """Process a client's CAPTCHA solution. @@ -730,7 +730,7 @@ class ReCaptchaProtectedResource(CaptchaProtectedResource): rendered = redirectTo(request.uri, request) try: - request.write(rendered) + request.write(rendered.encode('utf-8') if isinstance(rendered, str) else rendered) request.finish() except Exception as err: # pragma: no cover logging.exception(err) @@ -919,7 +919,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): logging.exception(err) response = self.renderAnswer(request) - return response + return response.decode('utf-8') if isinstance(response, bytes) else response def getClientIP(self, request): """Get the client's IP address from the ``'X-Forwarded-For:'`` @@ -1048,7 +1048,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): except Exception as err: rendered = replaceErrorPage(request, err) - return rendered + return rendered.decode('utf-8') if isinstance(rendered, bytes) else rendered def addWebServer(config, distributor): @@ -1135,7 +1135,7 @@ def addWebServer(config, distributor): if config.HTTPS_ROTATION_PERIOD: count, period = config.HTTPS_ROTATION_PERIOD.split() - sched = ScheduledInterval(count, period) + sched = ScheduledInterval(int(count), period) else: sched = Unscheduled() diff --git a/bridgedb/test/test_https_server.py b/bridgedb/test/test_https_server.py index 3c799ba..a25a99d 100644 --- a/bridgedb/test/test_https_server.py +++ b/bridgedb/test/test_https_server.py @@ -27,7 +27,7 @@ from twisted.trial import unittest from twisted.web.resource import Resource from twisted.web.test import requesthelper -from bridgedb import translations +from bridgedb import _langs, translations from bridgedb.distributors.https import server from bridgedb.schedule import ScheduledInterval @@ -218,6 +218,10 @@ class IndexResourceTests(unittest.TestCase): def test_IndexResource_render_GET_lang_ta(self): """renderGet() with ?lang=ta should return the index page in Tamil.""" + + if 'ta' not in _langs.get_langs(): + self.skipTest("'ta' language unsupported") + request = DummyRequest([self.pagename]) request.method = b'GET' request.addArg('lang', 'ta') @@ -243,6 +247,10 @@ class HowtoResourceTests(unittest.TestCase): def test_HowtoResource_render_GET_lang_ru(self): """renderGet() with ?lang=ru should return the howto page in Russian.""" + + if 'ru' not in _langs.get_langs(): + self.skipTest("'ru' language unsupported") + request = DummyRequest([self.pagename]) request.method = b'GET' request.addArg('lang', 'ru') @@ -522,7 +530,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): def testCB(request): """Check the ``Request`` returned from ``_renderDeferred``.""" self.assertIsInstance(request, DummyRequest) - soup = BeautifulSoup(b''.join(request.written)).find('meta')['http-equiv'] + soup = BeautifulSoup(b''.join(request.written)).find(b'meta')['http-equiv'] self.assertEqual(soup, 'refresh') d = task.deferLater(reactor, 0, lambda x: x, (False, self.request)) @@ -541,7 +549,7 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): """Check the ``Request`` returned from ``_renderDeferred``.""" self.assertIsInstance(request, DummyRequest) html = b''.join(request.written) - self.assertSubstring('Uh oh, spaghettios!', html) + self.assertSubstring(b'Uh oh, spaghettios!', html) d = task.deferLater(reactor, 0, lambda x: x, (True, self.request)) d.addCallback(self.captchaResource._renderDeferred) @@ -668,7 +676,7 @@ class BridgesResourceTests(unittest.TestCase): # The bridge lines are contained in a <div class='bridges'> tag: soup = BeautifulSoup(page) well = soup.find('div', {'class': 'bridge-lines'}) - content = well.renderContents().strip() + content = well.renderContents().decode('utf-8').strip() lines = content.splitlines() bridges = [] @@ -794,6 +802,10 @@ class BridgesResourceTests(unittest.TestCase): def test_render_GET_RTLlang(self): """Test rendering a request for plain bridges in Arabic.""" + + if 'ar' not in _langs.get_langs(): + self.skipTest("'ar' language unsupported") + self.useBenignBridges() request = DummyRequest([b"bridges?transport=obfs3"]) @@ -816,6 +828,10 @@ class BridgesResourceTests(unittest.TestCase): def test_render_GET_RTLlang_obfs3(self): """Test rendering a request for obfs3 bridges in Farsi.""" + + if 'fa' not in _langs.get_langs(): + self.skipTest("'ar' language unsupported") + self.useBenignBridges() request = DummyRequest([b"bridges?transport=obfs3"]) @@ -915,6 +931,10 @@ class OptionsResourceTests(unittest.TestCase): def test_render_GET_RTLlang(self): """Test rendering a request for obfs3 bridges in Hebrew.""" + + if 'he' not in _langs.get_langs(): + self.skipTest("'ar' language unsupported") + request = DummyRequest(["bridges?transport=obfs3"]) request.method = b'GET' request.getClientIP = lambda: '3.3.3.3' diff --git a/bridgedb/util.py b/bridgedb/util.py index 7b68831..db8e749 100644 --- a/bridgedb/util.py +++ b/bridgedb/util.py @@ -18,6 +18,7 @@ import logging import logging.config import logging.handlers import os +import re import time from twisted.python import components @@ -260,17 +261,15 @@ def replaceControlChars(text, replacement=None, encoding="utf-8"): :rtype: str :returns: The sanitized **text**. """ - escaped = bytearray() - for byte in bytearray(text, encoding): - if byte in list(range(0, 32)) + [92, 127]: - if replacement: - byte = replacement - else: - continue - escaped += bytearray([byte]) + if replacement is None: + replacement = '' + + # the following replaces characters 0-31, 92, and 127 + + text = text.decode(encoding) if isinstance(text, bytes) else text + return re.sub(r'[\x00-\x1f\x5c\x7f]', '', text) - return str(escaped) def registerAdapter(adapter, adapted, interface): """Register a Zope interface adapter for global use.

1 0

[bridgedb/master] Replace remaining bytes() calls
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 03494c59e13e08b0f153e46a8831ea664cabcbcc Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 15 16:55:59 2020 -0800 Replace remaining bytes() calls Python 3 converts unicode to bytes with its encode() method. This fixes... Traceback (most recent call last): File "/home/atagar/Desktop/tor/bridgedb/bridgedb/test/test_https_server.py", line 442, in test_render_GET_missingTemplate page = self.captchaResource.render_GET(self.request) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/https/server.py", line 678, in render_GET return CaptchaProtectedResource.render_GET(self, request) File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/https/server.py", line 525, in render_GET rendered = replaceErrorPage(request, err, 'captcha.html') File "/home/atagar/Desktop/tor/bridgedb/bridgedb/distributors/https/server.py", line 147, in replaceErrorPage rendered = bytes(errorMessage) builtins.TypeError: string argument without an encoding Test results changed as follows... before: FAILED (skips=109, failures=23, errors=225, successes=624) after: FAILED (skips=109, failures=24, errors=220, successes=628) --- bridgedb/distributors/https/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index d1c058c..aea19da 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -135,13 +135,13 @@ def replaceErrorPage(request, error, template_name=None, html=True): errorMessage = _("Sorry! Something went wrong with your request.") if not html: - return bytes(errorMessage) + return errorMessage.encode('utf-8') try: rendered = resource500.render(request) except Exception as err: logging.exception(err) - rendered = bytes(errorMessage) + rendered = errorMessage.encode('utf-8') return rendered @@ -1024,7 +1024,7 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): if format == 'plain': request.setHeader("Content-Type", "text/plain") try: - rendered = bytes('\n'.join(bridgeLines)) + rendered = '\n'.join(bridgeLines).encode('utf-8') except Exception as err: rendered = replaceErrorPage(request, err, html=False) else: @@ -1100,7 +1100,7 @@ def addWebServer(config, distributor): howto = HowtoResource() robots = static.File(os.path.join(TEMPLATE_DIR, 'robots.txt')) assets = static.File(os.path.join(TEMPLATE_DIR, 'assets/')) - keys = static.Data(bytes(strings.BRIDGEDB_OPENPGP_KEY), 'text/plain') + keys = static.Data(strings.BRIDGEDB_OPENPGP_KEY.encode('utf-8'), 'text/plain') csp = CSPResource(enabled=config.CSP_ENABLED, includeSelf=config.CSP_INCLUDE_SELF, reportViolations=config.CSP_REPORT_ONLY,

1 0

[bridgedb/master] Fix descriptor parsing tests
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit dc07e4e34d8b5f31f015362ee239f07c0b203a44 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 13 19:07:29 2020 -0800 Fix descriptor parsing tests Mostly just simple byte normalizaiton issues. Test results changed as follows... before: FAILED (skips=109, failures=24, errors=364, successes=483) after: FAILED (skips=109, failures=23, errors=349, successes=499) --- bridgedb/parse/descriptors.py | 24 +------ bridgedb/test/test_parse_descriptors.py | 110 ++++++++++++++++---------------- 2 files changed, 56 insertions(+), 78 deletions(-) diff --git a/bridgedb/parse/descriptors.py b/bridgedb/parse/descriptors.py index ff32226..f6dee83 100644 --- a/bridgedb/parse/descriptors.py +++ b/bridgedb/parse/descriptors.py @@ -164,28 +164,6 @@ def parseServerDescriptorsFile(filename, validate=True): return list(document) -def __cmp_published__(x, y): - """A custom ``cmp()`` which sorts descriptors by published date. - - :rtype: int - :returns: Return negative if x<y, zero if x==y, positive if x>y. - """ - if x.published < y.published: - return -1 - elif x.published == y.published: - # This *shouldn't* happen. It would mean that two descriptors for - # the same router had the same timestamps, probably meaning there - # is a severely-messed up OR implementation out there. Let's log - # its fingerprint (no matter what!) so that we can look up its - # ``platform`` line in its server-descriptor and tell whoever - # wrote that code that they're probably (D)DOSing the Tor network. - logging.warn(("Duplicate descriptor with identical timestamp (%s) " - "for bridge %s with fingerprint %s !") % - (x.published, x.nickname, x.fingerprint)) - return 0 - elif x.published > y.published: - return 1 - def deduplicate(descriptors, statistics=False): """Deduplicate some descriptors, returning only the newest for each router. @@ -215,7 +193,7 @@ def deduplicate(descriptors, statistics=False): duplicates[fingerprint] = [descriptor,] for fingerprint, dupes in duplicates.items(): - dupes.sort(cmp=__cmp_published__) + dupes.sort(key=lambda x: x.published) first = dupes.pop() newest[fingerprint] = first duplicates[fingerprint] = dupes diff --git a/bridgedb/test/test_parse_descriptors.py b/bridgedb/test/test_parse_descriptors.py index 7fd7582..eb86a0e 100644 --- a/bridgedb/test/test_parse_descriptors.py +++ b/bridgedb/test/test_parse_descriptors.py @@ -36,7 +36,7 @@ else: from bridgedb.test.util import Benchmarker -BRIDGE_NETWORKSTATUS_0 = '''\ +BRIDGE_NETWORKSTATUS_0 = b'''\ r MiserLandfalls 4IsyTSCtChPhFPAnq5rD8yymlqA /GMC4lz8RXT/62v6kZNdmzSmopk 2014-11-04 06:23:22 2.215.61.223 4056 0 a [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055 s Fast Guard Running Stable Valid @@ -44,7 +44,7 @@ w Bandwidth=1678904 p reject 1-65535 ''' -BRIDGE_NETWORKSTATUS_1 = '''\ +BRIDGE_NETWORKSTATUS_1 = b'''\ r Unmentionable BgOrX0ViP5hNsK5ZvixAuPZ6EY0 NTg9NoE5ls9KjF96Dp/UdrabZ9Y 2014-11-04 12:23:37 80.44.173.87 51691 0 a [da14:7d1e:ba8e:60d0:b078:3f88:382b:5c70]:51690 s Fast Guard Running Stable Valid @@ -52,7 +52,7 @@ w Bandwidth=24361 p reject 1-65535 ''' -BRIDGE_SERVER_DESCRIPTOR = '''\ +BRIDGE_SERVER_DESCRIPTOR = b'''\ @purpose bridge router MiserLandfalls 2.215.61.223 4056 0 0 or-address [c5fd:4467:98a7:90be:c76a:b449:8e6f:f0a7]:4055 @@ -86,7 +86,7 @@ P2aB/+XQfzFBA5TaWF83coDng4OGodhwHaOx10Kn7Bg= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR = b'''\ extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 published 2014-11-04 06:23:22 write-history 2014-11-04 06:23:22 (900 s) 3188736,2226176,2866176 @@ -117,7 +117,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWER_DUPLICATE = b'''\ extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 published 2014-11-04 08:10:25 write-history 2014-11-04 08:10:25 (900 s) 3188736,2226176,2866176,2226176 @@ -148,7 +148,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE = b'''\ extra-info MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0 published 2014-12-04 03:10:25 write-history 2014-12-04 03:10:25 (900 s) 3188736,2226176,2866176,2226176 @@ -179,7 +179,7 @@ U36EY4UoN5ABPowhNZFeyr5A3vKiDr6j0hCOqYOhxPY= -----END SIGNATURE----- ''' -BRIDGE_SERVER_DESCRIPTOR_ED25519 = '''\ +BRIDGE_SERVER_DESCRIPTOR_ED25519 = b'''\ @purpose bridge router piratepartei 80.92.79.70 80 0 0 identity-ed25519 @@ -233,7 +233,7 @@ VC4FdHgFlAkXbiqkpWtD0ojJJjLlEeXbmGILjC1Ls2I= -----END SIGNATURE----- ''' -BRIDGE_EXTRA_INFO_DESCRIPTOR_ED25519 = '''\ +BRIDGE_EXTRA_INFO_DESCRIPTOR_ED25519 = b'''\ extra-info piratepartei 312D64274C29156005843EECB19C6865FA3CC10C identity-ed25519 -----BEGIN ED25519 CERT----- @@ -307,7 +307,7 @@ class ParseDescriptorsTests(unittest.TestCase): :returns: The full path to the file which was written to. """ descFilename = os.path.join(os.getcwd(), filename) - with open(descFilename, 'w') as fh: + with open(descFilename, 'wb') as fh: for desc in descriptors: fh.write(desc) fh.flush() @@ -381,8 +381,8 @@ class ParseDescriptorsTests(unittest.TestCase): raise InvalidRouterNickname. """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( - 'MiserLandfalls', - 'MiserLandfallsWaterfallsSnowfallsAvalanche') + b'MiserLandfalls', + b'MiserLandfallsWaterfallsSnowfallsAvalanche') # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', @@ -399,8 +399,8 @@ class ParseDescriptorsTests(unittest.TestCase): See also: :trac:`16616` """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( - 's Fast Guard Running Stable Valid', - 's Fast Guard Running Stable Valid HSDir') + b's Fast Guard Running Stable Valid', + b's Fast Guard Running Stable Valid HSDir') # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', @@ -420,7 +420,7 @@ class ParseDescriptorsTests(unittest.TestCase): a ValueError. """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( - '2.215.61.223', '[2837:fcd2:387b:e376:34c:1ec7:11ff:1686]') + b'2.215.61.223', b'[2837:fcd2:387b:e376:34c:1ec7:11ff:1686]') descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', unparseable) self.assertRaises(ValueError, @@ -434,9 +434,9 @@ class ParseDescriptorsTests(unittest.TestCase): expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] descFile = 'networkstatus-bridges' - with open(descFile, 'w') as fh: - fh.write('signature and stuff from the BridgeAuth would go here\n') - fh.write('some more annotations with parameters and stuff\n') + with open(descFile, 'wb') as fh: + fh.write(b'signature and stuff from the BridgeAuth would go here\n') + fh.write(b'some more annotations with parameters and stuff\n') fh.write(BRIDGE_NETWORKSTATUS_0) fh.write(BRIDGE_NETWORKSTATUS_1) fh.flush() @@ -454,9 +454,9 @@ class ParseDescriptorsTests(unittest.TestCase): expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] descFile = 'networkstatus-bridges' - with open(descFile, 'w') as fh: - fh.write('signature and stuff from the BridgeAuth would go here\n') - fh.write('some more annotations with parameters and stuff\n') + with open(descFile, 'wb') as fh: + fh.write(b'signature and stuff from the BridgeAuth would go here\n') + fh.write(b'some more annotations with parameters and stuff\n') fh.write(BRIDGE_NETWORKSTATUS_0) fh.write(BRIDGE_NETWORKSTATUS_1) fh.flush() @@ -479,7 +479,7 @@ class ParseDescriptorsTests(unittest.TestCase): """ descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseExtraInfoFiles(descFile) - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertIsInstance(bridge, RelayExtraInfoDescriptor) def test_parse_descriptors_parseExtraInfoFiles_one_file(self): @@ -488,12 +488,12 @@ class ParseDescriptorsTests(unittest.TestCase): """ descFile = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) routers = descriptors.parseExtraInfoFiles(descFile) - bridge = routers.values()[0] + bridge = list(routers.values())[0] # The number of transports we parsed should be equal to the number of # 'transport' lines in the descriptor: self.assertEqual(len(bridge.transport), - BRIDGE_EXTRA_INFO_DESCRIPTOR.count('transport ')) + BRIDGE_EXTRA_INFO_DESCRIPTOR.count(b'transport ')) self.assertEqual(bridge.fingerprint, self.expectedFprBridge0) @@ -522,7 +522,7 @@ class ParseDescriptorsTests(unittest.TestCase): "We shouldn't have any duplicate descriptors.") # We should only have the newest descriptor: - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.published, datetime.datetime.strptime("2014-11-04 08:10:25", "%Y-%m-%d %H:%M:%S"), @@ -541,7 +541,7 @@ class ParseDescriptorsTests(unittest.TestCase): self.assertEqual(len(routers), 1, "We shouldn't have any duplicate descriptors.") - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.published, datetime.datetime.strptime("2014-11-04 08:10:25", "%Y-%m-%d %H:%M:%S"), @@ -564,7 +564,7 @@ class ParseDescriptorsTests(unittest.TestCase): "We shouldn't have any duplicate descriptors.") # We should only have the newest descriptor: - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.published, datetime.datetime.strptime("2014-12-04 03:10:25", "%Y-%m-%d %H:%M:%S"), @@ -581,10 +581,10 @@ class ParseDescriptorsTests(unittest.TestCase): descFiles = [] # The timestamp and fingerprint from BRIDGE_EXTRA_INFO_DESCRIPTOR: - timestamp = "2014-11-04 06:23:22" - Y, M, rest = timestamp.split("-") - fpr = "E08B324D20AD0A13E114F027AB9AC3F32CA696A0" - newerFpr = "E08B324D20AD0A13E114F027AB9AC3F32CA696A0" + timestamp = b"2014-11-04 06:23:22" + Y, M, rest = timestamp.split(b"-") + fpr = b"E08B324D20AD0A13E114F027AB9AC3F32CA696A0" + newerFpr = b"E08B324D20AD0A13E114F027AB9AC3F32CA696A0" total = 0 needed = b * n @@ -592,15 +592,15 @@ class ParseDescriptorsTests(unittest.TestCase): if total >= needed: break # Re-digest the fingerprint to create a "new" bridge - newerFpr = hashlib.sha1(newerFpr).hexdigest().upper() + newerFpr = hashlib.sha1(newerFpr).hexdigest().upper().encode('utf-8') # Generate n extrainfos with different timestamps: count = 0 - for year in range(1, ((n + 1)/ 12) + 2): # Start from the next year + for year in range(1, ((n + 1)// 12) + 2): # Start from the next year if count >= n: break for month in range(1, 13): if count < n: - newerTimestamp = "-".join([str(int(Y) + year), "%02d" % month, rest]) + newerTimestamp = b"-".join([str(int(Y) + year).encode('utf-8'), b"%02d" % month, rest]) newerDuplicate = BRIDGE_EXTRA_INFO_DESCRIPTOR[:].replace( fpr, newerFpr).replace( timestamp, newerTimestamp) @@ -663,10 +663,10 @@ class ParseDescriptorsTests(unittest.TestCase): """ # Give it a bad geoip-db-digest: unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( - "geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", - "geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( + b"geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", + b"geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) @@ -684,7 +684,7 @@ class ParseDescriptorsTests(unittest.TestCase): "and one was unparseable, so that should only leave one " "descriptor remaining.")) - bridge = routers.values()[0] + bridge = list(routers.values())[0] self.assertEqual( bridge.fingerprint, "E08B324D20AD0A13E114F027AB9AC3F32CA696A0", @@ -703,14 +703,14 @@ class ParseDescriptorsTests(unittest.TestCase): """ # Mess up the bridge-ip-transports line: unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( - "bridge-ip-transports <OR>=8", - "bridge-ip-transports <OR>") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( + b"bridge-ip-transports <OR>=8", + b"bridge-ip-transports <OR>") parseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "ImOkWithBeingParsed 2B5DA67FBA13A6449DE625673B7AE9E3AA7DF75F") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"ImOkWithBeingParsed 2B5DA67FBA13A6449DE625673B7AE9E3AA7DF75F") descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE) @@ -750,8 +750,8 @@ class ParseDescriptorsTests(unittest.TestCase): zero parsed descriptors. """ unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - '-----END SIGNATURE-----', - '-----END SIGNATURE FOR REALZ-----') + b'-----END SIGNATURE-----', + b'-----END SIGNATURE FOR REALZ-----') # This must be a "real" file or _copyUnparseableDescriptorFile() will # raise an AttributeError saying: # '_io.BytesIO' object has no attribute 'rpartition'" @@ -766,7 +766,7 @@ class ParseDescriptorsTests(unittest.TestCase): missing the signature should return zero parsed descriptors. """ # Remove the signature - BEGIN_SIG = '-----BEGIN SIGNATURE-----' + BEGIN_SIG = b'-----BEGIN SIGNATURE-----' unparseable, _ = BRIDGE_EXTRA_INFO_DESCRIPTOR.split(BEGIN_SIG) # This must be a "real" file or _copyUnparseableDescriptorFile() will # raise an AttributeError saying: @@ -782,7 +782,7 @@ class ParseDescriptorsTests(unittest.TestCase): bad signature should raise an InvalidExtraInfoSignature exception. """ # Truncate the signature to 50 bytes - BEGIN_SIG = '-----BEGIN SIGNATURE-----' + BEGIN_SIG = b'-----BEGIN SIGNATURE-----' doc, sig = BRIDGE_EXTRA_INFO_DESCRIPTOR.split(BEGIN_SIG) unparseable = BEGIN_SIG.join([doc, sig[:50]]) # This must be a "real" file or _copyUnparseableDescriptorFile() will @@ -803,10 +803,10 @@ class ParseDescriptorsTests(unittest.TestCase): """ # Give it a bad geoip-db-digest: unparseable = BRIDGE_EXTRA_INFO_DESCRIPTOR.replace( - "MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", - "DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( - "geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", - "geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") + b"MiserLandfalls E08B324D20AD0A13E114F027AB9AC3F32CA696A0", + b"DontParseMe F373CC1D86D82267F1F1F5D39470F0E0A022122E").replace( + b"geoip-db-digest 09A0E093100B279AD9CFF47A67B13A21C6E1483F", + b"geoip-db-digest FOOOOOOOOOOOOOOOOOOBAAAAAAAAAAAAAAAAAARR") descFileOne = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR) descFileTwo = io.BytesIO(BRIDGE_EXTRA_INFO_DESCRIPTOR_NEWEST_DUPLICATE) @@ -817,7 +817,7 @@ class ParseDescriptorsTests(unittest.TestCase): def test_parse_descriptors_parseExtraInfoFiles_empty_file(self): """Test parsing an empty extrainfo descriptors file.""" - routers = descriptors.parseExtraInfoFiles(io.BytesIO('')) + routers = descriptors.parseExtraInfoFiles(io.BytesIO(b'')) self.assertIsInstance(routers, dict) self.assertEqual(len(routers), 0) @@ -846,7 +846,7 @@ class ParseDescriptorsTests(unittest.TestCase): True when the new file is successfully created. """ filename = "bridge-descriptors" - with open(filename, 'w') as fh: + with open(filename, 'wb') as fh: fh.write(BRIDGE_SERVER_DESCRIPTOR) fh.flush() @@ -858,7 +858,7 @@ class ParseDescriptorsTests(unittest.TestCase): copy of the bad file with a specific filename format. """ filename = "bridge-descriptors" - with open(filename, 'w') as fh: + with open(filename, 'wb') as fh: fh.write(BRIDGE_SERVER_DESCRIPTOR) fh.flush()

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits February 2020