February 2020 - tor-commits - lists.torproject.org

[bridgedb/develop] Make shebangs use Python 3.
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit f9978bf5678d77eb1b03fcd2422c44b62a5733d3 Author: Philipp Winter <phw(a)nymity.ch> Date: Wed Feb 12 09:57:40 2020 -0800 Make shebangs use Python 3. --- bridgedb/test/test_Storage.py | 2 +- scripts/bridgedb | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bridgedb/test/test_Storage.py b/bridgedb/test/test_Storage.py index dffba2b..ca10bb9 100644 --- a/bridgedb/test/test_Storage.py +++ b/bridgedb/test/test_Storage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unittests for the :mod:`bridgedb.Storage` module.""" import os diff --git a/scripts/bridgedb b/scripts/bridgedb index 876fde9..5c92179 100644 --- a/scripts/bridgedb +++ b/scripts/bridgedb @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # This file is part of BridgeDB, a Tor bridge distribution system. diff --git a/setup.py b/setup.py index 55dea95..18c391b 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #_____________________________________________________________________________ # # This file is part of BridgeDB, a Tor bridge distribution system.

1 0

[bridgedb/develop] Convert filter() output to list() before len().
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 39ea49d5ebaa27e907c3c87e94ab70471e191eaf Author: Philipp Winter <phw(a)nymity.ch> Date: Mon Jan 27 16:29:14 2020 -0800 Convert filter() output to list() before len(). ...otherwise Python 3 will complain: TypeError: object of type 'filter' has no len() --- bridgedb/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgedb/main.py b/bridgedb/main.py index 7776750..bc309c5 100644 --- a/bridgedb/main.py +++ b/bridgedb/main.py @@ -198,7 +198,7 @@ def load(state, hashring, clear=False): inserted = 0 logging.info("Trying to insert %d bridges into hashring, %d of which " "have the 'Running' flag..." % (len(bridges), - len(filter(lambda b: b.flags.running, bridges.values())))) + len(list(filter(lambda b: b.flags.running, bridges.values()))))) for fingerprint, bridge in bridges.items(): # Skip insertion of bridges which are geolocated to be in one of the

1 0

[bridgedb/develop] Fix remaining str/bytes issues in moat server.
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 468dfa0fc5e13da9ba384f9bcb6e7e11bfc26482 Author: Philipp Winter <phw(a)nymity.ch> Date: Thu Feb 13 15:05:24 2020 -0800 Fix remaining str/bytes issues in moat server. We ran into these issues while test-deploying the Python 3 branch on polyanthum. --- bridgedb/distributors/moat/server.py | 16 ++++++++-------- bridgedb/test/test_distributors_moat_server.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bridgedb/distributors/moat/server.py b/bridgedb/distributors/moat/server.py index e49a6bb..cdf2022 100644 --- a/bridgedb/distributors/moat/server.py +++ b/bridgedb/distributors/moat/server.py @@ -178,11 +178,11 @@ class JsonAPIResource(resource.Resource): :param request: A ``Request`` for a :api:`twisted.web.resource.Resource`. :returns: The encoded data. """ - request.responseHeaders.addRawHeader(b"Content-Type", b"application/vnd.api+json") - request.responseHeaders.addRawHeader(b"Server", b"moat/%s" % MOAT_API_VERSION.encode('utf-8')) + request.responseHeaders.addRawHeader("Content-Type", "application/vnd.api+json") + request.responseHeaders.addRawHeader("Server", "moat/%s" % MOAT_API_VERSION) if data: - rendered = json.dumps(data) + rendered = json.dumps(data).encode("utf-8") else: rendered = b"" @@ -824,13 +824,13 @@ def addMoatServer(config, distributor): hmacKey, publicKey, secretKey, fwdHeaders, skipLoopback) - moat.putChild("fetch", fetch) - moat.putChild("check", check) - meek.putChild("moat", moat) + moat.putChild(b"fetch", fetch) + moat.putChild(b"check", check) + meek.putChild(b"moat", moat) root = CustomErrorHandlingResource() - root.putChild("meek", meek) - root.putChild("moat", moat) + root.putChild(b"meek", meek) + root.putChild(b"moat", moat) site = Site(root) site.displayTracebacks = False diff --git a/bridgedb/test/test_distributors_moat_server.py b/bridgedb/test/test_distributors_moat_server.py index 222518d..695116b 100644 --- a/bridgedb/test/test_distributors_moat_server.py +++ b/bridgedb/test/test_distributors_moat_server.py @@ -884,7 +884,7 @@ class CaptchaCheckResourceTests(unittest.TestCase): response = self.resource.failureResponse(6, request, bridgeRequest) - self.assertIn("No bridges available", response) + self.assertIn(b"No bridges available", response) def test_render_POST_no_bridges(self): self.mock_getBridgeLines() @@ -892,7 +892,7 @@ class CaptchaCheckResourceTests(unittest.TestCase): request = self.create_valid_POST_make_new_challenge() response = self.resource.render(request) - self.assertIn("No bridges available", response) + self.assertIn(b"No bridges available", response) def test_render_POST_unexpired(self): request = self.create_valid_POST_make_new_challenge()

1 0

[bridgedb/develop] Make unit test use ipaddress instead of ipaddr.
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit d035088f8eaa8ac7d0d0616b3e680c808a76e33b Author: Philipp Winter <phw(a)nymity.ch> Date: Wed Jan 29 18:22:18 2020 -0800 Make unit test use ipaddress instead of ipaddr. This should fix the following unit test: [FAIL] Traceback (most recent call last): File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_https.py", line 227, in test_get_vanilla_ipv6 self.assertIsInstance(ipaddr.IPAddress(addr), ipaddr.IPv6Address) File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/twisted/trial/_synctest.py", line 649, in assertIsInstance instance, classOrTuple, suffix)) twisted.trial.unittest.FailTest: IPv4Address('98.255.18.132') is not an instance of <class 'ipaddr.IPv6Address'> --- bridgedb/test/test_https.py | 6 +++--- bridgedb/test/test_https_server.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bridgedb/test/test_https.py b/bridgedb/test/test_https.py index 00a6fd9..2349e28 100644 --- a/bridgedb/test/test_https.py +++ b/bridgedb/test/test_https.py @@ -27,7 +27,7 @@ repository. from __future__ import print_function import gettext -import ipaddr +import ipaddress import mechanize import os @@ -199,7 +199,7 @@ class HTTPTests(unittest.TestCase): for bridge in bridges: self.assertTrue(bridge != None) addr = bridge[0].rsplit(':', 1)[0] - self.assertIsInstance(ipaddr.IPAddress(addr), ipaddr.IPv4Address) + self.assertIsInstance(ipaddress.ip_address(addr), ipaddress.IPv4Address) def test_get_vanilla_ipv6(self): self.openBrowser() @@ -212,7 +212,7 @@ class HTTPTests(unittest.TestCase): for bridge in bridges: self.assertTrue(bridge != None) addr = bridge[0].rsplit(':', 1)[0].strip('[]') - self.assertIsInstance(ipaddr.IPAddress(addr), ipaddr.IPv6Address) + self.assertIsInstance(ipaddress.ip_address(addr), ipaddress.IPv6Address) def test_get_obfs4_ipv4(self): """Try asking for obfs4 bridges, and check that the PT arguments in the diff --git a/bridgedb/test/test_https_server.py b/bridgedb/test/test_https_server.py index 8482d9d..a4d6873 100644 --- a/bridgedb/test/test_https_server.py +++ b/bridgedb/test/test_https_server.py @@ -17,7 +17,7 @@ import logging import os import shutil -import ipaddr +import ipaddress from bs4 import BeautifulSoup @@ -588,14 +588,14 @@ class ReCaptchaProtectedResourceTests(unittest.TestCase): """Check that removing our remoteip setting produces a random IP.""" self.captchaResource.remoteIP = None ip = self.captchaResource.getRemoteIP() - realishIP = ipaddr.IPv4Address(ip).compressed + realishIP = ipaddress.IPv4Address(ip).compressed self.assertTrue(realishIP) self.assertNotEquals(realishIP, '111.111.111.111') def test_getRemoteIP_useConfiguredIP(self): """Check that our remoteip setting is used if configured.""" ip = self.captchaResource.getRemoteIP() - realishIP = ipaddr.IPv4Address(ip).compressed + realishIP = ipaddress.IPv4Address(ip).compressed self.assertTrue(realishIP) self.assertEquals(realishIP, '111.111.111.111') @@ -774,7 +774,7 @@ class BridgesResourceTests(unittest.TestCase): # Check that the IP and port seem okay: ip, port = fields[0].rsplit(':') - self.assertIsInstance(ipaddr.IPv4Address(ip), ipaddr.IPv4Address) + self.assertIsInstance(ipaddress.ip_address(ip), ipaddress.IPv4Address) self.assertIsInstance(int(port), int) self.assertGreater(int(port), 0) self.assertLessEqual(int(port), 65535) @@ -858,7 +858,7 @@ class BridgesResourceTests(unittest.TestCase): # Check that the IP and port seem okay: ip, port = bridgeLine[1].rsplit(':') - self.assertIsInstance(ipaddr.IPv4Address(ip), ipaddr.IPv4Address) + self.assertIsInstance(ipaddress.ip_address(ip), ipaddress.IPv4Address) self.assertIsInstance(int(port), int) self.assertGreater(int(port), 0) self.assertLessEqual(int(port), 65535) @@ -892,7 +892,7 @@ class BridgesResourceTests(unittest.TestCase): # Check that the IP and port seem okay: ip, port = bridgeLine[0].rsplit(b':') - self.assertIsInstance(ipaddr.IPv4Address(ip), ipaddr.IPv4Address) + self.assertIsInstance(ipaddress.ip_address(ip.decode("utf-8")), ipaddress.IPv4Address) self.assertIsInstance(int(port), int) self.assertGreater(int(port), 0) self.assertLessEqual(int(port), 65535)

1 0

[bridgedb/develop] Turn str into bytes in SMTP unit test.
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit b0c74f0ae932fffcd260eb26fd6594968b11908b Author: Philipp Winter <phw(a)nymity.ch> Date: Wed Jan 29 17:58:03 2020 -0800 Turn str into bytes in SMTP unit test. This should fix the following unit test errors: [ERROR] Traceback (most recent call last): File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_smtp.py", line 155, in test_getBridges self.server.getAndCheckMessageContains("Here are your bridges") File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_smtp.py", line 99, in getAndCheckMessageContains assert message.find(text) != -1, ("Message did not contain text '%s'." builtins.TypeError: argument should be integer or bytes-like object, not 'str' bridgedb.test.test_smtp.SMTPTests.test_getBridges =============================================================================== [ERROR] Traceback (most recent call last): File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_smtp.py", line 171, in test_getBridges_rateLimitExceeded self.server.getAndCheckMessageContains("Here are your bridges") File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_smtp.py", line 99, in getAndCheckMessageContains assert message.find(text) != -1, ("Message did not contain text '%s'." builtins.TypeError: argument should be integer or bytes-like object, not 'str' bridgedb.test.test_smtp.SMTPTests.test_getBridges_rateLimitExceeded =============================================================================== [ERROR] Traceback (most recent call last): File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_smtp.py", line 206, in test_getBridges_stressTest self.server.getAndCheckMessageContains("Here are your bridges") File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_smtp.py", line 99, in getAndCheckMessageContains assert message.find(text) != -1, ("Message did not contain text '%s'." builtins.TypeError: argument should be integer or bytes-like object, not 'str' --- bridgedb/test/test_smtp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bridgedb/test/test_smtp.py b/bridgedb/test/test_smtp.py index 3fd06f3..f7261da 100644 --- a/bridgedb/test/test_smtp.py +++ b/bridgedb/test/test_smtp.py @@ -152,7 +152,7 @@ class SMTPTests(unittest.TestCase): # then check that our local SMTP server received a response # and that response contained some bridges - self.server.getAndCheckMessageContains("Here are your bridges") + self.server.getAndCheckMessageContains(b"Here are your bridges") def test_getBridges_rateLimitExceeded(self): if os.environ.get("CI"): @@ -168,14 +168,14 @@ class SMTPTests(unittest.TestCase): # then check that our local SMTP server received a response # and that response contained some bridges - self.server.getAndCheckMessageContains("Here are your bridges") + self.server.getAndCheckMessageContains(b"Here are your bridges") # send another request from the same email address sendMail(FROM_ADDRESS) # this time, the email response should not contain any bridges self.server.getAndCheckMessageContains( - "You have exceeded the rate limit. Please slow down!") + b"You have exceeded the rate limit. Please slow down!") # then we send another request from the same email address sendMail(FROM_ADDRESS) @@ -203,4 +203,4 @@ class SMTPTests(unittest.TestCase): % random.randint(MIN_FROM_ADDRESS, MAX_FROM_ADDRESS)) for i in range(NUM_MAILS): - self.server.getAndCheckMessageContains("Here are your bridges") + self.server.getAndCheckMessageContains(b"Here are your bridges")

1 0

[bridgedb/develop] Discard attribute that's no longer used.
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit b452f6f9ffb4992bb6be00763ce53ee0e5479878 Author: Philipp Winter <phw(a)nymity.ch> Date: Wed Jan 29 18:10:41 2020 -0800 Discard attribute that's no longer used. This fixes the following unit test: [ERROR] Traceback (most recent call last): File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_https.py", line 177, in test_content_security_policy headers = ''.join(self.br.response().info().headers) builtins.AttributeError: 'HTTPMessage' object has no attribute 'headers' bridgedb.test.test_https.HTTPTests.test_content_security_policy --- bridgedb/test/test_https.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridgedb/test/test_https.py b/bridgedb/test/test_https.py index 52ad936..00a6fd9 100644 --- a/bridgedb/test/test_https.py +++ b/bridgedb/test/test_https.py @@ -174,10 +174,10 @@ class HTTPTests(unittest.TestCase): self.br.set_debug_http(True) self.br.open(HTTP_ROOT) - headers = ''.join(self.br.response().info().headers) + headers = self.br.response().info() - self.assertIn("Content-Security-Policy", headers) - self.assertIn("default-src 'none';", headers) + self.assertIn("Content-Security-Policy", headers.keys()) + self.assertIn("default-src 'none';", ''.join(headers.values())) def test_404(self): """Asking for a non-existent resource should yield our custom 404 page,

1 0

[bridgedb/develop] Merge branch 'defect/30946' into develop
by phw＠torproject.org 19 Feb '20

19 Feb '20

1 0

[bridgedb/develop] Rejection-sample randomly-generated v4 addresses.
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 60ed840f3882f80b81a984d419cc7f78d1c96cfb Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Feb 11 16:28:33 2020 -0800 Rejection-sample randomly-generated v4 addresses. So far, we generated completely random IPv4 addresses which would sometimes contain multicast addresses. This patch adds rejection sampling, as we're already doing it for IPv6 addresses. --- scripts/create_descriptors | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/scripts/create_descriptors b/scripts/create_descriptors index 0b9d26d..60022db 100755 --- a/scripts/create_descriptors +++ b/scripts/create_descriptors @@ -117,10 +117,13 @@ def get_hex_string(size): return hexstr -def get_random_ipv6_addr(): +def get_random_addr(ip_version=4): valid_addr = None while not valid_addr: - maybe = ipaddress.IPv6Address(random.getrandbits(128)) + if ip_version == 4: + maybe = ipaddress.IPv4Address(random.getrandbits(32)) + else: + maybe = ipaddress.IPv6Address(random.getrandbits(128)) valid = check_ip_validity(maybe) if valid: valid_addr = maybe @@ -128,13 +131,6 @@ def get_random_ipv6_addr(): return str(valid_addr) -def get_random_ipv4_addr(): - return "%i.%i.%i.%i" % (random.randint(0, 255), - random.randint(0, 255), - random.randint(0, 255), - random.randint(0, 255)) - - def get_protocol(tor_version): line = "" if tor_version is not None: @@ -198,8 +194,8 @@ def create_server_desc(signing_key): timestamp = make_timestamp(variation=True, period=36) server_desc = RelayDescriptor.create({ - "router": "%s %s %s 0 0" % (nickname, get_random_ipv4_addr(), port), - "or-address": "[%s]:%s" % (get_random_ipv6_addr(), port-1), + "router": "%s %s %s 0 0" % (nickname, get_random_addr(ip_version=4), port), + "or-address": "[%s]:%s" % (get_random_addr(ip_version=6), port-1), "platform": "Tor %s on Linux" % tor_version, get_protocol(tor_version): "", "published": timestamp,

1 0

[bridgedb/develop] Turn hostname from str into bytes.
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 4a7f7de90c24fc583325892f44103eca1004fc3e Author: Philipp Winter <phw(a)nymity.ch> Date: Thu Jan 30 17:31:32 2020 -0800 Turn hostname from str into bytes. This fixes the following two failed unit tests: =============================================================================== [FAIL] Traceback (most recent call last): File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_email_server.py", line 549, in test_addServer noisy=True) File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_email_server.py", line 381, in _test self.assertSubstring(expected, recv) File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/twisted/trial/_synctest.py", line 571, in assertSubstring return self.failUnlessIn(substring, astring, msg) File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/twisted/trial/_synctest.py", line 492, in assertIn % (containee, container)) twisted.trial.unittest.FailTest: '250 Delivery in progress' not in '220 bridges.torproject.org NO UCE NO UBE NO RELAY PROBES\r\n250 bridges.torproject.org Hello 192.168.1.1, nice to meet you\r\n550 Cannot receive from specified address <testing@travis-job-29d1fb7f-84ab-4139-977c-5f18fc201f57>: Sender not acceptable\r\n503 Must have sender before recipient\r\n503 Must have valid receiver and originator\r\n500 Command not implemented\r\n500 Command not implemented\r\n500 Command not implemented\r\n500 Error: bad syntax\r\n500 Command not implemented\r\n500 Command not implemented\r\n' bridgedb.test.test_email_server.EmailServerServiceTests.test_addServer =============================================================================== [FAIL] Traceback (most recent call last): File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_email_server.py", line 422, in test_SMTPIncomingServerFactory_MAIL_FROM_testing_at_gethostname "250 Sender address accepted") File "/home/travis/build/NullHypothesis/bridgedb/bridgedb/test/test_email_server.py", line 381, in _test self.assertSubstring(expected, recv) File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/twisted/trial/_synctest.py", line 571, in assertSubstring return self.failUnlessIn(substring, astring, msg) File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/twisted/trial/_synctest.py", line 492, in assertIn % (containee, container)) twisted.trial.unittest.FailTest: '250 Sender address accepted' not in '220 bridges.torproject.org NO UCE NO UBE NO RELAY PROBES\r\n250 bridges.torproject.org Hello 192.168.1.1, nice to meet you\r\n550 Cannot receive from specified address <testing@travis-job-29d1fb7f-84ab-4139-977c-5f18fc201f57>: Sender not acceptable\r\n' bridgedb.test.test_email_server.SMTPIncomingServerFactoryTests.test_SMTPIncomingServerFactory_MAIL_FROM_testing_at_gethostname =============================================================================== --- bridgedb/distributors/email/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridgedb/distributors/email/server.py b/bridgedb/distributors/email/server.py index 38ed988..063b640 100644 --- a/bridgedb/distributors/email/server.py +++ b/bridgedb/distributors/email/server.py @@ -126,7 +126,7 @@ class MailServerContext(object): self.nBridges = config.EMAIL_N_BRIDGES_PER_ANSWER self.username = (config.EMAIL_USERNAME or "bridges") - self.hostname = socket.gethostname() + self.hostname = socket.gethostname().encode("utf-8") self.fromAddr = (config.EMAIL_FROM_ADDR or "bridges(a)torproject.org") self.smtpFromAddr = (config.EMAIL_SMTP_FROM_ADDR or self.fromAddr) self.smtpServerPort = (config.EMAIL_SMTP_PORT or 25)

1 0

[bridgedb/develop] Fix str/bytes issue in serving of QR code.
by phw＠torproject.org 19 Feb '20

19 Feb '20

commit 63ffb833e2592558265d5b23b8b0e1d146daaae7 Author: Philipp Winter <phw(a)nymity.ch> Date: Fri Feb 14 10:06:46 2020 -0800 Fix str/bytes issue in serving of QR code. --- bridgedb/distributors/https/server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bridgedb/distributors/https/server.py b/bridgedb/distributors/https/server.py index 98b31d0..503744e 100644 --- a/bridgedb/distributors/https/server.py +++ b/bridgedb/distributors/https/server.py @@ -1066,7 +1066,9 @@ class BridgesResource(CustomErrorHandlingResource, CSPResource): qrjpeg = generateQR(bridgeLines) if qrjpeg: - qrcode = 'data:image/jpeg;base64,%s' % base64.b64encode(qrjpeg) + qrcode = b'data:image/jpeg;base64,%s' % base64.b64encode(qrjpeg) + qrcode = qrcode.decode("utf-8") + try: langs = translations.getLocaleFromHTTPRequest(request) rtl = translations.usingRTLLang(langs)

1 0