commit cc20b182cd2c4deaee0b793d2fbf05d550af280d Author: Damian Johnson <atagar@torproject.org> Date: Fri Jan 3 16:15:26 2020 -0800 Drop urllib fallback Python 3.x renamed the urllib2 module to urllib.request. --- cache_fallback_directories.py | 9 ++------- cache_manual.py | 9 ++------- stem/__init__.py | 2 +- stem/descriptor/remote.py | 11 +++-------- stem/directory.py | 11 +++-------- stem/manual.py | 9 ++------- stem/util/connection.py | 9 ++------- test/integ/util/connection.py | 9 ++------- test/unit/manual.py | 9 ++------- test/unit/util/connection.py | 15 +++++---------- 10 files changed, 24 insertions(+), 69 deletions(-) diff --git a/cache_fallback_directories.py b/cache_fallback_directories.py index cb413c07..91ad40c0 100755 --- a/cache_fallback_directories.py +++ b/cache_fallback_directories.py @@ -8,22 +8,17 @@ Caches tor's latest fallback directories. import re import sys +import urllib.request import stem.directory import stem.util.system -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - GITWEB_FALLBACK_LOG = 'https://gitweb.torproject.org/tor.git/log/src/app/config/fallback_dirs.inc' FALLBACK_DIR_LINK = "href='/tor.git/commit/src/app/config/fallback_dirs.inc\\?id=([^']*)'" if __name__ == '__main__': try: - fallback_dir_page = urllib.urlopen(GITWEB_FALLBACK_LOG).read() + fallback_dir_page = urllib.request.urlopen(GITWEB_FALLBACK_LOG).read() fallback_dir_commit = re.search(FALLBACK_DIR_LINK, fallback_dir_page).group(1) except: print("Unable to determine the latest commit to edit tor's fallback directories: %s" % sys.exc_info()[1]) diff --git a/cache_manual.py b/cache_manual.py index 7fe6ea70..5bc68b57 100755 --- a/cache_manual.py +++ b/cache_manual.py @@ -8,22 +8,17 @@ Caches tor's latest manual content. Run this to pick new man page changes. import re import sys +import urllib.request import stem.manual import stem.util.system -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - GITWEB_MAN_LOG = 'https://gitweb.torproject.org/tor.git/log/doc/tor.1.txt' MAN_LOG_LINK = "href='/tor.git/commit/doc/tor.1.txt\\?id=([^']*)'" if __name__ == '__main__': try: - man_log_page = urllib.urlopen(GITWEB_MAN_LOG).read() + man_log_page = urllib.request.urlopen(GITWEB_MAN_LOG).read() man_commit = re.search(MAN_LOG_LINK, man_log_page).group(1) except: print("Unable to determine the latest commit to edit tor's man page: %s" % sys.exc_info()[1]) diff --git a/stem/__init__.py b/stem/__init__.py index 66bc24b5..3600e920 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -722,7 +722,7 @@ class SocketClosed(SocketError): class DownloadFailed(IOError): """ Inability to download a resource. Python's urllib module raises - a wide variety of undocumented exceptions (urllib2.URLError, + a wide variety of undocumented exceptions (urllib.request.URLError, socket.timeout, and others). This wraps lower level failures in a common exception type that diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index c88b338e..c7132a38 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -104,6 +104,7 @@ import socket import sys import threading import time +import urllib.request import stem import stem.client @@ -116,12 +117,6 @@ import stem.util.tor_tools from stem.util import log, str_tools -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - # TODO: remove in stem 2.x, replaced with stem.descriptor.Compression Compression = stem.util.enum.Enum( @@ -1070,8 +1065,8 @@ def _download_from_dirport(url, compression, timeout): """ try: - response = urllib.urlopen( - urllib.Request( + response = urllib.request.urlopen( + urllib.request.Request( url, headers = { 'Accept-Encoding': ', '.join(map(lambda c: c.encoding, compression)), diff --git a/stem/directory.py b/stem/directory.py index 0ca089c3..7a1ce7b8 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -42,6 +42,7 @@ import collections import os import re import sys +import urllib.request import stem import stem.util @@ -49,12 +50,6 @@ import stem.util.conf from stem.util import connection, str_tools, tor_tools -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - GITWEB_AUTHORITY_URL = 'https://gitweb.torproject.org/tor.git/plain/src/app/config/auth_dirs.inc' GITWEB_FALLBACK_URL = 'https://gitweb.torproject.org/tor.git/plain/src/app/config/fallback_dirs.inc' FALLBACK_CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_fallbacks.cfg') @@ -260,7 +255,7 @@ class Authority(Directory): @staticmethod def from_remote(timeout = 60): try: - lines = str_tools._to_unicode(urllib.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() + lines = str_tools._to_unicode(urllib.request.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() if not lines: raise IOError('no content') @@ -408,7 +403,7 @@ class Fallback(Directory): @staticmethod def from_remote(timeout = 60): try: - lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() + lines = str_tools._to_unicode(urllib.request.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() if not lines: raise IOError('no content') diff --git a/stem/manual.py b/stem/manual.py index d9ad5aa7..fe49e3de 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -54,6 +54,7 @@ import os import shutil import sys import tempfile +import urllib.request import stem import stem.prereq @@ -63,12 +64,6 @@ import stem.util.enum import stem.util.log import stem.util.system -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'RELAY', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'DENIAL_OF_SERVICE', 'TESTING', 'UNKNOWN') GITWEB_MANUAL_URL = 'https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt' CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_manual.sqlite') @@ -300,7 +295,7 @@ def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, try: try: with open(asciidoc_path, 'wb') as asciidoc_file: - request = urllib.urlopen(url, timeout = timeout) + request = urllib.request.urlopen(url, timeout = timeout) shutil.copyfileobj(request, asciidoc_file) except: exc, stacktrace = sys.exc_info()[1:3] diff --git a/stem/util/connection.py b/stem/util/connection.py index 7c8e864b..f88b3f85 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -62,6 +62,7 @@ import re import socket import sys import time +import urllib.request import stem import stem.util @@ -70,12 +71,6 @@ import stem.util.system from stem.util import conf, enum, log, str_tools -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - # Connection resolution is risky to log about since it's highly likely to # contain sensitive information. That said, it's also difficult to get right in # a platform independent fashion. To opt into the logging requried to @@ -197,7 +192,7 @@ def download(url, timeout = None, retries = None): start_time = time.time() try: - return urllib.urlopen(url, timeout = timeout).read() + return urllib.request.urlopen(url, timeout = timeout).read() except socket.timeout as exc: raise stem.DownloadTimeout(url, exc, sys.exc_info()[2], timeout) except: diff --git a/test/integ/util/connection.py b/test/integ/util/connection.py index e31ee865..861b8ba9 100644 --- a/test/integ/util/connection.py +++ b/test/integ/util/connection.py @@ -4,6 +4,7 @@ that we're running. """ import unittest +import urllib.request import stem import stem.util.connection @@ -13,12 +14,6 @@ import test.runner from stem.util.connection import Resolver -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - class TestConnection(unittest.TestCase): @test.require.ptrace @@ -58,7 +53,7 @@ class TestConnection(unittest.TestCase): self.assertEqual('Failed to download from https://no.such.testing.url (URLError): Name or service not known', str(exc)) self.assertEqual('https://no.such.testing.url', exc.url) self.assertEqual('Name or service not known', exc.error.reason.strerror) - self.assertEqual(urllib.URLError, type(exc.error)) + self.assertEqual(urllib.request.URLError, type(exc.error)) def test_connections_by_proc(self): self.check_resolver(Resolver.PROC) diff --git a/test/unit/manual.py b/test/unit/manual.py index 6e80543e..eb76d6e9 100644 --- a/test/unit/manual.py +++ b/test/unit/manual.py @@ -8,6 +8,7 @@ import os import sqlite3 import tempfile import unittest +import urllib.request import stem.prereq import stem.manual @@ -16,12 +17,6 @@ import test.require from unittest.mock import Mock, patch -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - EXAMPLE_MAN_PATH = os.path.join(os.path.dirname(__file__), 'tor_man_example') UNKNOWN_OPTIONS_MAN_PATH = os.path.join(os.path.dirname(__file__), 'tor_man_with_unknown') @@ -247,7 +242,7 @@ class TestManual(unittest.TestCase): @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', Mock(return_value = io.BytesIO()), create = True) @patch('stem.util.system.is_available', Mock(return_value = True)) - @patch('urllib.request.urlopen', Mock(side_effect = urllib.URLError('<urlopen error [Errno -2] Name or service not known>'))) + @patch('urllib.request.urlopen', Mock(side_effect = urllib.request.URLError('<urlopen error [Errno -2] Name or service not known>'))) def test_download_man_page_when_download_fails(self): exc_msg = "Unable to download tor's manual from https://www.atagar.com/foo/bar to /no/such/path/tor.1.txt: <urlopen error <urlopen error [Errno -2] Name or service not known>>" self.assertRaisesWith(IOError, exc_msg, stem.manual.download_man_page, '/tmp/no_such_file', url = 'https://www.atagar.com/foo/bar') diff --git a/test/unit/util/connection.py b/test/unit/util/connection.py index 047dced7..d848c40b 100644 --- a/test/unit/util/connection.py +++ b/test/unit/util/connection.py @@ -5,6 +5,7 @@ Unit tests for the stem.util.connection functions. import io import platform import unittest +import urllib.request import stem import stem.util.connection @@ -13,12 +14,6 @@ from unittest.mock import Mock, patch from stem.util.connection import Resolver, Connection -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - URL = 'https://example.unit.test.url' NETSTAT_OUTPUT = """\ @@ -181,7 +176,7 @@ class TestConnection(unittest.TestCase): @patch('urllib.request.urlopen') def test_download_failure(self, urlopen_mock): - urlopen_mock.side_effect = urllib.URLError('boom') + urlopen_mock.side_effect = urllib.request.URLError('boom') try: stem.util.connection.download(URL) @@ -190,12 +185,12 @@ class TestConnection(unittest.TestCase): self.assertEqual('Failed to download from https://example.unit.test.url (URLError): boom', str(exc)) self.assertEqual(URL, exc.url) self.assertEqual('boom', exc.error.reason) - self.assertEqual(urllib.URLError, type(exc.error)) - self.assertTrue('return urllib.urlopen(url, timeout = timeout).read()' in exc.stacktrace_str) + self.assertEqual(urllib.request.URLError, type(exc.error)) + self.assertTrue('return urllib.request.urlopen(url, timeout = timeout).read()' in exc.stacktrace_str) @patch('urllib.request.urlopen') def test_download_retries(self, urlopen_mock): - urlopen_mock.side_effect = urllib.URLError('boom') + urlopen_mock.side_effect = urllib.request.URLError('boom') self.assertRaisesRegexp(IOError, 'boom', stem.util.connection.download, URL) self.assertEqual(1, urlopen_mock.call_count)