January 2020 - tor-commits - lists.torproject.org

[stem/master] Drop OrderedDict copy
by atagar＠torproject.org 05 Jan '20

05 Jan '20

commit a1eaf853775316e20649e49115a4e9839dfa4393 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 3 13:46:48 2020 -0800 Drop OrderedDict copy Python added OrderedDict as a builtin in Python 2.7. We included a copy for Python 2.6 compatibity and as such is no longer required. --- stem/control.py | 10 +- stem/descriptor/__init__.py | 12 +- stem/descriptor/bandwidth_file.py | 11 +- stem/directory.py | 11 +- stem/manual.py | 33 +++--- stem/util/conf.py | 15 +-- stem/util/ordereddict.py | 133 ---------------------- test/unit/descriptor/bandwidth_file.py | 13 +-- test/unit/descriptor/hidden_service_v3.py | 9 +- test/unit/descriptor/networkstatus/document_v3.py | 11 +- test/unit/descriptor/router_status_entry.py | 9 +- test/unit/descriptor/server_descriptor.py | 9 +- test/unit/directory/fallback.py | 11 +- test/unit/manual.py | 11 +- 14 files changed, 49 insertions(+), 249 deletions(-) diff --git a/stem/control.py b/stem/control.py index 56567467..25a65f43 100644 --- a/stem/control.py +++ b/stem/control.py @@ -257,12 +257,6 @@ import threading import time try: - # Added in 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - -try: # Added in 3.x import queue except ImportError: @@ -2623,7 +2617,7 @@ class Controller(BaseController): log.debug('GETCONF HiddenServiceOptions (failed: %s)' % exc) raise - service_dir_map = OrderedDict() + service_dir_map = collections.OrderedDict() directory = None for status_code, divider, content in response.content(): @@ -2779,7 +2773,7 @@ class Controller(BaseController): if path in conf and (port, target_address, target_port) in conf[path]['HiddenServicePort']: return None - conf.setdefault(path, OrderedDict()).setdefault('HiddenServicePort', []).append((port, target_address, target_port)) + conf.setdefault(path, collections.OrderedDict()).setdefault('HiddenServicePort', []).append((port, target_address, target_port)) if auth_type and client_names: hsac = "%s %s" % (auth_type, ','.join(client_names)) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index c97db183..bab339ac 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -106,12 +106,6 @@ import stem.util.enum import stem.util.str_tools import stem.util.system -try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - __all__ = [ 'bandwidth_file', 'certificate', @@ -589,7 +583,7 @@ def _descriptor_content(attr = None, exclude = (), header_template = (), footer_ """ header_content, footer_content = [], [] - attr = {} if attr is None else OrderedDict(attr) # shallow copy since we're destructive + attr = {} if attr is None else collections.OrderedDict(attr) # shallow copy since we're destructive for content, template in ((header_content, header_template), (footer_content, footer_template)): @@ -707,7 +701,7 @@ def _parse_protocol_line(keyword, attribute): # parses 'protocol' entries like: Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 value = _value(keyword, entries) - protocols = OrderedDict() + protocols = collections.OrderedDict() for k, v in _mappings_for(keyword, value): versions = [] @@ -1468,7 +1462,7 @@ def _descriptor_components(raw_contents, validate, extra_keywords = (), non_asci if isinstance(raw_contents, bytes): raw_contents = stem.util.str_tools._to_unicode(raw_contents) - entries = OrderedDict() + entries = collections.OrderedDict() extra_entries = [] # entries with a keyword in extra_keywords remaining_lines = raw_contents.split('\n') diff --git a/stem/descriptor/bandwidth_file.py b/stem/descriptor/bandwidth_file.py index 658f02b8..43c43aff 100644 --- a/stem/descriptor/bandwidth_file.py +++ b/stem/descriptor/bandwidth_file.py @@ -14,6 +14,7 @@ Parsing for Bandwidth Authority metrics as described in Tor's .. versionadded:: 1.8.0 """ +import collections import datetime import io import time @@ -25,12 +26,6 @@ from stem.descriptor import ( Descriptor, ) -try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - # Four character dividers are allowed for backward compatability, but five is # preferred. @@ -175,7 +170,7 @@ def _parse_file(descriptor_file, validate = False, **kwargs): def _parse_header(descriptor, entries): - header = OrderedDict() + header = collections.OrderedDict() content = io.BytesIO(descriptor.get_bytes()) content.readline() # skip the first line, which should be the timestamp @@ -332,7 +327,7 @@ class BandwidthFile(Descriptor): if sign: raise NotImplementedError('Signing of %s not implemented' % cls.__name__) - header = OrderedDict(attr) if attr is not None else OrderedDict() + header = collections.OrderedDict(attr) if attr is not None else collections.OrderedDict() timestamp = header.pop('timestamp', str(int(time.time()))) content = header.pop('content', []) version = header.get('version', HEADER_DEFAULT.get('version')) diff --git a/stem/directory.py b/stem/directory.py index b93b49a2..0ca089c3 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -38,6 +38,7 @@ as follows... .. versionadded:: 1.7.0 """ +import collections import os import re import sys @@ -49,12 +50,6 @@ import stem.util.conf from stem.util import connection, str_tools, tor_tools try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - -try: # account for urllib's change between python 2.x and 3.x import urllib.request as urllib except ImportError: @@ -369,13 +364,13 @@ class Fallback(Directory): def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, has_extrainfo = False, orport_v6 = None, header = None): super(Fallback, self).__init__(address, or_port, dir_port, fingerprint, nickname, orport_v6) self.has_extrainfo = has_extrainfo - self.header = OrderedDict(header) if header else OrderedDict() + self.header = collections.OrderedDict(header) if header else collections.OrderedDict() @staticmethod def from_cache(path = FALLBACK_CACHE_PATH): conf = stem.util.conf.Config() conf.load(path) - headers = OrderedDict([(k.split('.', 1)[1], conf.get(k)) for k in conf.keys() if k.startswith('header.')]) + headers = collections.OrderedDict([(k.split('.', 1)[1], conf.get(k)) for k in conf.keys() if k.startswith('header.')]) results = {} diff --git a/stem/manual.py b/stem/manual.py index 25d435d4..b94d4e0b 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -48,6 +48,7 @@ us what our torrc options do... .. versionadded:: 1.5.0 """ +import collections import os import shutil import sys @@ -61,12 +62,6 @@ import stem.util.enum import stem.util.log import stem.util.system -try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - if stem.prereq._is_lru_cache_available(): from functools import lru_cache else: @@ -96,7 +91,7 @@ SCHEMA = ( 'CREATE TABLE torrc(key TEXT PRIMARY KEY, name TEXT, category TEXT, usage TEXT, summary TEXT, description TEXT, position INTEGER)', ) -CATEGORY_SECTIONS = OrderedDict(( +CATEGORY_SECTIONS = collections.OrderedDict(( ('GENERAL OPTIONS', Category.GENERAL), ('CLIENT OPTIONS', Category.CLIENT), ('SERVER OPTIONS', Category.RELAY), @@ -374,10 +369,10 @@ class Manual(object): self.name = name self.synopsis = synopsis self.description = description - self.commandline_options = OrderedDict(commandline_options) - self.signals = OrderedDict(signals) - self.files = OrderedDict(files) - self.config_options = OrderedDict(config_options) + self.commandline_options = collections.OrderedDict(commandline_options) + self.signals = collections.OrderedDict(signals) + self.files = collections.OrderedDict(files) + self.config_options = collections.OrderedDict(config_options) self.man_commit = None self.stem_commit = None self.schema = None @@ -442,7 +437,7 @@ class Manual(object): signals = dict(conn.execute('SELECT name, description FROM signals').fetchall()) files = dict(conn.execute('SELECT name, description FROM files').fetchall()) - config_options = OrderedDict() + config_options = collections.OrderedDict() for entry in conn.execute('SELECT name, category, usage, summary, description FROM torrc ORDER BY position').fetchall(): option, category, usage, summary, option_description = entry @@ -460,7 +455,7 @@ class Manual(object): conf = stem.util.conf.Config() conf.load(path, commenting = False) - config_options = OrderedDict() + config_options = collections.OrderedDict() for key in conf.keys(): if key.startswith('config_options.'): @@ -479,9 +474,9 @@ class Manual(object): conf.get('name', ''), conf.get('synopsis', ''), conf.get('description', ''), - conf.get('commandline_options', OrderedDict()), - conf.get('signals', OrderedDict()), - conf.get('files', OrderedDict()), + conf.get('commandline_options', collections.OrderedDict()), + conf.get('signals', collections.OrderedDict()), + conf.get('files', collections.OrderedDict()), config_options, ) @@ -514,7 +509,7 @@ class Manual(object): except OSError as exc: raise IOError("Unable to run '%s': %s" % (man_cmd, exc)) - categories, config_options = _get_categories(man_output), OrderedDict() + categories, config_options = _get_categories(man_output), collections.OrderedDict() for category_header, category_enum in CATEGORY_SECTIONS.items(): _add_config_options(config_options, category_enum, categories.get(category_header, [])) @@ -677,7 +672,7 @@ def _get_categories(content): if content and content[-1].startswith('Tor'): content = content[:-1] - categories = OrderedDict() + categories = collections.OrderedDict() category, lines = None, [] for line in content: @@ -727,7 +722,7 @@ def _get_indented_descriptions(lines): ignoring those. """ - options, last_arg = OrderedDict(), None + options, last_arg = collections.OrderedDict(), None for line in lines: if line == ' Note': diff --git a/stem/util/conf.py b/stem/util/conf.py index b4580ed9..7dffe95a 100644 --- a/stem/util/conf.py +++ b/stem/util/conf.py @@ -157,6 +157,7 @@ Here's an expanation of what happened... +- get_value - provides the value for a given key as a string """ +import collections import inspect import os import threading @@ -165,12 +166,6 @@ import stem.prereq from stem.util import log -try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - CONFS = {} # mapping of identifier to singleton instances of configs @@ -453,9 +448,9 @@ class Config(object): """ def __init__(self): - self._path = None # location we last loaded from or saved to - self._contents = OrderedDict() # configuration key/value pairs - self._listeners = [] # functors to be notified of config changes + self._path = None # location we last loaded from or saved to + self._contents = collections.OrderedDict() # configuration key/value pairs + self._listeners = [] # functors to be notified of config changes # used for accessing _contents self._contents_lock = threading.RLock() @@ -735,7 +730,7 @@ class Config(object): elif isinstance(default, tuple): val = tuple(val) elif isinstance(default, dict): - val_map = OrderedDict() + val_map = collections.OrderedDict() for entry in val: if '=>' in entry: entry_key, entry_val = entry.split('=>', 1) diff --git a/stem/util/ordereddict.py b/stem/util/ordereddict.py deleted file mode 100644 index ec228f31..00000000 --- a/stem/util/ordereddict.py +++ /dev/null @@ -1,133 +0,0 @@ -# Drop in replacement for python 2.7's OrderedDict, from... -# https://pypi.org/project/ordereddict/ -# -# Stem users should *not* rely upon this module. It will be removed when we -# drop support for python 2.6 and below. - -# Copyright (c) 2009 Raymond Hettinger -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -from UserDict import DictMixin - - -class OrderedDict(dict, DictMixin): - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - if last: - key = reversed(self).next() - else: - key = iter(self).next() - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - if len(self) != len(other): - return False - for p, q in zip(self.items(), other.items()): - if p != q: - return False - return True - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other diff --git a/test/unit/descriptor/bandwidth_file.py b/test/unit/descriptor/bandwidth_file.py index 3040c38b..bb1eeffa 100644 --- a/test/unit/descriptor/bandwidth_file.py +++ b/test/unit/descriptor/bandwidth_file.py @@ -2,6 +2,7 @@ Unit tests for stem.descriptor.bandwidth_file. """ +import collections import datetime import unittest @@ -11,12 +12,6 @@ from stem.descriptor.bandwidth_file import BandwidthFile from test.unit.descriptor import get_resource try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - -try: # added in python 3.3 from unittest.mock import Mock, patch except ImportError: @@ -272,7 +267,7 @@ class TestBandwidthFile(unittest.TestCase): Exercise the example in our content method's pydoc. """ - content = BandwidthFile.content(OrderedDict([ + content = BandwidthFile.content(collections.OrderedDict([ ('timestamp', '12345'), ('version', '1.2.0'), ('content', []), @@ -286,7 +281,7 @@ class TestBandwidthFile(unittest.TestCase): Include an unrecognized header field. """ - desc = BandwidthFile.create(OrderedDict([('version', '1.1.0'), ('new_header', 'neat stuff')])) + desc = BandwidthFile.create(collections.OrderedDict([('version', '1.1.0'), ('new_header', 'neat stuff')])) self.assertEqual(EXPECTED_NEW_HEADER_CONTENT, str(desc)) self.assertEqual('1.1.0', desc.version) self.assertEqual({'version': '1.1.0', 'new_header': 'neat stuff'}, desc.header) @@ -309,7 +304,7 @@ class TestBandwidthFile(unittest.TestCase): self.assertRaisesWith(ValueError, "The 'version' header must be in the second position", BandwidthFile.from_str, WRONG_VERSION_POSITION, validate = True) - content = BandwidthFile.content(OrderedDict([ + content = BandwidthFile.content(collections.OrderedDict([ ('timestamp', '1410723598'), ('file_created', '2019-01-14T05:35:06'), ('version', '1.1.0'), diff --git a/test/unit/descriptor/hidden_service_v3.py b/test/unit/descriptor/hidden_service_v3.py index 33507314..0c172fcb 100644 --- a/test/unit/descriptor/hidden_service_v3.py +++ b/test/unit/descriptor/hidden_service_v3.py @@ -3,6 +3,7 @@ Unit tests for stem.descriptor.hidden_service for version 3. """ import base64 +import collections import functools import unittest @@ -28,12 +29,6 @@ from test.unit.descriptor import ( ) try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - -try: # added in python 3.3 from unittest.mock import patch, Mock except ImportError: @@ -328,7 +323,7 @@ class TestHiddenServiceDescriptorV3(unittest.TestCase): # include optional parameters - desc = InnerLayer.create(OrderedDict(( + desc = InnerLayer.create(collections.OrderedDict(( ('intro-auth-required', 'ed25519'), ('single-onion-service', ''), ))) diff --git a/test/unit/descriptor/networkstatus/document_v3.py b/test/unit/descriptor/networkstatus/document_v3.py index ce7cfdb7..93f98d77 100644 --- a/test/unit/descriptor/networkstatus/document_v3.py +++ b/test/unit/descriptor/networkstatus/document_v3.py @@ -2,6 +2,7 @@ Unit tests for the NetworkStatusDocumentV3 of stem.descriptor.networkstatus. """ +import collections import datetime import io import unittest @@ -31,12 +32,6 @@ from stem.descriptor.router_status_entry import ( from test.unit.descriptor import get_resource -try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - BANDWIDTH_WEIGHT_ENTRIES = ( 'Wbd', 'Wbe', 'Wbg', 'Wbm', 'Wdb', @@ -869,7 +864,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= Parses the parameters attributes. """ - document = NetworkStatusDocumentV3.create(OrderedDict([ + document = NetworkStatusDocumentV3.create(collections.OrderedDict([ ('vote-status', 'vote'), ('recommended-client-protocols', 'HSDir=1 HSIntro=3'), ('recommended-relay-protocols', 'Cons=1 Desc=1'), @@ -1231,7 +1226,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= COMMITMENT_1 = '1 sha3-256 4CAEC248004A0DC6CE86EBD5F608C9B05500C70C AAAAAFd4/kAaklgYr4ijHZjXXy/B354jQfL31BFhhE46nuOHSPITyw== AAAAAFd4/kCpZeis3yJyr//rz8hXCeeAhHa4k3lAcAiMJd1vEMTPuw==' COMMITMENT_2 = '1 sha3-256 598536A9DD4E6C0F18B4AD4B88C7875A0A29BA31 AAAAAFd4/kC7S920awC5/HF5RfX4fKZtYqjm6qMh9G91AcjZm13DQQ==' - authority = DirectoryAuthority.create(OrderedDict([ + authority = DirectoryAuthority.create(collections.OrderedDict([ ('shared-rand-participate', ''), ('shared-rand-commit', '%s\nshared-rand-commit %s' % (COMMITMENT_1, COMMITMENT_2)), ('shared-rand-previous-value', '8 hAQLxyt0U3gu7QR2owixRCbIltcyPrz3B0YBfUshOkE='), diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py index c428d6b2..7ee942e0 100644 --- a/test/unit/descriptor/router_status_entry.py +++ b/test/unit/descriptor/router_status_entry.py @@ -2,6 +2,7 @@ Unit tests for stem.descriptor.router_status_entry. """ +import collections import datetime import functools import unittest @@ -27,12 +28,6 @@ from stem.descriptor.router_status_entry import ( _base64_to_hex, ) -try: - # Added in 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - ENTRY_WITHOUT_ED25519 = """\ r seele AAoQ1DAR6kkoo19hBAX5K0QztNw m0ynPuwzSextzsiXYJYA0Hce+Cs 2015-08-23 00:26:35 73.15.150.172 9001 0 s Running Stable Valid @@ -258,7 +253,7 @@ class TestRouterStatusEntry(unittest.TestCase): Parse a router status entry with an IPv6 address. """ - expected_protocols = OrderedDict(( + expected_protocols = collections.OrderedDict(( ('Cons', [1]), ('Desc', [1]), ('DirCache', [1]), diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index d87d51e9..3878c8af 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -2,6 +2,7 @@ Unit tests for stem.descriptor.server_descriptor. """ +import collections import datetime import functools import hashlib @@ -32,12 +33,6 @@ from test.unit.descriptor import ( ) try: - # Added in 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - -try: # added in python 3.3 from unittest.mock import Mock, patch except ImportError: @@ -288,7 +283,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= exc_msg = 'Server descriptor lacks a fingerprint. This is an optional field, but required to make a router status entry.' self.assertRaisesWith(ValueError, exc_msg, desc_without_fingerprint.make_router_status_entry) - desc = RelayDescriptor.create(OrderedDict(( + desc = RelayDescriptor.create(collections.OrderedDict(( ('router', 'caerSidi 71.35.133.197 9001 0 0'), ('published', '2012-02-29 04:03:19'), ('fingerprint', '4F0C 867D F0EF 6816 0568 C826 838F 482C EA7C FE44'), diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py index 06b4510d..00965187 100644 --- a/test/unit/directory/fallback.py +++ b/test/unit/directory/fallback.py @@ -2,6 +2,7 @@ Unit tests for stem.directory.Fallback. """ +import collections import io import re import tempfile @@ -12,12 +13,6 @@ import stem.directory import stem.util.conf try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - -try: # added in python 3.3 from unittest.mock import patch, Mock except ImportError: @@ -58,7 +53,7 @@ URL: https:onionoo.torproject.orguptime?first_seen_days=30-&flag=V2Dir&type=rela /* ===== */ """ -HEADER = OrderedDict(( +HEADER = collections.OrderedDict(( ('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'), @@ -75,7 +70,7 @@ class TestFallback(unittest.TestCase): 'nickname': 'rueckgrat', 'has_extrainfo': True, 'orport_v6': ('2a01:4f8:162:51e2::2', 9001), - 'header': OrderedDict(( + 'header': collections.OrderedDict(( ('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'), diff --git a/test/unit/manual.py b/test/unit/manual.py index 72f847e5..b7a2de6f 100644 --- a/test/unit/manual.py +++ b/test/unit/manual.py @@ -2,6 +2,7 @@ Unit testing for the stem.manual module. """ +import collections import io import os import sqlite3 @@ -25,12 +26,6 @@ try: except ImportError: from mock import Mock, patch -try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' EXAMPLE_MAN_PATH = os.path.join(os.path.dirname(__file__), 'tor_man_example') UNKNOWN_OPTIONS_MAN_PATH = os.path.join(os.path.dirname(__file__), 'tor_man_with_unknown') @@ -59,7 +54,7 @@ EXPECTED_FILES = { '$HOME/.torrc': 'Fallback location for torrc, if @CONFDIR@/torrc is not found.', } -EXPECTED_CONFIG_OPTIONS = OrderedDict() +EXPECTED_CONFIG_OPTIONS = collections.OrderedDict() EXPECTED_CONFIG_OPTIONS['BandwidthRate'] = stem.manual.ConfigOption( name = 'BandwidthRate', @@ -315,4 +310,4 @@ class TestManual(unittest.TestCase): self.assertEqual({}, manual.commandline_options) self.assertEqual({}, manual.signals) self.assertEqual({}, manual.files) - self.assertEqual(OrderedDict(), manual.config_options) + self.assertEqual(collections.OrderedDict(), manual.config_options)

1 0

[stem/master] Drop lru_cache copy
by atagar＠torproject.org 05 Jan '20

05 Jan '20

commit 2e0fb02f938c6c2da3b31780753df908655a6ba8 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 3 13:56:14 2020 -0800 Drop lru_cache copy Python added lru_cache as a builtin in Python 3.2. We included a copy for Python 2.x compatibity and as such is no longer required. --- stem/descriptor/extrainfo_descriptor.py | 7 +- stem/descriptor/hidden_service.py | 8 +- stem/descriptor/microdescriptor.py | 8 +- stem/descriptor/server_descriptor.py | 13 +-- stem/exit_policy.py | 20 ++-- stem/interpreter/autocomplete.py | 9 +- stem/interpreter/help.py | 9 +- stem/manual.py | 8 +- stem/prereq.py | 15 --- stem/util/lru_cache.py | 182 -------------------------------- stem/util/proc.py | 12 +-- stem/version.py | 8 +- 12 files changed, 31 insertions(+), 268 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 17082f88..f53d9502 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -100,11 +100,6 @@ from stem.descriptor import ( _random_crypto_blob, ) -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - # known statuses for dirreq-v2-resp and dirreq-v3-resp... DirResponse = stem.util.enum.Enum( ('OK', 'ok'), @@ -950,7 +945,7 @@ class RelayExtraInfoDescriptor(ExtraInfoDescriptor): def create(cls, attr = None, exclude = (), validate = True, sign = False, signing_key = None): return cls(cls.content(attr, exclude, sign, signing_key), validate = validate) - @lru_cache() + @functools.lru_cache() def digest(self, hash_type = DigestHash.SHA1, encoding = DigestEncoding.HEX): if hash_type == DigestHash.SHA1: # our digest is calculated from everything except our signature diff --git a/stem/descriptor/hidden_service.py b/stem/descriptor/hidden_service.py index e5f82861..7994bffb 100644 --- a/stem/descriptor/hidden_service.py +++ b/stem/descriptor/hidden_service.py @@ -35,6 +35,7 @@ import base64 import binascii import collections import datetime +import functools import hashlib import io import os @@ -70,11 +71,6 @@ from stem.descriptor import ( _random_crypto_blob, ) -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - try: from cryptography.hazmat.backends.openssl.backend import backend X25519_AVAILABLE = hasattr(backend, 'x25519_supported') and backend.x25519_supported() @@ -745,7 +741,7 @@ class HiddenServiceDescriptorV2(BaseHiddenServiceDescriptor): else: self._entries = entries - @lru_cache() + @functools.lru_cache() def introduction_points(self, authentication_cookie = None): """ Provided this service's introduction points. diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index 81bcb43c..17d06d90 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -64,6 +64,7 @@ Doing the same is trivial with server descriptors... Microdescriptor - Tor microdescriptor. """ +import functools import hashlib import stem.exit_policy @@ -88,11 +89,6 @@ from stem.descriptor.router_status_entry import ( _parse_p_line, ) -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - REQUIRED_FIELDS = ( 'onion-key', ) @@ -305,7 +301,7 @@ class Microdescriptor(Descriptor): else: raise NotImplementedError('Microdescriptor digests are only available in sha1 and sha256, not %s' % hash_type) - @lru_cache() + @functools.lru_cache() def get_annotations(self): """ Provides content that appeared prior to the descriptor. If this comes from diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 1d9adf32..84ba8b65 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -92,11 +92,6 @@ from stem.descriptor import ( _random_crypto_blob, ) -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - # relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( 'router', @@ -668,7 +663,7 @@ class ServerDescriptor(Descriptor): raise NotImplementedError('Unsupported Operation: this should be implemented by the ServerDescriptor subclass') - @lru_cache() + @functools.lru_cache() def get_annotations(self): """ Provides content that appeared prior to the descriptor. If this comes from @@ -910,7 +905,7 @@ class RelayDescriptor(ServerDescriptor): def create(cls, attr = None, exclude = (), validate = True, sign = False, signing_key = None, exit_policy = None): return cls(cls.content(attr, exclude, sign, signing_key, exit_policy), validate = validate, skip_crypto_validation = not sign) - @lru_cache() + @functools.lru_cache() def digest(self, hash_type = DigestHash.SHA1, encoding = DigestEncoding.HEX): """ Provides the digest of our descriptor's content. @@ -967,7 +962,7 @@ class RelayDescriptor(ServerDescriptor): return RouterStatusEntryV3.create(attr) - @lru_cache() + @functools.lru_cache() def _onion_key_crosscert_digest(self): """ Provides the digest of the onion-key-crosscert data. This consists of the @@ -1051,7 +1046,7 @@ class BridgeDescriptor(ServerDescriptor): return self.get_scrubbing_issues() == [] - @lru_cache() + @functools.lru_cache() def get_scrubbing_issues(self): """ Provides issues with our scrubbing. diff --git a/stem/exit_policy.py b/stem/exit_policy.py index ddaf719c..f9d7e6e0 100644 --- a/stem/exit_policy.py +++ b/stem/exit_policy.py @@ -67,6 +67,7 @@ exiting to a destination is permissible or not. For instance... from __future__ import absolute_import +import functools import re import socket import zlib @@ -77,11 +78,6 @@ import stem.util.connection import stem.util.enum import stem.util.str_tools -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - AddressType = stem.util.enum.Enum(('WILDCARD', 'Wildcard'), ('IPv4', 'IPv4'), ('IPv6', 'IPv6')) # Addresses aliased by the 'private' policy. From the tor man page... @@ -271,7 +267,7 @@ class ExitPolicy(object): self._is_allowed_default = True - @lru_cache() + @functools.lru_cache() def can_exit_to(self, address = None, port = None, strict = False): """ Checks if this policy allows exiting to a given destination or not. If the @@ -295,7 +291,7 @@ class ExitPolicy(object): return self._is_allowed_default - @lru_cache() + @functools.lru_cache() def is_exiting_allowed(self): """ Provides **True** if the policy allows exiting whatsoever, **False** @@ -317,7 +313,7 @@ class ExitPolicy(object): return self._is_allowed_default - @lru_cache() + @functools.lru_cache() def summary(self): """ Provides a short description of our policy chain, similar to a @@ -520,7 +516,7 @@ class ExitPolicy(object): for rule in self._get_rules(): yield rule - @lru_cache() + @functools.lru_cache() def __str__(self): return ', '.join([str(rule) for rule in self._get_rules()]) @@ -873,7 +869,7 @@ class ExitPolicyRule(object): return self._is_default_suffix - @lru_cache() + @functools.lru_cache() def __str__(self): """ Provides the string representation of our policy. This does not @@ -917,13 +913,13 @@ class ExitPolicyRule(object): return label - @lru_cache() + @functools.lru_cache() def _get_mask_bin(self): # provides an integer representation of our mask return int(stem.util.connection._address_to_binary(self.get_mask(False)), 2) - @lru_cache() + @functools.lru_cache() def _get_address_bin(self): # provides an integer representation of our address diff --git a/stem/interpreter/autocomplete.py b/stem/interpreter/autocomplete.py index b6c5354c..05585b48 100644 --- a/stem/interpreter/autocomplete.py +++ b/stem/interpreter/autocomplete.py @@ -5,15 +5,12 @@ Tab completion for our interpreter prompt. """ +import functools + import stem.prereq from stem.interpreter import uses_settings -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - @uses_settings def _get_commands(controller, config): @@ -84,7 +81,7 @@ class Autocompleter(object): def __init__(self, controller): self._commands = _get_commands(controller) - @lru_cache() + @functools.lru_cache() def matches(self, text): """ Provides autocompletion matches for the given text. diff --git a/stem/interpreter/help.py b/stem/interpreter/help.py index d2e08d5c..5fde9246 100644 --- a/stem/interpreter/help.py +++ b/stem/interpreter/help.py @@ -5,6 +5,8 @@ Provides our /help responses. """ +import functools + import stem.prereq from stem.interpreter import ( @@ -17,11 +19,6 @@ from stem.interpreter import ( from stem.util.term import format -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - def response(controller, arg): """ @@ -55,7 +52,7 @@ def _normalize(arg): return arg -@lru_cache() +(a)functools.lru_cache() @uses_settings def _response(controller, arg, config): if not arg: diff --git a/stem/manual.py b/stem/manual.py index b94d4e0b..2176067c 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -49,6 +49,7 @@ us what our torrc options do... """ import collections +import functools import os import shutil import sys @@ -62,11 +63,6 @@ import stem.util.enum import stem.util.log import stem.util.system -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - try: # account for urllib's change between python 2.x and 3.x import urllib.request as urllib @@ -194,7 +190,7 @@ class ConfigOption(object): return not self == other -@lru_cache() +(a)functools.lru_cache() def _config(lowercase = True): """ Provides a dictionary for our settings.cfg. This has a couple categories... diff --git a/stem/prereq.py b/stem/prereq.py index 480f070c..0de2191e 100644 --- a/stem/prereq.py +++ b/stem/prereq.py @@ -249,21 +249,6 @@ def is_mock_available(): return False -def _is_lru_cache_available(): - """ - Functools added lru_cache to the standard library in Python 3.2. Prior to - this using a bundled implementation. We're also using this with Python 3.5 - due to a buggy implementation. (:trac:`26412`) - """ - - major_version, minor_version = sys.version_info[0:2] - - if major_version == 3 and minor_version == 5: - return False - else: - return hasattr(functools, 'lru_cache') - - def _is_sha3_available(): """ Check if hashlib has sha3 support. This requires Python 3.6+ *or* the `pysha3 diff --git a/stem/util/lru_cache.py b/stem/util/lru_cache.py deleted file mode 100644 index 011d4456..00000000 --- a/stem/util/lru_cache.py +++ /dev/null @@ -1,182 +0,0 @@ -# Drop in replace for python 3.2's collections.lru_cache, from... -# http://code.activestate.com/recipes/578078-py26-and-py30-backport-of-python… -# -# ... which is under the MIT license. Stem users should *not* rely upon this -# module. It will be removed when we drop support for python 3.2 and below. - -""" -Memoization decorator that caches a function's return value. If later called -with the same arguments then the cached value is returned rather than -reevaluated. - -This is a a python 2.x port of `functools.lru_cache -<http://docs.python.org/3/library/functools.html#functools.lru_cache>`_. If -using python 3.2 or later you should use that instead. -""" - -from collections import namedtuple -from functools import update_wrapper -from threading import RLock - -_CacheInfo = namedtuple('CacheInfo', ['hits', 'misses', 'maxsize', 'currsize']) - - -class _HashedSeq(list): - __slots__ = 'hashvalue' - - def __init__(self, tup, hash=hash): - self[:] = tup - self.hashvalue = hash(tup) - - def __hash__(self): - return self.hashvalue - - -def _make_key(args, kwds, typed, - kwd_mark = (object(),), - fasttypes = set([int, str, frozenset, type(None)]), - sorted=sorted, tuple=tuple, type=type, len=len): - 'Make a cache key from optionally typed positional and keyword arguments' - key = args - if kwds: - sorted_items = sorted(kwds.items()) - key += kwd_mark - for item in sorted_items: - key += item - if typed: - key += tuple(type(v) for v in args) - if kwds: - key += tuple(type(v) for k, v in sorted_items) - elif len(key) == 1 and type(key[0]) in fasttypes: - return key[0] - return _HashedSeq(key) - - -def lru_cache(maxsize=100, typed=False): - """Least-recently-used cache decorator. - - If *maxsize* is set to None, the LRU features are disabled and the cache - can grow without bound. - - If *typed* is True, arguments of different types will be cached separately. - For example, f(3.0) and f(3) will be treated as distinct calls with - distinct results. - - Arguments to the cached function must be hashable. - - View the cache statistics named tuple (hits, misses, maxsize, currsize) with - f.cache_info(). Clear the cache and statistics with f.cache_clear(). - Access the underlying function with f.__wrapped__. - - See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used - - """ - - # Users should only access the lru_cache through its public API: - # cache_info, cache_clear, and f.__wrapped__ - # The internals of the lru_cache are encapsulated for thread safety and - # to allow the implementation to change (including a possible C version). - - def decorating_function(user_function): - - cache = dict() - stats = [0, 0] # make statistics updateable non-locally - HITS, MISSES = 0, 1 # names for the stats fields - make_key = _make_key - cache_get = cache.get # bound method to lookup key or return None - _len = len # localize the global len() function - lock = RLock() # because linkedlist updates aren't threadsafe - root = [] # root of the circular doubly linked list - root[:] = [root, root, None, None] # initialize by pointing to self - nonlocal_root = [root] # make updateable non-locally - PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields - - if maxsize == 0: - - def wrapper(*args, **kwds): - # no caching, just do a statistics update after a successful call - result = user_function(*args, **kwds) - stats[MISSES] += 1 - return result - - elif maxsize is None: - - def wrapper(*args, **kwds): - # simple caching without ordering or size limit - key = make_key(args, kwds, typed) - result = cache_get(key, root) # root used here as a unique not-found sentinel - if result is not root: - stats[HITS] += 1 - return result - result = user_function(*args, **kwds) - cache[key] = result - stats[MISSES] += 1 - return result - - else: - - def wrapper(*args, **kwds): - # size limited caching that tracks accesses by recency - key = make_key(args, kwds, typed) if kwds or typed else args - with lock: - link = cache_get(key) - if link is not None: - # record recent use of the key by moving it to the front of the list - root, = nonlocal_root - link_prev, link_next, key, result = link - link_prev[NEXT] = link_next - link_next[PREV] = link_prev - last = root[PREV] - last[NEXT] = root[PREV] = link - link[PREV] = last - link[NEXT] = root - stats[HITS] += 1 - return result - result = user_function(*args, **kwds) - with lock: - root, = nonlocal_root - if key in cache: - # getting here means that this same key was added to the - # cache while the lock was released. since the link - # update is already done, we need only return the - # computed result and update the count of misses. - pass - elif _len(cache) >= maxsize: - # use the old root to store the new key and result - oldroot = root - oldroot[KEY] = key - oldroot[RESULT] = result - # empty the oldest link and make it the new root - root = nonlocal_root[0] = oldroot[NEXT] - oldkey = root[KEY] - root[KEY] = root[RESULT] = None - # now update the cache dictionary for the new links - del cache[oldkey] - cache[key] = oldroot - else: - # put result in a new link at the front of the list - last = root[PREV] - link = [last, root, key, result] - last[NEXT] = root[PREV] = cache[key] = link - stats[MISSES] += 1 - return result - - def cache_info(): - """Report cache statistics""" - with lock: - return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache)) - - def cache_clear(): - """Clear the cache and cache statistics""" - with lock: - cache.clear() - root = nonlocal_root[0] - root[:] = [root, root, None, None] - stats[:] = [0, 0] - - wrapper.__wrapped__ = user_function - wrapper.cache_info = cache_info - wrapper.cache_clear = cache_clear - return update_wrapper(wrapper, user_function) - - return decorating_function diff --git a/stem/util/proc.py b/stem/util/proc.py index f0e0104f..ecb7f3f7 100644 --- a/stem/util/proc.py +++ b/stem/util/proc.py @@ -48,6 +48,7 @@ future, use them at your own risk.** """ import base64 +import functools import os import platform import socket @@ -68,11 +69,6 @@ try: except ImportError: IS_PWD_AVAILABLE = False -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - # os.sysconf is only defined on unix try: CLOCK_TICKS = os.sysconf(os.sysconf_names['SC_CLK_TCK']) @@ -88,7 +84,7 @@ Stat = stem.util.enum.Enum( ) -@lru_cache() +(a)functools.lru_cache() def is_available(): """ Checks if proc information is available on this platform. @@ -109,7 +105,7 @@ def is_available(): return True -@lru_cache() +(a)functools.lru_cache() def system_start_time(): """ Provides the unix time (seconds since epoch) when the system started. @@ -132,7 +128,7 @@ def system_start_time(): raise exc -@lru_cache() +(a)functools.lru_cache() def physical_memory(): """ Provides the total physical memory on the system in bytes. diff --git a/stem/version.py b/stem/version.py index 6bf0befe..71f16e2c 100644 --- a/stem/version.py +++ b/stem/version.py @@ -84,6 +84,7 @@ easily parsed and compared, for instance... ===================================== =========== """ +import functools import os import re @@ -92,11 +93,6 @@ import stem.util import stem.util.enum import stem.util.system -if stem.prereq._is_lru_cache_available(): - from functools import lru_cache -else: - from stem.util.lru_cache import lru_cache - # cache for the get_system_tor_version function VERSION_CACHE = {} @@ -150,7 +146,7 @@ def get_system_tor_version(tor_cmd = 'tor'): return VERSION_CACHE[tor_cmd] -@lru_cache() +(a)functools.lru_cache() def _get_version(version_str): return Version(version_str)

1 0

[stem/master] Resume 'GETINFO status/fresh-relay-descs' test
by atagar＠torproject.org 05 Jan '20

05 Jan '20

commit 4e21371e88662992fa21bc1f845c05c16ccee4f4 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 3 15:55:42 2020 -0800 Resume 'GETINFO status/fresh-relay-descs' test To work around these failures Nick suggested adding an Address line to our torrc... https://trac.torproject.org/projects/tor/ticket/32873 Specifying localhost ('Address 127.0.0.1') didn't work, but anecdotally seems as though supplying a dummy address like this does. This reverts commit 983afa8a67e8ef013ff8c03afbd75d0219ea1188. --- test/integ/control/controller.py | 22 ++++++++++++++++++++++ test/settings.cfg | 1 + 2 files changed, 23 insertions(+) diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index 428f516e..a32f66f0 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -256,6 +256,28 @@ class TestController(unittest.TestCase): self.assertEqual({}, controller.get_info([], {})) @test.require.controller + def test_getinfo_freshrelaydescs(self): + """ + Exercises 'GETINFO status/fresh-relay-descs'. + """ + + with test.runner.get_runner().get_tor_controller() as controller: + response = controller.get_info('status/fresh-relay-descs') + div = response.find('\nextra-info ') + nickname = controller.get_conf('Nickname') + + if div == -1: + self.fail('GETINFO response should have both a server and extrainfo descriptor:\n%s' % response) + + server_desc = stem.descriptor.server_descriptor.ServerDescriptor(response[:div], validate = True) + extrainfo_desc = stem.descriptor.extrainfo_descriptor.ExtraInfoDescriptor(response[div:], validate = True) + + self.assertEqual(nickname, server_desc.nickname) + self.assertEqual(nickname, extrainfo_desc.nickname) + self.assertEqual(controller.get_info('address'), server_desc.address) + self.assertEqual(test.runner.ORPORT, server_desc.or_port) + + @test.require.controller @test.require.online def test_getinfo_dir_status(self): """ diff --git a/test/settings.cfg b/test/settings.cfg index 2c18110f..9c746234 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -25,6 +25,7 @@ integ.torrc |SocksPort [SOCKS_PORT] |ORPort [OR_PORT] | +|Address 1.2.3.4 |DataDirectory [DATA_DIR] |Log notice stdout |Log debug file [DATA_DIR]/tor_log

1 0

[stem/master] Drop is_python_3 checks
by atagar＠torproject.org 05 Jan '20

05 Jan '20

commit 1f2cc309a0785d2d5885cf16fb739707ead195c2 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 3 15:10:07 2020 -0800 Drop is_python_3 checks These checks now always evaluate to 'true'. Dropping these checks and their alternate code paths. --- stem/client/cell.py | 4 ++-- stem/client/datatype.py | 4 ++-- stem/control.py | 10 ++++----- stem/descriptor/__init__.py | 9 +++----- stem/descriptor/reader.py | 2 +- stem/descriptor/router_status_entry.py | 4 +--- stem/exit_policy.py | 8 +++---- stem/interpreter/__init__.py | 4 ++-- stem/manual.py | 3 +-- stem/prereq.py | 29 ------------------------ stem/response/__init__.py | 13 +++++------ stem/response/events.py | 20 +++++++---------- stem/response/getinfo.py | 3 +-- stem/response/protocolinfo.py | 4 +--- stem/socket.py | 5 ++--- stem/util/__init__.py | 38 ++----------------------------- stem/util/conf.py | 4 +--- stem/util/connection.py | 4 ++-- stem/util/enum.py | 2 +- stem/util/log.py | 3 +-- stem/util/str_tools.py | 39 +++++++++++--------------------- stem/util/system.py | 2 +- test/integ/process.py | 10 +++------ test/task.py | 2 +- test/unit/descriptor/collector.py | 23 +++++++------------ test/unit/descriptor/remote.py | 41 +++++++++++++--------------------- test/unit/directory/authority.py | 6 ++--- test/unit/directory/fallback.py | 12 +++++----- test/unit/manual.py | 7 +++--- test/unit/tutorial_examples.py | 16 +++++-------- test/unit/util/connection.py | 7 +++--- 31 files changed, 105 insertions(+), 233 deletions(-) diff --git a/stem/client/cell.py b/stem/client/cell.py index 57a71749..83888556 100644 --- a/stem/client/cell.py +++ b/stem/client/cell.py @@ -354,10 +354,10 @@ class RelayCell(CircuitCell): digest_packed = digest.digest()[:RELAY_DIGEST_SIZE.size] digest = RELAY_DIGEST_SIZE.unpack(digest_packed) - elif stem.util._is_str(digest): + elif isinstance(digest, (bytes, str)): digest_packed = digest[:RELAY_DIGEST_SIZE.size] digest = RELAY_DIGEST_SIZE.unpack(digest_packed) - elif stem.util._is_int(digest): + elif isinstance(digest, int): pass else: raise ValueError('RELAY cell digest must be a hash, string, or int but was a %s' % type(digest).__name__) diff --git a/stem/client/datatype.py b/stem/client/datatype.py index 367074b2..0b39d1b5 100644 --- a/stem/client/datatype.py +++ b/stem/client/datatype.py @@ -183,7 +183,7 @@ class _IntegerEnum(stem.util.enum.Enum): Provides the (enum, int_value) tuple for a given value. """ - if stem.util._is_int(val): + if isinstance(val, int): return self._int_to_enum.get(val, self.UNKNOWN), val elif val in self: return val, self._enum_to_int.get(val, val) @@ -402,7 +402,7 @@ class Size(Field): try: packed = struct.pack(self.format, content) except struct.error: - if not stem.util._is_int(content): + if not isinstance(content, int): raise ValueError('Size.pack encodes an integer, but was a %s' % type(content).__name__) elif content < 0: raise ValueError('Packed values must be positive (attempted to pack %i as a %s)' % (content, self.name)) diff --git a/stem/control.py b/stem/control.py index 25a65f43..c046755c 100644 --- a/stem/control.py +++ b/stem/control.py @@ -1160,7 +1160,7 @@ class Controller(BaseController): start_time = time.time() reply = {} - if stem.util._is_str(params): + if isinstance(params, (bytes, str)): is_multiple = False params = set([params]) else: @@ -1205,7 +1205,7 @@ class Controller(BaseController): # usually we want unicode values under python 3.x - if stem.prereq.is_python_3() and not get_bytes: + if not get_bytes: response.entries = dict((k, stem.util.str_tools._to_unicode(v)) for (k, v) in response.entries.items()) reply.update(response.entries) @@ -2308,7 +2308,7 @@ class Controller(BaseController): start_time = time.time() reply = {} - if stem.util._is_str(params): + if isinstance(params, (bytes, str)): params = [params] # remove strings which contain only whitespace @@ -3476,7 +3476,7 @@ class Controller(BaseController): * :class:`stem.InvalidArguments` if features passed were invalid """ - if stem.util._is_str(features): + if isinstance(features, (bytes, str)): features = [features] response = self.msg('USEFEATURE %s' % ' '.join(features)) @@ -3631,7 +3631,7 @@ class Controller(BaseController): args = [circuit_id] - if stem.util._is_str(path): + if isinstance(path, (bytes, str)): path = [path] if path: diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index bab339ac..ec299289 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -357,7 +357,7 @@ def parse_file(descriptor_file, descriptor_type = None, validate = False, docume handler = None - if stem.util._is_str(descriptor_file): + if isinstance(descriptor_file, (bytes, str)): if stem.util.system.is_tarfile(descriptor_file): handler = _parse_file_for_tar_path else: @@ -1157,10 +1157,7 @@ class Descriptor(object): return super(Descriptor, self).__getattribute__(name) def __str__(self): - if stem.prereq.is_python_3(): - return stem.util.str_tools._to_unicode(self._raw_contents) - else: - return self._raw_contents + return stem.util.str_tools._to_unicode(self._raw_contents) def _compare(self, other, method): if type(self) != type(other): @@ -1234,7 +1231,7 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi content = None if skip else [] ending_keyword = None - if stem.util._is_str(keywords): + if isinstance(keywords, (bytes, str)): keywords = (keywords,) if ignore_first: diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index b4cc4279..ebd41c73 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -270,7 +270,7 @@ class DescriptorReader(object): """ def __init__(self, target, validate = False, follow_links = False, buffer_size = 100, persistence_path = None, document_handler = stem.descriptor.DocumentHandler.ENTRIES, **kwargs): - self._targets = [target] if stem.util._is_str(target) else target + self._targets = [target] if isinstance(target, (bytes, str)) else target # expand any relative paths we got diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index cf1c52f5..d248774a 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -374,9 +374,7 @@ def _base64_to_hex(identity, check_if_fingerprint = True): raise ValueError("Unable to decode identity string '%s'" % identity) fingerprint = binascii.hexlify(identity_decoded).upper() - - if stem.prereq.is_python_3(): - fingerprint = stem.util.str_tools._to_unicode(fingerprint) + fingerprint = stem.util.str_tools._to_unicode(fingerprint) if check_if_fingerprint: if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): diff --git a/stem/exit_policy.py b/stem/exit_policy.py index f9d7e6e0..0d1e7e42 100644 --- a/stem/exit_policy.py +++ b/stem/exit_policy.py @@ -124,7 +124,7 @@ def get_config_policy(rules, ip_address = None): elif ip_address and stem.util.connection.is_valid_ipv6_address(ip_address, allow_brackets = True) and not (ip_address[0] == '[' and ip_address[-1] == ']'): ip_address = '[%s]' % ip_address # ExitPolicy validation expects IPv6 addresses to be bracketed - if stem.util._is_str(rules): + if isinstance(rules, (bytes, str)): rules = rules.split(',') result = [] @@ -238,7 +238,7 @@ class ExitPolicy(object): # sanity check the types for rule in rules: - if not stem.util._is_str(rule) and not isinstance(rule, ExitPolicyRule): + if not isinstance(rule, (bytes, str)) and not isinstance(rule, ExitPolicyRule): raise TypeError('Exit policy rules can only contain strings or ExitPolicyRules, got a %s (%s)' % (type(rule), rules)) # Unparsed representation of the rules we were constructed with. Our @@ -249,7 +249,7 @@ class ExitPolicy(object): is_all_str = True for rule in rules: - if not stem.util._is_str(rule): + if not isinstance(rule, (bytes, str)): is_all_str = False if rules and is_all_str: @@ -466,7 +466,7 @@ class ExitPolicy(object): if isinstance(rule, bytes): rule = stem.util.str_tools._to_unicode(rule) - if stem.util._is_str(rule): + if isinstance(rule, (bytes, str)): if not rule.strip(): continue diff --git a/stem/interpreter/__init__.py b/stem/interpreter/__init__.py index 30af3f62..fc0c2cf3 100644 --- a/stem/interpreter/__init__.py +++ b/stem/interpreter/__init__.py @@ -171,14 +171,14 @@ def main(): while True: try: prompt = '... ' if interpreter.is_multiline_context else PROMPT - user_input = input(prompt) if stem.prereq.is_python_3() else raw_input(prompt) + user_input = input(prompt) interpreter.run_command(user_input, print_response = True) except stem.SocketClosed: if showed_close_confirmation: print(format('Unable to run tor commands. The control connection has been closed.', *ERROR_OUTPUT)) else: prompt = format("Tor's control port has closed. Do you want to continue this interpreter? (y/n) ", *HEADER_BOLD_OUTPUT) - user_input = input(prompt) if stem.prereq.is_python_3() else raw_input(prompt) + user_input = input(prompt) print('') # blank line if user_input.lower() in ('y', 'yes'): diff --git a/stem/manual.py b/stem/manual.py index 2176067c..d9ad5aa7 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -678,8 +678,7 @@ def _get_categories(content): # \u2014 - extra long dash # \xb7 - centered dot - char_for = chr if stem.prereq.is_python_3() else unichr - line = line.replace(char_for(0x2019), "'").replace(char_for(0x2014), '-').replace(char_for(0xb7), '*') + line = line.replace(chr(0x2019), "'").replace(chr(0x2014), '-').replace(chr(0xb7), '*') if line and not line.startswith(' '): if category: diff --git a/stem/prereq.py b/stem/prereq.py index d748c2ab..bd006bd0 100644 --- a/stem/prereq.py +++ b/stem/prereq.py @@ -12,7 +12,6 @@ stem will still read descriptors - just without signature checks. :: check_requirements - checks for minimum requirements for running stem - is_python_3 - checks if python 3.0 or later is available is_sqlite_available - checks if the sqlite3 module is available is_crypto_available - checks if the cryptography module is available is_zstd_available - checks if the zstd module is available @@ -50,34 +49,6 @@ def check_requirements(): raise ImportError('stem requires python version 3.6 or greater') -def is_python_27(): - """ - Checks if we're running python 2.7 or above (including the 3.x series). - - .. deprecated:: 1.5.0 - Stem 2.x will remove this method along with Python 2.x support. - - :returns: **True** if we meet this requirement and **False** otherwise - """ - - major_version, minor_version = sys.version_info[0:2] - - return major_version > 2 or (major_version == 2 and minor_version >= 7) - - -def is_python_3(): - """ - Checks if we're in the 3.0 - 3.x range. - - .. deprecated:: 1.8.0 - Stem 2.x will remove this method along with Python 2.x support. - - :returns: **True** if we meet this requirement and **False** otherwise - """ - - return sys.version_info[0] == 3 - - def is_pypy(): """ Checks if we're running PyPy. diff --git a/stem/response/__init__.py b/stem/response/__init__.py index 1a5b2b45..2fbb9c48 100644 --- a/stem/response/__init__.py +++ b/stem/response/__init__.py @@ -229,7 +229,7 @@ class ControlMessage(object): :returns: **list** of (str, str, str) tuples for the components of this message """ - if stem.prereq.is_python_3() and not get_bytes: + if not get_bytes: return [(code, div, stem.util.str_tools._to_unicode(content)) for (code, div, content) in self._parsed_content] else: return list(self._parsed_content) @@ -246,7 +246,7 @@ class ControlMessage(object): :returns: **str** of the socket data used to generate this message """ - if stem.prereq.is_python_3() and not get_bytes: + if not get_bytes: return stem.util.str_tools._to_unicode(self._raw_content) else: return self._raw_content @@ -286,8 +286,7 @@ class ControlMessage(object): """ for _, _, content in self._parsed_content: - if stem.prereq.is_python_3(): - content = stem.util.str_tools._to_unicode(content) + content = stem.util.str_tools._to_unicode(content) yield ControlLine(content) @@ -304,9 +303,7 @@ class ControlMessage(object): """ content = self._parsed_content[index][2] - - if stem.prereq.is_python_3(): - content = stem.util.str_tools._to_unicode(content) + content = stem.util.str_tools._to_unicode(content) return ControlLine(content) @@ -534,7 +531,7 @@ def _parse_entry(line, quoted, escaped, get_bytes): next_entry = codecs.escape_decode(next_entry)[0] - if stem.prereq.is_python_3() and not get_bytes: + if not get_bytes: next_entry = stem.util.str_tools._to_unicode(next_entry) # normalize back to str if get_bytes: diff --git a/stem/response/events.py b/stem/response/events.py index e634a5d6..9b6d8393 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -23,10 +23,6 @@ QUOTED_KW_ARG = re.compile('^(.*) ([A-Za-z0-9_]+)="(.*)"$') CELL_TYPE = re.compile('^[a-z0-9_]+$') PARSE_NEWCONSENSUS_EVENTS = True -# TODO: We can remove the following when we drop python2.6 support. - -INT_TYPE = int if stem.prereq.is_python_3() else long - class Event(stem.response.ControlMessage): """ @@ -163,7 +159,7 @@ class Event(stem.response.ControlMessage): attr_values = getattr(self, attr) if attr_values: - if stem.util._is_str(attr_values): + if isinstance(attr_values, (bytes, str)): attr_values = [attr_values] for value in attr_values: @@ -284,8 +280,8 @@ class BandwidthEvent(Event): elif not self.read.isdigit() or not self.written.isdigit(): raise stem.ProtocolError("A BW event's bytes sent and received should be a positive numeric value, received: %s" % self) - self.read = INT_TYPE(self.read) - self.written = INT_TYPE(self.written) + self.read = int(self.read) + self.written = int(self.written) class BuildTimeoutSetEvent(Event): @@ -1095,8 +1091,8 @@ class StreamBwEvent(Event): elif not self.read.isdigit() or not self.written.isdigit(): raise stem.ProtocolError("A STREAM_BW event's bytes sent and received should be a positive numeric value, received: %s" % self) - self.read = INT_TYPE(self.read) - self.written = INT_TYPE(self.written) + self.read = int(self.read) + self.written = int(self.written) self.time = self._iso_timestamp(self.time) @@ -1174,8 +1170,8 @@ class ConnectionBandwidthEvent(Event): elif not tor_tools.is_valid_connection_id(self.id): raise stem.ProtocolError("Connection IDs must be one to sixteen alphanumeric characters, got '%s': %s" % (self.id, self)) - self.read = INT_TYPE(self.read) - self.written = INT_TYPE(self.written) + self.read = int(self.read) + self.written = int(self.written) self._log_if_unrecognized('conn_type', stem.ConnectionType) @@ -1247,7 +1243,7 @@ class CircuitBandwidthEvent(Event): value = getattr(self, attr) if value: - setattr(self, attr, INT_TYPE(value)) + setattr(self, attr, int(value)) class CellStatsEvent(Event): diff --git a/stem/response/getinfo.py b/stem/response/getinfo.py index 0b9766ba..27442ffd 100644 --- a/stem/response/getinfo.py +++ b/stem/response/getinfo.py @@ -53,8 +53,7 @@ class GetInfoResponse(stem.response.ControlMessage): except ValueError: raise stem.ProtocolError('GETINFO replies should only contain parameter=value mappings:\n%s' % self) - if stem.prereq.is_python_3(): - key = stem.util.str_tools._to_unicode(key) + key = stem.util.str_tools._to_unicode(key) # if the value is a multiline value then it *must* be of the form # '<key>=\n<value>' diff --git a/stem/response/protocolinfo.py b/stem/response/protocolinfo.py index 1763e59f..46f6ab4f 100644 --- a/stem/response/protocolinfo.py +++ b/stem/response/protocolinfo.py @@ -108,9 +108,7 @@ class ProtocolInfoResponse(stem.response.ControlMessage): if line.is_next_mapping('COOKIEFILE', True, True): self.cookie_path = line.pop_mapping(True, True, get_bytes = True)[1].decode(sys.getfilesystemencoding()) - - if stem.prereq.is_python_3(): - self.cookie_path = stem.util.str_tools._to_unicode(self.cookie_path) # normalize back to str + self.cookie_path = stem.util.str_tools._to_unicode(self.cookie_path) # normalize back to str elif line_type == 'VERSION': # Line format: # VersionLine = "250-VERSION" SP "Tor=" TorVersion OptArguments CRLF diff --git a/stem/socket.py b/stem/socket.py index 26e8f22e..275a55d4 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -710,9 +710,8 @@ def recv_message(control_file, arrived_at = None): status_code, divider, content = line[:3], line[3:4], line[4:-2] # strip CRLF off content - if stem.prereq.is_python_3(): - status_code = stem.util.str_tools._to_unicode(status_code) - divider = stem.util.str_tools._to_unicode(divider) + status_code = stem.util.str_tools._to_unicode(status_code) + divider = stem.util.str_tools._to_unicode(divider) # Most controller responses are single lines, in which case we don't need # so much overhead. diff --git a/stem/util/__init__.py b/stem/util/__init__.py index 226f5fbf..c6b9ce34 100644 --- a/stem/util/__init__.py +++ b/stem/util/__init__.py @@ -56,7 +56,7 @@ def _hash_value(val): # # This hack will go away when we drop Python 2.x support. - if _is_str(val): + if isinstance(val, (bytes, str)): my_hash = hash('str') else: # Hashing common builtins (ints, bools, etc) provide consistant values but many others vary their value on interpreter invokation. @@ -75,40 +75,6 @@ def _hash_value(val): return my_hash -def _is_str(val): - """ - Check if a value is a string. This will be removed when we no longer provide - backward compatibility for the Python 2.x series. - - :param object val: value to be checked - - :returns: **True** if the value is some form of string (unicode or bytes), - and **False** otherwise - """ - - if stem.prereq.is_python_3(): - return isinstance(val, (bytes, str)) - else: - return isinstance(val, (bytes, unicode)) - - -def _is_int(val): - """ - Check if a value is an integer. This will be removed when we no longer - provide backward compatibility for the Python 2.x series. - - :param object val: value to be checked - - :returns: **True** if the value is some form of integer (int or long), - and **False** otherwise - """ - - if stem.prereq.is_python_3(): - return isinstance(val, int) - else: - return isinstance(val, (int, long)) - - def datetime_to_unix(timestamp): """ Converts a utc datetime object to a unix timestamp. @@ -128,7 +94,7 @@ def _pubkey_bytes(key): Normalizes X25509 and ED25519 keys into their public key bytes. """ - if _is_str(key): + if isinstance(key, (bytes, str)): return key if not stem.prereq.is_crypto_available(): diff --git a/stem/util/conf.py b/stem/util/conf.py index 7dffe95a..1dbcc243 100644 --- a/stem/util/conf.py +++ b/stem/util/conf.py @@ -635,14 +635,12 @@ class Config(object): """ with self._contents_lock: - unicode_type = str if stem.prereq.is_python_3() else unicode - if value is None: if overwrite and key in self._contents: del self._contents[key] else: pass # no value so this is a no-op - elif isinstance(value, (bytes, unicode_type)): + elif isinstance(value, (bytes, str)): if not overwrite and key in self._contents: self._contents[key].append(value) else: diff --git a/stem/util/connection.py b/stem/util/connection.py index bd1fb5a2..7c8e864b 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -458,7 +458,7 @@ def is_valid_ipv4_address(address): if isinstance(address, bytes): address = str_tools._to_unicode(address) - elif not stem.util._is_str(address): + elif not isinstance(address, (bytes, str)): return False # checks if theres four period separated values @@ -488,7 +488,7 @@ def is_valid_ipv6_address(address, allow_brackets = False): if isinstance(address, bytes): address = str_tools._to_unicode(address) - elif not stem.util._is_str(address): + elif not isinstance(address, (bytes, str)): return False if allow_brackets: diff --git a/stem/util/enum.py b/stem/util/enum.py index 76a52a55..abaf2490 100644 --- a/stem/util/enum.py +++ b/stem/util/enum.py @@ -76,7 +76,7 @@ class Enum(object): keys, values = [], [] for entry in args: - if stem.util._is_str(entry): + if isinstance(entry, (bytes, str)): key, val = entry, _to_camel_case(entry) elif isinstance(entry, tuple) and len(entry) == 2: key, val = entry diff --git a/stem/util/log.py b/stem/util/log.py index e8d61c1d..ccf6b4ae 100644 --- a/stem/util/log.py +++ b/stem/util/log.py @@ -153,8 +153,7 @@ def escape(message): :returns: str that is escaped """ - if stem.prereq.is_python_3(): - message = stem.util.str_tools._to_unicode(message) + message = stem.util.str_tools._to_unicode(message) for pattern, replacement in (('\n', '\\n'), ('\r', '\\r'), ('\t', '\\t')): message = message.replace(pattern, replacement) diff --git a/stem/util/str_tools.py b/stem/util/str_tools.py index 58effbee..6b852f3d 100644 --- a/stem/util/str_tools.py +++ b/stem/util/str_tools.py @@ -61,30 +61,17 @@ TIME_UNITS = ( _timestamp_re = re.compile(r'(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})') -if stem.prereq.is_python_3(): - def _to_bytes_impl(msg): - if isinstance(msg, str): - return codecs.latin_1_encode(msg, 'replace')[0] - else: - return msg - - def _to_unicode_impl(msg): - if msg is not None and not isinstance(msg, str): - return msg.decode('utf-8', 'replace') - else: - return msg -else: - def _to_bytes_impl(msg): - if msg is not None and isinstance(msg, unicode): - return codecs.latin_1_encode(msg, 'replace')[0] - else: - return msg +def _to_bytes_impl(msg): + if isinstance(msg, str): + return codecs.latin_1_encode(msg, 'replace')[0] + else: + return msg - def _to_unicode_impl(msg): - if msg is not None and not isinstance(msg, unicode): - return msg.decode('utf-8', 'replace') - else: - return msg +def _to_unicode_impl(msg): + if msg is not None and not isinstance(msg, str): + return msg.decode('utf-8', 'replace') + else: + return msg def _to_bytes(msg): @@ -137,7 +124,7 @@ def _to_int(msg): :returns: **int** representation of the string """ - if stem.prereq.is_python_3() and isinstance(msg, bytes): + if isinstance(msg, bytes): # iterating over bytes in python3 provides ints rather than characters return sum([pow(256, (len(msg) - i - 1)) * c for (i, c) in enumerate(msg)]) else: @@ -508,7 +495,7 @@ def _parse_timestamp(entry): :raises: **ValueError** if the timestamp is malformed """ - if not stem.util._is_str(entry): + if not isinstance(entry, (bytes, str)): raise ValueError('parse_timestamp() input must be a str, got a %s' % type(entry)) try: @@ -534,7 +521,7 @@ def _parse_iso_timestamp(entry): :raises: **ValueError** if the timestamp is malformed """ - if not stem.util._is_str(entry): + if not isinstance(entry, (bytes, str)): raise ValueError('parse_iso_timestamp() input must be a str, got a %s' % type(entry)) # based after suggestions from... diff --git a/stem/util/system.py b/stem/util/system.py index 0d2262f5..e514c3a4 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -454,7 +454,7 @@ def is_running(command): if command_listing: command_listing = [c.strip() for c in command_listing] - if stem.util._is_str(command): + if isinstance(command, (bytes, str)): command = [command] for cmd in command: diff --git a/test/integ/process.py b/test/integ/process.py index 26346ece..88230805 100644 --- a/test/integ/process.py +++ b/test/integ/process.py @@ -106,7 +106,7 @@ def run_tor(tor_cmd, *args, **kwargs): elif not exit_status and expect_failure: raise AssertionError("Didn't expect tor to be able to start when we run: %s\n%s" % (' '.join(args), stdout)) - return stem.util.str_tools._to_unicode(stdout) if stem.prereq.is_python_3() else stdout + return stem.util.str_tools._to_unicode(stdout) class TestProcess(unittest.TestCase): @@ -177,12 +177,8 @@ class TestProcess(unittest.TestCase): # I'm not gonna even pretend to understand the following. Ported directly # from tor's test_cmdline_args.py. - if stem.prereq.is_python_3(): - output_hex = binascii.a2b_hex(stem.util.str_tools._to_bytes(output).strip()[3:]) - salt, how, hashed = output_hex[:8], output_hex[8], output_hex[9:] - else: - output_hex = binascii.a2b_hex(output.strip()[3:]) - salt, how, hashed = output_hex[:8], ord(output_hex[8]), output_hex[9:] + output_hex = binascii.a2b_hex(stem.util.str_tools._to_bytes(output).strip()[3:]) + salt, how, hashed = output_hex[:8], output_hex[8], output_hex[9:] count = (16 + (how & 15)) << ((how >> 4) + 6) stuff = salt + b'my_password' diff --git a/test/task.py b/test/task.py index 1fbcf9a4..31c7d628 100644 --- a/test/task.py +++ b/test/task.py @@ -273,7 +273,7 @@ class Task(object): self.is_successful = True output_msg = 'running' if self._is_background_task else 'done' - if self.result and self.print_result and stem.util._is_str(self.result): + if self.result and self.print_result and isinstance(self.result, (bytes, str)): output_msg = self.result elif self.print_runtime: output_msg += ' (%0.1fs)' % (time.time() - start_time) diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 3d6b2da0..acdcc0d4 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -19,9 +19,6 @@ try: except ImportError: from mock import Mock, patch -URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' - - with open(get_resource('collector/index.json'), 'rb') as index_file: EXAMPLE_INDEX_JSON = index_file.read() @@ -60,7 +57,7 @@ class TestCollector(unittest.TestCase): # tests for the CollecTor class - @patch(URL_OPEN) + @patch('urllib.request.urlopen') def test_index_plaintext(self, urlopen_mock): urlopen_mock.return_value = io.BytesIO(EXAMPLE_INDEX_JSON) @@ -68,7 +65,7 @@ class TestCollector(unittest.TestCase): self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.PLAINTEXT)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json', timeout = None) - @patch(URL_OPEN) + @patch('urllib.request.urlopen') def test_index_gzip(self, urlopen_mock): if not Compression.GZIP.available: self.skipTest('(gzip compression unavailable)') @@ -80,7 +77,7 @@ class TestCollector(unittest.TestCase): self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.GZIP)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None) - @patch(URL_OPEN) + @patch('urllib.request.urlopen') def test_index_bz2(self, urlopen_mock): if not Compression.BZ2.available: self.skipTest('(bz2 compression unavailable)') @@ -92,7 +89,7 @@ class TestCollector(unittest.TestCase): self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.BZ2)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None) - @patch(URL_OPEN) + @patch('urllib.request.urlopen') def test_index_lzma(self, urlopen_mock): if not Compression.LZMA.available: self.skipTest('(lzma compression unavailable)') @@ -104,7 +101,7 @@ class TestCollector(unittest.TestCase): self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.LZMA)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.xz', timeout = None) - @patch(URL_OPEN) + @patch('urllib.request.urlopen') def test_index_retries(self, urlopen_mock): urlopen_mock.side_effect = IOError('boom') @@ -118,21 +115,17 @@ class TestCollector(unittest.TestCase): self.assertRaisesRegexp(IOError, 'boom', collector.index) self.assertEqual(5, urlopen_mock.call_count) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json'))) + @patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(b'not json'))) def test_index_malformed_json(self): collector = CollecTor() - - if stem.prereq.is_python_3(): - self.assertRaisesRegexp(ValueError, 'Expecting value: line 1 column 1', collector.index, Compression.PLAINTEXT) - else: - self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index, Compression.PLAINTEXT) + self.assertRaisesRegexp(ValueError, 'Expecting value: line 1 column 1', collector.index, Compression.PLAINTEXT) def test_index_malformed_compression(self): for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA): if not compression.available: continue - with patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not compressed'))): + with patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(b'not compressed'))): collector = CollecTor() self.assertRaisesRegexp(IOError, 'Failed to decompress as %s' % compression, collector.index, compression) diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 7703c62c..f8421757 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -27,12 +27,6 @@ try: except ImportError: from mock import patch, Mock, MagicMock -# The urlopen() method is in a different location depending on if we're using -# python 2.x or 3.x. The 2to3 converter accounts for this in imports, but not -# mock annotations. - -URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' - TEST_RESOURCE = '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31' # Output from requesting moria1's descriptor from itself... @@ -104,16 +98,13 @@ def _dirport_mock(data, encoding = 'identity'): dirport_mock = Mock() dirport_mock().read.return_value = data - if stem.prereq.is_python_3(): - headers = HTTPMessage() + headers = HTTPMessage() - for line in HEADER.splitlines(): - key, value = line.split(': ', 1) - headers.add_header(key, encoding if key == 'Content-Encoding' else value) + for line in HEADER.splitlines(): + key, value = line.split(': ', 1) + headers.add_header(key, encoding if key == 'Content-Encoding' else value) - dirport_mock().headers = headers - else: - dirport_mock().headers = HTTPMessage(io.BytesIO(HEADER % encoding)) + dirport_mock().headers = headers return dirport_mock @@ -165,7 +156,7 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertRaisesRegexp(stem.ProtocolError, "^Response should begin with HTTP success, but was 'HTTP/1.0 500 Kaboom'", request.run) - @patch(URL_OPEN, _dirport_mock(TEST_DESCRIPTOR)) + @patch('urllib.request.urlopen', _dirport_mock(TEST_DESCRIPTOR)) def test_using_dirport(self): """ Download a descriptor through the DirPort. @@ -203,7 +194,7 @@ class TestDescriptorDownloader(unittest.TestCase): query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.LZMA, start = False) self.assertEqual([stem.descriptor.Compression.PLAINTEXT], query.compression) - @patch(URL_OPEN, _dirport_mock(read_resource('compressed_identity'), encoding = 'identity')) + @patch('urllib.request.urlopen', _dirport_mock(read_resource('compressed_identity'), encoding = 'identity')) def test_compression_plaintext(self): """ Download a plaintext descriptor. @@ -218,7 +209,7 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual(1, len(descriptors)) self.assertEqual('moria1', descriptors[0].nickname) - @patch(URL_OPEN, _dirport_mock(read_resource('compressed_gzip'), encoding = 'gzip')) + @patch('urllib.request.urlopen', _dirport_mock(read_resource('compressed_gzip'), encoding = 'gzip')) def test_compression_gzip(self): """ Download a gip compressed descriptor. @@ -233,7 +224,7 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual(1, len(descriptors)) self.assertEqual('moria1', descriptors[0].nickname) - @patch(URL_OPEN, _dirport_mock(read_resource('compressed_zstd'), encoding = 'x-zstd')) + @patch('urllib.request.urlopen', _dirport_mock(read_resource('compressed_zstd'), encoding = 'x-zstd')) def test_compression_zstd(self): """ Download a zstd compressed descriptor. @@ -251,7 +242,7 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual(1, len(descriptors)) self.assertEqual('moria1', descriptors[0].nickname) - @patch(URL_OPEN, _dirport_mock(read_resource('compressed_lzma'), encoding = 'x-tor-lzma')) + @patch('urllib.request.urlopen', _dirport_mock(read_resource('compressed_lzma'), encoding = 'x-tor-lzma')) def test_compression_lzma(self): """ Download a lzma compressed descriptor. @@ -269,7 +260,7 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual(1, len(descriptors)) self.assertEqual('moria1', descriptors[0].nickname) - @patch(URL_OPEN) + @patch('urllib.request.urlopen') def test_each_getter(self, dirport_mock): """ Surface level exercising of each getter method for downloading descriptors. @@ -286,7 +277,7 @@ class TestDescriptorDownloader(unittest.TestCase): downloader.get_bandwidth_file() downloader.get_detached_signatures() - @patch(URL_OPEN, _dirport_mock(TEST_DESCRIPTOR)) + @patch('urllib.request.urlopen', _dirport_mock(TEST_DESCRIPTOR)) def test_reply_headers(self): query = stem.descriptor.remote.get_server_descriptors('9695DFC35FFEB861329B9F1AB04C46397020CE31', start = False) self.assertEqual(None, query.reply_headers) # initially we don't have a reply @@ -309,7 +300,7 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual(1, len(descriptors)) self.assertEqual('moria1', descriptors[0].nickname) - @patch(URL_OPEN, _dirport_mock(TEST_DESCRIPTOR)) + @patch('urllib.request.urlopen', _dirport_mock(TEST_DESCRIPTOR)) def test_query_download(self): """ Check Query functionality when we successfully download a descriptor. @@ -334,7 +325,7 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual('9695DFC35FFEB861329B9F1AB04C46397020CE31', desc.fingerprint) self.assertEqual(TEST_DESCRIPTOR, desc.get_bytes()) - @patch(URL_OPEN, _dirport_mock(b'some malformed stuff')) + @patch('urllib.request.urlopen', _dirport_mock(b'some malformed stuff')) def test_query_with_malformed_content(self): """ Query with malformed descriptor content. @@ -361,7 +352,7 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertRaises(ValueError, query.run) - @patch(URL_OPEN) + @patch('urllib.request.urlopen') def test_query_with_timeout(self, dirport_mock): def urlopen_call(*args, **kwargs): time.sleep(0.06) @@ -396,7 +387,7 @@ class TestDescriptorDownloader(unittest.TestCase): expected_error = 'Endpoints must be an stem.ORPort, stem.DirPort, or two value tuple. ' + error_suffix self.assertRaisesWith(ValueError, expected_error, stem.descriptor.remote.Query, TEST_RESOURCE, 'server-descriptor 1.0', endpoints = endpoints) - @patch(URL_OPEN, _dirport_mock(TEST_DESCRIPTOR)) + @patch('urllib.request.urlopen', _dirport_mock(TEST_DESCRIPTOR)) def test_can_iterate_multiple_times(self): query = stem.descriptor.remote.Query( TEST_RESOURCE, diff --git a/test/unit/directory/authority.py b/test/unit/directory/authority.py index 1c2a86b4..c2ebd322 100644 --- a/test/unit/directory/authority.py +++ b/test/unit/directory/authority.py @@ -15,8 +15,6 @@ try: except ImportError: from mock import patch, Mock -URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' - AUTHORITY_GITWEB_CONTENT = b"""\ "moria1 orport=9101 " "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " @@ -53,7 +51,7 @@ class TestAuthority(unittest.TestCase): self.assertTrue(len(authorities) > 4) self.assertEqual('128.31.0.39', authorities['moria1'].address) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(AUTHORITY_GITWEB_CONTENT))) + @patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(AUTHORITY_GITWEB_CONTENT))) def test_from_remote(self): expected = { 'moria1': stem.directory.Authority( @@ -77,6 +75,6 @@ class TestAuthority(unittest.TestCase): self.assertEqual(expected, stem.directory.Authority.from_remote()) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) + @patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(b''))) def test_from_remote_empty(self): self.assertRaisesRegexp(stem.DownloadFailed, 'no content', stem.directory.Authority.from_remote) diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py index 00965187..f999943f 100644 --- a/test/unit/directory/fallback.py +++ b/test/unit/directory/fallback.py @@ -18,8 +18,6 @@ try: except ImportError: from mock import patch, Mock -URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' - FALLBACK_GITWEB_CONTENT = b"""\ /* type=fallback */ /* version=2.0.0 */ @@ -90,7 +88,7 @@ class TestFallback(unittest.TestCase): self.assertTrue(len(fallbacks) > 10) self.assertEqual('185.13.39.197', fallbacks['001524DD403D729F08F7E5D77813EF12756CFA8D'].address) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(FALLBACK_GITWEB_CONTENT))) + @patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(FALLBACK_GITWEB_CONTENT))) def test_from_remote(self): expected = { '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.directory.Fallback( @@ -117,15 +115,15 @@ class TestFallback(unittest.TestCase): self.assertEqual(expected, stem.directory.Fallback.from_remote()) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) + @patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(b''))) def test_from_remote_empty(self): self.assertRaisesRegexp(stem.DownloadFailed, 'no content', stem.directory.Fallback.from_remote) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'\n'.join(FALLBACK_GITWEB_CONTENT.splitlines()[1:])))) + @patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(b'\n'.join(FALLBACK_GITWEB_CONTENT.splitlines()[1:])))) def test_from_remote_no_header(self): self.assertRaisesRegexp(IOError, 'does not have a type field indicating it is fallback directory metadata', stem.directory.Fallback.from_remote) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(FALLBACK_GITWEB_CONTENT.replace(b'version=2.0.0', b'version')))) + @patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(FALLBACK_GITWEB_CONTENT.replace(b'version=2.0.0', b'version')))) def test_from_remote_malformed_header(self): self.assertRaisesRegexp(IOError, 'Malformed fallback directory header line: /\\* version \\*/', stem.directory.Fallback.from_remote) @@ -140,7 +138,7 @@ class TestFallback(unittest.TestCase): } for entry, expected in test_values.items(): - with patch(URL_OPEN, Mock(return_value = io.BytesIO(entry))): + with patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(entry))): self.assertRaisesRegexp(IOError, re.escape(expected), stem.directory.Fallback.from_remote) def test_persistence(self): diff --git a/test/unit/manual.py b/test/unit/manual.py index c108af19..b10ce2a0 100644 --- a/test/unit/manual.py +++ b/test/unit/manual.py @@ -26,7 +26,6 @@ try: except ImportError: from mock import Mock, patch -URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' EXAMPLE_MAN_PATH = os.path.join(os.path.dirname(__file__), 'tor_man_example') UNKNOWN_OPTIONS_MAN_PATH = os.path.join(os.path.dirname(__file__), 'tor_man_with_unknown') @@ -252,7 +251,7 @@ class TestManual(unittest.TestCase): @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', Mock(return_value = io.BytesIO()), create = True) @patch('stem.util.system.is_available', Mock(return_value = True)) - @patch(URL_OPEN, Mock(side_effect = urllib.URLError('<urlopen error [Errno -2] Name or service not known>'))) + @patch('urllib.request.urlopen', Mock(side_effect = urllib.URLError('<urlopen error [Errno -2] Name or service not known>'))) def test_download_man_page_when_download_fails(self): exc_msg = "Unable to download tor's manual from https://www.atagar.com/foo/bar to /no/such/path/tor.1.txt: <urlopen error <urlopen error [Errno -2] Name or service not known>>" self.assertRaisesWith(IOError, exc_msg, stem.manual.download_man_page, '/tmp/no_such_file', url = 'https://www.atagar.com/foo/bar') @@ -262,7 +261,7 @@ class TestManual(unittest.TestCase): @patch('stem.manual.open', Mock(return_value = io.BytesIO()), create = True) @patch('stem.util.system.call', Mock(side_effect = stem.util.system.CallError('call failed', 'a2x -f manpage /no/such/path/tor.1.txt', 1, None, None, 'call failed'))) @patch('stem.util.system.is_available', Mock(return_value = True)) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'test content'))) + @patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(b'test content'))) def test_download_man_page_when_a2x_fails(self): exc_msg = "Unable to run 'a2x -f manpage /no/such/path/tor.1.txt': call failed" self.assertRaisesWith(IOError, exc_msg, stem.manual.download_man_page, '/tmp/no_such_file', url = 'https://www.atagar.com/foo/bar') @@ -273,7 +272,7 @@ class TestManual(unittest.TestCase): @patch('stem.util.system.call') @patch('stem.util.system.is_available', Mock(return_value = True)) @patch('os.path.exists', Mock(return_value = True)) - @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'test content'))) + @patch('urllib.request.urlopen', Mock(return_value = io.BytesIO(b'test content'))) def test_download_man_page_when_successful(self, call_mock, open_mock): open_mock.side_effect = lambda path, *args: { '/no/such/path/tor.1.txt': io.BytesIO(), diff --git a/test/unit/tutorial_examples.py b/test/unit/tutorial_examples.py index de00a2b8..67a688b8 100644 --- a/test/unit/tutorial_examples.py +++ b/test/unit/tutorial_examples.py @@ -129,12 +129,6 @@ def _get_router_status(address = None, port = None, nickname = None, fingerprint class TestTutorialExamples(unittest.TestCase): - def assert_equal_unordered(self, expected, actual): - if stem.prereq.is_python_3(): - self.assertCountEqual(expected.splitlines(), actual.splitlines()) - else: - self.assertItemsEqual(expected.splitlines(), actual.splitlines()) - @patch('sys.stdout', new_callable = StringIO) @patch('stem.control.Controller.from_port', spec = Controller) def test_list_circuits(self, from_port_mock, stdout_mock): @@ -164,7 +158,7 @@ class TestTutorialExamples(unittest.TestCase): }[fingerprint] exec_documentation_example('list_circuits.py') - self.assert_equal_unordered(LIST_CIRCUITS_OUTPUT, stdout_mock.getvalue()) + self.assertCountEqual(LIST_CIRCUITS_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) @patch('sys.stdout', new_callable = StringIO) @patch('stem.control.Controller.from_port', spec = Controller) @@ -215,7 +209,7 @@ class TestTutorialExamples(unittest.TestCase): controller.get_info.return_value = 'unknown' tutorial_example(event) - self.assert_equal_unordered(EXIT_USED_OUTPUT, stdout_mock.getvalue()) + self.assertCountEqual(EXIT_USED_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) @patch('sys.stdout', new_callable = StringIO) @patch('stem.descriptor.remote.DescriptorDownloader') @@ -229,7 +223,7 @@ class TestTutorialExamples(unittest.TestCase): exec_documentation_example('outdated_relays.py') - self.assert_equal_unordered(OUTDATED_RELAYS_OUTPUT, stdout_mock.getvalue()) + self.assertCountEqual(OUTDATED_RELAYS_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) @patch('sys.stdout', new_callable = StringIO) @patch('stem.descriptor.remote.Query') @@ -270,7 +264,7 @@ class TestTutorialExamples(unittest.TestCase): exec_documentation_example('compare_flags.py') - self.assert_equal_unordered(COMPARE_FLAGS_OUTPUT, stdout_mock.getvalue()) + self.assertCountEqual(COMPARE_FLAGS_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) @patch('sys.stdout', new_callable = StringIO) @patch('stem.directory.Authority.from_cache') @@ -303,7 +297,7 @@ class TestTutorialExamples(unittest.TestCase): query_mock.side_effect = [query1, query2, query3] exec_documentation_example('votes_by_bandwidth_authorities.py') - self.assert_equal_unordered(VOTES_BY_BANDWIDTH_AUTHORITIES_OUTPUT, stdout_mock.getvalue()) + self.assertCountEqual(VOTES_BY_BANDWIDTH_AUTHORITIES_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) @patch('sys.stdout', new_callable = StringIO) @patch('stem.descriptor.parse_file') diff --git a/test/unit/util/connection.py b/test/unit/util/connection.py index 8721ff6d..73cb3c38 100644 --- a/test/unit/util/connection.py +++ b/test/unit/util/connection.py @@ -23,7 +23,6 @@ try: except ImportError: from mock import Mock, patch -URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' URL = 'https://example.unit.test.url' NETSTAT_OUTPUT = """\ @@ -177,14 +176,14 @@ _tor tor 15843 20* internet stream tcp 0x0 192.168.1.100:36174 --> class TestConnection(unittest.TestCase): - @patch(URL_OPEN) + @patch('urllib.request.urlopen') def test_download(self, urlopen_mock): urlopen_mock.return_value = io.BytesIO(b'hello') self.assertEqual(b'hello', stem.util.connection.download(URL)) urlopen_mock.assert_called_with(URL, timeout = None) - @patch(URL_OPEN) + @patch('urllib.request.urlopen') def test_download_failure(self, urlopen_mock): urlopen_mock.side_effect = urllib.URLError('boom') @@ -198,7 +197,7 @@ class TestConnection(unittest.TestCase): self.assertEqual(urllib.URLError, type(exc.error)) self.assertTrue('return urllib.urlopen(url, timeout = timeout).read()' in exc.stacktrace_str) - @patch(URL_OPEN) + @patch('urllib.request.urlopen') def test_download_retries(self, urlopen_mock): urlopen_mock.side_effect = urllib.URLError('boom')

1 0

[stem/master] Drop mock fallback
by atagar＠torproject.org 05 Jan '20

05 Jan '20

commit 5691ff000a40059a2a9812e627574ac88cc7c754 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 3 15:54:22 2020 -0800 Drop mock fallback Python 3.3 added mock as a builtin. As such we no longer need to use python 2.x's standalone mock module as a fallback. --- run_tests.py | 15 ----------- stem/prereq.py | 43 ------------------------------- test/integ/connection/connect.py | 8 ++---- test/integ/process.py | 8 ++---- test/integ/response/protocolinfo.py | 6 +---- test/integ/util/system.py | 8 ++---- test/task.py | 2 -- test/unit/connection/authentication.py | 8 ++---- test/unit/connection/connect.py | 15 +++++------ test/unit/control/controller.py | 8 ++---- test/unit/descriptor/bandwidth_file.py | 8 ++---- test/unit/descriptor/collector.py | 8 ++---- test/unit/descriptor/hidden_service_v3.py | 8 ++---- test/unit/descriptor/reader.py | 6 +---- test/unit/descriptor/remote.py | 8 ++---- test/unit/descriptor/server_descriptor.py | 8 ++---- test/unit/directory/authority.py | 6 +---- test/unit/directory/fallback.py | 6 +---- test/unit/doctest.py | 8 ++---- test/unit/exit_policy/policy.py | 6 +---- test/unit/interpreter/__init__.py | 6 +---- test/unit/interpreter/autocomplete.py | 9 ++----- test/unit/interpreter/commands.py | 8 ++---- test/unit/manual.py | 8 ++---- test/unit/response/events.py | 8 ++---- test/unit/response/protocolinfo.py | 8 ++---- test/unit/tutorial.py | 8 ++---- test/unit/tutorial_examples.py | 8 ++---- test/unit/util/connection.py | 8 ++---- test/unit/util/proc.py | 7 ++--- test/unit/util/system.py | 8 ++---- test/unit/version.py | 8 ++---- 32 files changed, 56 insertions(+), 231 deletions(-) diff --git a/run_tests.py b/run_tests.py index 8d7ea45e..fc67af3c 100755 --- a/run_tests.py +++ b/run_tests.py @@ -194,20 +194,6 @@ def main(): println('Nothing to run (for usage provide --help)\n') sys.exit() - if not stem.prereq.is_mock_available(): - try: - import mock - println(MOCK_OUT_OF_DATE_MSG % mock.__version__) - except ImportError: - println(MOCK_UNAVAILABLE_MSG) - - if stem.util.system.is_available('pip'): - println("You can get it by running 'sudo pip install mock'.") - elif stem.util.system.is_available('apt-get'): - println("You can get it by running 'sudo apt-get install python-mock'.") - - sys.exit(1) - test.task.run( 'INITIALISING', test.task.STEM_VERSION, @@ -215,7 +201,6 @@ def main(): test.task.PYTHON_VERSION, test.task.PLATFORM_VERSION, test.task.CRYPTO_VERSION, - test.task.MOCK_VERSION, test.task.PYFLAKES_VERSION, test.task.PYCODESTYLE_VERSION, test.task.CLEAN_PYC, diff --git a/stem/prereq.py b/stem/prereq.py index bd006bd0..74584165 100644 --- a/stem/prereq.py +++ b/stem/prereq.py @@ -16,7 +16,6 @@ stem will still read descriptors - just without signature checks. is_crypto_available - checks if the cryptography module is available is_zstd_available - checks if the zstd module is available is_lzma_available - checks if the lzma module is available - is_mock_available - checks if the mock module is available """ import functools @@ -162,48 +161,6 @@ def is_lzma_available(): return False -def is_mock_available(): - """ - Checks if the mock module is available. In python 3.3 and up it is a builtin - unittest module, but before this it needed to be `installed separately - <https://pypi.org/project/mock/>`_. Imports should be as follows.... - - :: - - try: - # added in python 3.3 - from unittest.mock import Mock - except ImportError: - from mock import Mock - - :returns: **True** if the mock module is available and **False** otherwise - """ - - try: - # checks for python 3.3 version - import unittest.mock - return True - except ImportError: - pass - - try: - import mock - - # check for mock's patch.dict() which was introduced in version 0.7.0 - - if not hasattr(mock.patch, 'dict'): - raise ImportError() - - # check for mock's new_callable argument for patch() which was introduced in version 0.8.0 - - if 'new_callable' not in inspect.getargspec(mock.patch).args: - raise ImportError() - - return True - except ImportError: - return False - - def _is_sha3_available(): """ Check if hashlib has sha3 support. This requires Python 3.6+ *or* the `pysha3 diff --git a/test/integ/connection/connect.py b/test/integ/connection/connect.py index f4df233f..bfeefb19 100644 --- a/test/integ/connection/connect.py +++ b/test/integ/connection/connect.py @@ -8,17 +8,13 @@ import stem.connection import test.require import test.runner +from unittest.mock import patch + try: from StringIO import StringIO except ImportError: from io import StringIO -try: - # added in python 3.3 - from unittest.mock import patch -except ImportError: - from mock import patch - class TestConnect(unittest.TestCase): @test.require.controller diff --git a/test/integ/process.py b/test/integ/process.py index 88230805..e7d2cae3 100644 --- a/test/integ/process.py +++ b/test/integ/process.py @@ -28,13 +28,9 @@ import test import test.require from contextlib import contextmanager -from stem.util.test_tools import asynchronous, assert_equal, assert_in, skip +from unittest.mock import patch, Mock -try: - # added in python 3.3 - from unittest.mock import patch, Mock -except ImportError: - from mock import patch, Mock +from stem.util.test_tools import asynchronous, assert_equal, assert_in, skip BASIC_RELAY_TORRC = """\ SocksPort 9089 diff --git a/test/integ/response/protocolinfo.py b/test/integ/response/protocolinfo.py index 5d8c74f6..917b87c3 100644 --- a/test/integ/response/protocolinfo.py +++ b/test/integ/response/protocolinfo.py @@ -14,11 +14,7 @@ import test.integ.util.system import test.require import test.runner -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch +from unittest.mock import Mock, patch class TestProtocolInfo(unittest.TestCase): diff --git a/test/integ/util/system.py b/test/integ/util/system.py index 3b48433d..4d83d695 100644 --- a/test/integ/util/system.py +++ b/test/integ/util/system.py @@ -14,13 +14,9 @@ import stem.util.system import test.require import test.runner -from stem.util.system import State, DaemonTask +from unittest.mock import Mock, patch -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch +from stem.util.system import State, DaemonTask def filter_system_call(prefixes): diff --git a/test/task.py b/test/task.py index 31c7d628..70ab5e4b 100644 --- a/test/task.py +++ b/test/task.py @@ -14,7 +14,6 @@ |- PYTHON_VERSION - checks our python version |- PLATFORM_VERSION - checks our operating system version |- CRYPTO_VERSION - checks our version of cryptography - |- MOCK_VERSION - checks our version of mock |- PYFLAKES_VERSION - checks our version of pyflakes |- PYCODESTYLE_VERSION - checks our version of pycodestyle |- CLEAN_PYC - removes any *.pyc without a corresponding *.py @@ -333,7 +332,6 @@ TOR_VERSION = Task('tor version', _check_tor_version) PYTHON_VERSION = Task('python version', _check_python_version) PLATFORM_VERSION = Task('operating system', _check_platform_version) CRYPTO_VERSION = ModuleVersion('cryptography version', 'cryptography', stem.prereq.is_crypto_available) -MOCK_VERSION = ModuleVersion('mock version', ['unittest.mock', 'mock'], stem.prereq.is_mock_available) PYFLAKES_VERSION = ModuleVersion('pyflakes version', 'pyflakes') PYCODESTYLE_VERSION = ModuleVersion('pycodestyle version', ['pycodestyle', 'pep8']) CLEAN_PYC = Task('checking for orphaned .pyc files', _clean_orphaned_pyc, (SRC_PATHS,), print_runtime = True) diff --git a/test/unit/connection/authentication.py b/test/unit/connection/authentication.py index 117b39d9..f6241e0e 100644 --- a/test/unit/connection/authentication.py +++ b/test/unit/connection/authentication.py @@ -14,15 +14,11 @@ import unittest import stem.connection import test +from unittest.mock import Mock, patch + from stem.response import ControlMessage from stem.util import log -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - class TestAuthenticate(unittest.TestCase): @patch('stem.connection.get_protocolinfo') diff --git a/test/unit/connection/connect.py b/test/unit/connection/connect.py index 37ded2f5..ec82f19f 100644 --- a/test/unit/connection/connect.py +++ b/test/unit/connection/connect.py @@ -4,20 +4,17 @@ Unit tests for the stem.connection.connect function. import unittest +import stem +import stem.connection +import stem.socket + +from unittest.mock import Mock, patch + try: from StringIO import StringIO except ImportError: from io import StringIO -try: - from mock import Mock, patch -except ImportError: - from unittest.mock import Mock, patch - -import stem -import stem.connection -import stem.socket - class TestConnect(unittest.TestCase): @patch('sys.stdout', new_callable = StringIO) diff --git a/test/unit/control/controller.py b/test/unit/control/controller.py index 94c1b65f..9628c913 100644 --- a/test/unit/control/controller.py +++ b/test/unit/control/controller.py @@ -14,17 +14,13 @@ import stem.socket import stem.util.system import stem.version +from unittest.mock import Mock, patch + from stem import ControllerError, DescriptorUnavailable, InvalidArguments, InvalidRequest, ProtocolError, UnsatisfiableRequest from stem.control import MALFORMED_EVENTS, _parse_circ_path, Listener, Controller, EventType from stem.response import ControlMessage from stem.exit_policy import ExitPolicy -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - NS_DESC = 'r %s %s u5lTXJKGsLKufRLnSyVqT7TdGYw 2012-12-30 22:02:49 77.223.43.54 9001 0\ns Fast Named Running Stable Valid\nw Bandwidth=75' TEST_TIMESTAMP = 12345 diff --git a/test/unit/descriptor/bandwidth_file.py b/test/unit/descriptor/bandwidth_file.py index bb1eeffa..9bee5f95 100644 --- a/test/unit/descriptor/bandwidth_file.py +++ b/test/unit/descriptor/bandwidth_file.py @@ -8,15 +8,11 @@ import unittest import stem.descriptor +from unittest.mock import Mock, patch + from stem.descriptor.bandwidth_file import BandwidthFile from test.unit.descriptor import get_resource -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - EXPECTED_MEASUREMENT_1 = { 'scanner': '/scanner.1/scan-data/bws-0.0:0.8-done-2019-01-13-22:55:22', 'measured_at': '1547441722', diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index acdcc0d4..99e19d7c 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -8,17 +8,13 @@ import unittest import stem.prereq +from unittest.mock import Mock, patch + from stem.descriptor import Compression, DocumentHandler from stem.descriptor.collector import CollecTor, File from test.unit.descriptor import get_resource from test.unit.descriptor.data.collector.index import EXAMPLE_INDEX -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - with open(get_resource('collector/index.json'), 'rb') as index_file: EXAMPLE_INDEX_JSON = index_file.read() diff --git a/test/unit/descriptor/hidden_service_v3.py b/test/unit/descriptor/hidden_service_v3.py index 0c172fcb..4004a94d 100644 --- a/test/unit/descriptor/hidden_service_v3.py +++ b/test/unit/descriptor/hidden_service_v3.py @@ -14,6 +14,8 @@ import stem.prereq import test.require +from unittest.mock import patch, Mock + from stem.descriptor.hidden_service import ( IntroductionPointV3, HiddenServiceDescriptorV3, @@ -28,12 +30,6 @@ from test.unit.descriptor import ( base_expect_invalid_attr_for_text, ) -try: - # added in python 3.3 - from unittest.mock import patch, Mock -except ImportError: - from mock import patch, Mock - require_sha3 = test.require.needs(stem.prereq._is_sha3_available, 'requires sha3') require_x25519 = test.require.needs(lambda: stem.descriptor.hidden_service.X25519_AVAILABLE, 'requires openssl x5509') diff --git a/test/unit/descriptor/reader.py b/test/unit/descriptor/reader.py index 589b4641..39a5d669 100644 --- a/test/unit/descriptor/reader.py +++ b/test/unit/descriptor/reader.py @@ -19,11 +19,7 @@ import stem.util.system import test.unit.descriptor -try: - # added in python 3.3 - from unittest.mock import patch -except ImportError: - from mock import patch +from unittest.mock import patch BASIC_LISTING = """ /tmp 123 diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index f8421757..98bb734b 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -13,6 +13,8 @@ import stem.descriptor.remote import stem.prereq import stem.util.str_tools +from unittest.mock import patch, Mock, MagicMock + from stem.descriptor.remote import Compression from test.unit.descriptor import read_resource @@ -21,12 +23,6 @@ try: except ImportError: from httplib import HTTPMessage # python2 -try: - # added in python 3.3 - from unittest.mock import patch, Mock, MagicMock -except ImportError: - from mock import patch, Mock, MagicMock - TEST_RESOURCE = '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31' # Output from requesting moria1's descriptor from itself... diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 3878c8af..2f448404 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -21,6 +21,8 @@ import stem.version import stem.util.str_tools import test.require +from unittest.mock import Mock, patch + from stem.client.datatype import CertType from stem.descriptor import DigestHash, DigestEncoding from stem.descriptor.certificate import ExtensionType @@ -32,12 +34,6 @@ from test.unit.descriptor import ( base_expect_invalid_attr_for_text, ) -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - TARFILE_FINGERPRINTS = set([ 'B6D83EC2D9E18B0A7A33428F8CFA9C536769E209', 'E0BD57A11F00041A9789577C53A1B784473669E4', diff --git a/test/unit/directory/authority.py b/test/unit/directory/authority.py index c2ebd322..1574bea3 100644 --- a/test/unit/directory/authority.py +++ b/test/unit/directory/authority.py @@ -9,11 +9,7 @@ import stem import stem.directory import stem.prereq -try: - # added in python 3.3 - from unittest.mock import patch, Mock -except ImportError: - from mock import patch, Mock +from unittest.mock import patch, Mock AUTHORITY_GITWEB_CONTENT = b"""\ "moria1 orport=9101 " diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py index f999943f..a7c54efb 100644 --- a/test/unit/directory/fallback.py +++ b/test/unit/directory/fallback.py @@ -12,11 +12,7 @@ import stem import stem.directory import stem.util.conf -try: - # added in python 3.3 - from unittest.mock import patch, Mock -except ImportError: - from mock import patch, Mock +from unittest.mock import patch, Mock FALLBACK_GITWEB_CONTENT = b"""\ /* type=fallback */ diff --git a/test/unit/doctest.py b/test/unit/doctest.py index 28eef0cf..84712dc2 100644 --- a/test/unit/doctest.py +++ b/test/unit/doctest.py @@ -15,13 +15,9 @@ import stem.util.system import stem.version import test -from stem.response import ControlMessage +from unittest.mock import Mock, patch -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch +from stem.response import ControlMessage EXPECTED_CIRCUIT_STATUS = """\ 20 EXTENDED $718BCEA286B531757ACAFF93AE04910EA73DE617=KsmoinOK,$649F2D0ACF418F7CFC6539AB2257EB2D5297BAFA=Eskimo BUILD_FLAGS=NEED_CAPACITY PURPOSE=GENERAL TIME_CREATED=2012-12-06T13:51:11.433755 diff --git a/test/unit/exit_policy/policy.py b/test/unit/exit_policy/policy.py index f6cf9957..0cb755ae 100644 --- a/test/unit/exit_policy/policy.py +++ b/test/unit/exit_policy/policy.py @@ -5,11 +5,7 @@ Unit tests for the stem.exit_policy.ExitPolicy class. import pickle import unittest -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch +from unittest.mock import Mock, patch from stem.exit_policy import ( DEFAULT_POLICY_RULES, diff --git a/test/unit/interpreter/__init__.py b/test/unit/interpreter/__init__.py index 7c107687..a48a19bc 100644 --- a/test/unit/interpreter/__init__.py +++ b/test/unit/interpreter/__init__.py @@ -9,11 +9,7 @@ __all__ = [ 'help', ] -try: - # added in python 3.3 - from unittest.mock import Mock -except ImportError: - from mock import Mock +from unittest.mock import Mock GETINFO_NAMES = """ info/names -- List of GETINFO options, types, and documentation. diff --git a/test/unit/interpreter/autocomplete.py b/test/unit/interpreter/autocomplete.py index 40bcab48..8971ddc7 100644 --- a/test/unit/interpreter/autocomplete.py +++ b/test/unit/interpreter/autocomplete.py @@ -1,15 +1,10 @@ import unittest -from stem.interpreter.autocomplete import _get_commands, Autocompleter +from unittest.mock import Mock +from stem.interpreter.autocomplete import _get_commands, Autocompleter from test.unit.interpreter import CONTROLLER -try: - # added in python 3.3 - from unittest.mock import Mock -except ImportError: - from mock import Mock - class TestAutocompletion(unittest.TestCase): def test_autocomplete_results_from_config(self): diff --git a/test/unit/interpreter/commands.py b/test/unit/interpreter/commands.py index 59ceadfe..412178e6 100644 --- a/test/unit/interpreter/commands.py +++ b/test/unit/interpreter/commands.py @@ -5,16 +5,12 @@ import stem import stem.response import stem.version +from unittest.mock import Mock, patch + from stem.interpreter.commands import ControlInterpreter, _get_fingerprint from stem.response import ControlMessage from test.unit.interpreter import CONTROLLER -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - EXPECTED_EVENTS_RESPONSE = """\ \x1b[34mBW 15 25\x1b[0m \x1b[34mBW 758 570\x1b[0m diff --git a/test/unit/manual.py b/test/unit/manual.py index b10ce2a0..6e80543e 100644 --- a/test/unit/manual.py +++ b/test/unit/manual.py @@ -14,18 +14,14 @@ import stem.manual import stem.util.system import test.require +from unittest.mock import Mock, patch + try: # account for urllib's change between python 2.x and 3.x import urllib.request as urllib except ImportError: import urllib2 as urllib -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - EXAMPLE_MAN_PATH = os.path.join(os.path.dirname(__file__), 'tor_man_example') UNKNOWN_OPTIONS_MAN_PATH = os.path.join(os.path.dirname(__file__), 'tor_man_with_unknown') diff --git a/test/unit/response/events.py b/test/unit/response/events.py index 82506e6d..27862054 100644 --- a/test/unit/response/events.py +++ b/test/unit/response/events.py @@ -10,16 +10,12 @@ import stem.response import stem.response.events import stem.util.log +from unittest.mock import Mock + from stem import * # enums and exceptions from stem.response import ControlMessage from stem.descriptor.router_status_entry import RouterStatusEntryV3 -try: - # added in python 3.3 - from unittest.mock import Mock -except ImportError: - from mock import Mock - # ADDRMAP event ADDRMAP = '650 ADDRMAP www.atagar.com 75.119.206.243 "2012-11-19 00:50:13" \ diff --git a/test/unit/response/protocolinfo.py b/test/unit/response/protocolinfo.py index ab6dd0eb..dd8d2160 100644 --- a/test/unit/response/protocolinfo.py +++ b/test/unit/response/protocolinfo.py @@ -11,15 +11,11 @@ import stem.util.proc import stem.util.system import stem.version +from unittest.mock import Mock, patch + from stem.response import ControlMessage from stem.response.protocolinfo import AuthMethod -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - NO_AUTH = """250-PROTOCOLINFO 1 250-AUTH METHODS=NULL 250-VERSION Tor="0.2.1.30" diff --git a/test/unit/tutorial.py b/test/unit/tutorial.py index e866f3ca..58abde9b 100644 --- a/test/unit/tutorial.py +++ b/test/unit/tutorial.py @@ -7,6 +7,8 @@ import unittest import stem.descriptor.remote +from unittest.mock import Mock, patch + from stem.control import Controller from stem.descriptor.router_status_entry import RouterStatusEntryV2, RouterStatusEntryV3 from stem.descriptor.networkstatus import NetworkStatusDocumentV3 @@ -19,12 +21,6 @@ try: except ImportError: from io import StringIO -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - OVER_THE_RIVER_OUTPUT = """\ * Connecting to tor diff --git a/test/unit/tutorial_examples.py b/test/unit/tutorial_examples.py index 67a688b8..c3359d1e 100644 --- a/test/unit/tutorial_examples.py +++ b/test/unit/tutorial_examples.py @@ -15,6 +15,8 @@ import stem.response import stem.descriptor.remote import stem.prereq +from unittest.mock import Mock, patch + from stem.control import Controller from stem.descriptor.networkstatus import NetworkStatusDocumentV3 from stem.descriptor.router_status_entry import RouterStatusEntryV3 @@ -24,12 +26,6 @@ from stem.response import ControlMessage from test.unit import exec_documentation_example -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - OPEN_FUNCTION = open # make a reference so mocking open() won't mess with us CIRC_CONTENT = '650 CIRC %d %s \ diff --git a/test/unit/util/connection.py b/test/unit/util/connection.py index 73cb3c38..047dced7 100644 --- a/test/unit/util/connection.py +++ b/test/unit/util/connection.py @@ -9,6 +9,8 @@ import unittest import stem import stem.util.connection +from unittest.mock import Mock, patch + from stem.util.connection import Resolver, Connection try: @@ -17,12 +19,6 @@ try: except ImportError: import urllib2 as urllib -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - URL = 'https://example.unit.test.url' NETSTAT_OUTPUT = """\ diff --git a/test/unit/util/proc.py b/test/unit/util/proc.py index 6ee29136..2316f669 100644 --- a/test/unit/util/proc.py +++ b/test/unit/util/proc.py @@ -7,14 +7,11 @@ import unittest import test +from unittest.mock import Mock, patch + from stem.util import proc from stem.util.connection import Connection -try: - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch - TITLE_LINE = b'sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout' TCP6_CONTENT = b"""\ diff --git a/test/unit/util/system.py b/test/unit/util/system.py index b4fb81ea..32be337c 100644 --- a/test/unit/util/system.py +++ b/test/unit/util/system.py @@ -14,13 +14,9 @@ import unittest import stem.prereq -from stem.util import system +from unittest.mock import Mock, patch -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch +from stem.util import system # Base responses for the pid_by_name tests. The 'success' and # 'multiple_results' entries are filled in by tests. diff --git a/test/unit/version.py b/test/unit/version.py index 3c21855c..abdb65c0 100644 --- a/test/unit/version.py +++ b/test/unit/version.py @@ -7,13 +7,9 @@ import unittest import stem.util.system import stem.version -from stem.version import Version +from unittest.mock import Mock, patch -try: - # added in python 3.3 - from unittest.mock import Mock, patch -except ImportError: - from mock import Mock, patch +from stem.version import Version VERSION_CMD_OUTPUT = """Mar 22 23:09:37.088 [notice] Tor v0.2.2.35 \ (git-73ff13ab3cc9570d). This is experimental software. Do not rely on it for \

1 0

[stem/master] Drop urllib fallback
by atagar＠torproject.org 05 Jan '20

05 Jan '20

commit cc20b182cd2c4deaee0b793d2fbf05d550af280d Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 3 16:15:26 2020 -0800 Drop urllib fallback Python 3.x renamed the urllib2 module to urllib.request. --- cache_fallback_directories.py | 9 ++------- cache_manual.py | 9 ++------- stem/__init__.py | 2 +- stem/descriptor/remote.py | 11 +++-------- stem/directory.py | 11 +++-------- stem/manual.py | 9 ++------- stem/util/connection.py | 9 ++------- test/integ/util/connection.py | 9 ++------- test/unit/manual.py | 9 ++------- test/unit/util/connection.py | 15 +++++---------- 10 files changed, 24 insertions(+), 69 deletions(-) diff --git a/cache_fallback_directories.py b/cache_fallback_directories.py index cb413c07..91ad40c0 100755 --- a/cache_fallback_directories.py +++ b/cache_fallback_directories.py @@ -8,22 +8,17 @@ Caches tor's latest fallback directories. import re import sys +import urllib.request import stem.directory import stem.util.system -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - GITWEB_FALLBACK_LOG = 'https://gitweb.torproject.org/tor.git/log/src/app/config/fallback_dirs.inc' FALLBACK_DIR_LINK = "href='/tor.git/commit/src/app/config/fallback_dirs.inc\\?id=([^']*)'" if __name__ == '__main__': try: - fallback_dir_page = urllib.urlopen(GITWEB_FALLBACK_LOG).read() + fallback_dir_page = urllib.request.urlopen(GITWEB_FALLBACK_LOG).read() fallback_dir_commit = re.search(FALLBACK_DIR_LINK, fallback_dir_page).group(1) except: print("Unable to determine the latest commit to edit tor's fallback directories: %s" % sys.exc_info()[1]) diff --git a/cache_manual.py b/cache_manual.py index 7fe6ea70..5bc68b57 100755 --- a/cache_manual.py +++ b/cache_manual.py @@ -8,22 +8,17 @@ Caches tor's latest manual content. Run this to pick new man page changes. import re import sys +import urllib.request import stem.manual import stem.util.system -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - GITWEB_MAN_LOG = 'https://gitweb.torproject.org/tor.git/log/doc/tor.1.txt' MAN_LOG_LINK = "href='/tor.git/commit/doc/tor.1.txt\\?id=([^']*)'" if __name__ == '__main__': try: - man_log_page = urllib.urlopen(GITWEB_MAN_LOG).read() + man_log_page = urllib.request.urlopen(GITWEB_MAN_LOG).read() man_commit = re.search(MAN_LOG_LINK, man_log_page).group(1) except: print("Unable to determine the latest commit to edit tor's man page: %s" % sys.exc_info()[1]) diff --git a/stem/__init__.py b/stem/__init__.py index 66bc24b5..3600e920 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -722,7 +722,7 @@ class SocketClosed(SocketError): class DownloadFailed(IOError): """ Inability to download a resource. Python's urllib module raises - a wide variety of undocumented exceptions (urllib2.URLError, + a wide variety of undocumented exceptions (urllib.request.URLError, socket.timeout, and others). This wraps lower level failures in a common exception type that diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index c88b338e..c7132a38 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -104,6 +104,7 @@ import socket import sys import threading import time +import urllib.request import stem import stem.client @@ -116,12 +117,6 @@ import stem.util.tor_tools from stem.util import log, str_tools -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - # TODO: remove in stem 2.x, replaced with stem.descriptor.Compression Compression = stem.util.enum.Enum( @@ -1070,8 +1065,8 @@ def _download_from_dirport(url, compression, timeout): """ try: - response = urllib.urlopen( - urllib.Request( + response = urllib.request.urlopen( + urllib.request.Request( url, headers = { 'Accept-Encoding': ', '.join(map(lambda c: c.encoding, compression)), diff --git a/stem/directory.py b/stem/directory.py index 0ca089c3..7a1ce7b8 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -42,6 +42,7 @@ import collections import os import re import sys +import urllib.request import stem import stem.util @@ -49,12 +50,6 @@ import stem.util.conf from stem.util import connection, str_tools, tor_tools -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - GITWEB_AUTHORITY_URL = 'https://gitweb.torproject.org/tor.git/plain/src/app/config/auth_dirs.inc' GITWEB_FALLBACK_URL = 'https://gitweb.torproject.org/tor.git/plain/src/app/config/fallback_dirs.inc' FALLBACK_CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_fallbacks.cfg') @@ -260,7 +255,7 @@ class Authority(Directory): @staticmethod def from_remote(timeout = 60): try: - lines = str_tools._to_unicode(urllib.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() + lines = str_tools._to_unicode(urllib.request.urlopen(GITWEB_AUTHORITY_URL, timeout = timeout).read()).splitlines() if not lines: raise IOError('no content') @@ -408,7 +403,7 @@ class Fallback(Directory): @staticmethod def from_remote(timeout = 60): try: - lines = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() + lines = str_tools._to_unicode(urllib.request.urlopen(GITWEB_FALLBACK_URL, timeout = timeout).read()).splitlines() if not lines: raise IOError('no content') diff --git a/stem/manual.py b/stem/manual.py index d9ad5aa7..fe49e3de 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -54,6 +54,7 @@ import os import shutil import sys import tempfile +import urllib.request import stem import stem.prereq @@ -63,12 +64,6 @@ import stem.util.enum import stem.util.log import stem.util.system -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - Category = stem.util.enum.Enum('GENERAL', 'CLIENT', 'RELAY', 'DIRECTORY', 'AUTHORITY', 'HIDDEN_SERVICE', 'DENIAL_OF_SERVICE', 'TESTING', 'UNKNOWN') GITWEB_MANUAL_URL = 'https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt' CACHE_PATH = os.path.join(os.path.dirname(__file__), 'cached_manual.sqlite') @@ -300,7 +295,7 @@ def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, try: try: with open(asciidoc_path, 'wb') as asciidoc_file: - request = urllib.urlopen(url, timeout = timeout) + request = urllib.request.urlopen(url, timeout = timeout) shutil.copyfileobj(request, asciidoc_file) except: exc, stacktrace = sys.exc_info()[1:3] diff --git a/stem/util/connection.py b/stem/util/connection.py index 7c8e864b..f88b3f85 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -62,6 +62,7 @@ import re import socket import sys import time +import urllib.request import stem import stem.util @@ -70,12 +71,6 @@ import stem.util.system from stem.util import conf, enum, log, str_tools -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - # Connection resolution is risky to log about since it's highly likely to # contain sensitive information. That said, it's also difficult to get right in # a platform independent fashion. To opt into the logging requried to @@ -197,7 +192,7 @@ def download(url, timeout = None, retries = None): start_time = time.time() try: - return urllib.urlopen(url, timeout = timeout).read() + return urllib.request.urlopen(url, timeout = timeout).read() except socket.timeout as exc: raise stem.DownloadTimeout(url, exc, sys.exc_info()[2], timeout) except: diff --git a/test/integ/util/connection.py b/test/integ/util/connection.py index e31ee865..861b8ba9 100644 --- a/test/integ/util/connection.py +++ b/test/integ/util/connection.py @@ -4,6 +4,7 @@ that we're running. """ import unittest +import urllib.request import stem import stem.util.connection @@ -13,12 +14,6 @@ import test.runner from stem.util.connection import Resolver -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - class TestConnection(unittest.TestCase): @test.require.ptrace @@ -58,7 +53,7 @@ class TestConnection(unittest.TestCase): self.assertEqual('Failed to download from https://no.such.testing.url (URLError): Name or service not known', str(exc)) self.assertEqual('https://no.such.testing.url', exc.url) self.assertEqual('Name or service not known', exc.error.reason.strerror) - self.assertEqual(urllib.URLError, type(exc.error)) + self.assertEqual(urllib.request.URLError, type(exc.error)) def test_connections_by_proc(self): self.check_resolver(Resolver.PROC) diff --git a/test/unit/manual.py b/test/unit/manual.py index 6e80543e..eb76d6e9 100644 --- a/test/unit/manual.py +++ b/test/unit/manual.py @@ -8,6 +8,7 @@ import os import sqlite3 import tempfile import unittest +import urllib.request import stem.prereq import stem.manual @@ -16,12 +17,6 @@ import test.require from unittest.mock import Mock, patch -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - EXAMPLE_MAN_PATH = os.path.join(os.path.dirname(__file__), 'tor_man_example') UNKNOWN_OPTIONS_MAN_PATH = os.path.join(os.path.dirname(__file__), 'tor_man_with_unknown') @@ -247,7 +242,7 @@ class TestManual(unittest.TestCase): @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', Mock(return_value = io.BytesIO()), create = True) @patch('stem.util.system.is_available', Mock(return_value = True)) - @patch('urllib.request.urlopen', Mock(side_effect = urllib.URLError('<urlopen error [Errno -2] Name or service not known>'))) + @patch('urllib.request.urlopen', Mock(side_effect = urllib.request.URLError('<urlopen error [Errno -2] Name or service not known>'))) def test_download_man_page_when_download_fails(self): exc_msg = "Unable to download tor's manual from https://www.atagar.com/foo/bar to /no/such/path/tor.1.txt: <urlopen error <urlopen error [Errno -2] Name or service not known>>" self.assertRaisesWith(IOError, exc_msg, stem.manual.download_man_page, '/tmp/no_such_file', url = 'https://www.atagar.com/foo/bar') diff --git a/test/unit/util/connection.py b/test/unit/util/connection.py index 047dced7..d848c40b 100644 --- a/test/unit/util/connection.py +++ b/test/unit/util/connection.py @@ -5,6 +5,7 @@ Unit tests for the stem.util.connection functions. import io import platform import unittest +import urllib.request import stem import stem.util.connection @@ -13,12 +14,6 @@ from unittest.mock import Mock, patch from stem.util.connection import Resolver, Connection -try: - # account for urllib's change between python 2.x and 3.x - import urllib.request as urllib -except ImportError: - import urllib2 as urllib - URL = 'https://example.unit.test.url' NETSTAT_OUTPUT = """\ @@ -181,7 +176,7 @@ class TestConnection(unittest.TestCase): @patch('urllib.request.urlopen') def test_download_failure(self, urlopen_mock): - urlopen_mock.side_effect = urllib.URLError('boom') + urlopen_mock.side_effect = urllib.request.URLError('boom') try: stem.util.connection.download(URL) @@ -190,12 +185,12 @@ class TestConnection(unittest.TestCase): self.assertEqual('Failed to download from https://example.unit.test.url (URLError): boom', str(exc)) self.assertEqual(URL, exc.url) self.assertEqual('boom', exc.error.reason) - self.assertEqual(urllib.URLError, type(exc.error)) - self.assertTrue('return urllib.urlopen(url, timeout = timeout).read()' in exc.stacktrace_str) + self.assertEqual(urllib.request.URLError, type(exc.error)) + self.assertTrue('return urllib.request.urlopen(url, timeout = timeout).read()' in exc.stacktrace_str) @patch('urllib.request.urlopen') def test_download_retries(self, urlopen_mock): - urlopen_mock.side_effect = urllib.URLError('boom') + urlopen_mock.side_effect = urllib.request.URLError('boom') self.assertRaisesRegexp(IOError, 'boom', stem.util.connection.download, URL) self.assertEqual(1, urlopen_mock.call_count)

1 0

[stem/master] Drop StringIO fallback
by atagar＠torproject.org 05 Jan '20

05 Jan '20

commit 902ce4186be618f6597baf7e57aee2c74852df09 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 3 16:23:14 2020 -0800 Drop StringIO fallback Python 3.x moved StringIO into its io module. --- run_tests.py | 8 ++------ stem/descriptor/export.py | 8 ++------ stem/interpreter/commands.py | 8 ++------ test/integ/connection/connect.py | 14 +++++--------- test/unit/connection/connect.py | 12 ++++-------- test/unit/descriptor/export.py | 8 ++------ test/unit/tutorial.py | 19 +++++++------------ test/unit/tutorial_examples.py | 18 +++++++----------- 8 files changed, 31 insertions(+), 64 deletions(-) diff --git a/run_tests.py b/run_tests.py index fc67af3c..d6ced384 100755 --- a/run_tests.py +++ b/run_tests.py @@ -7,6 +7,7 @@ Runs unit and integration tests. For usage information run this with '--help'. """ import errno +import io import importlib import logging import multiprocessing @@ -18,11 +19,6 @@ import time import traceback import unittest -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - import stem.prereq import stem.util.conf import stem.util.log @@ -427,7 +423,7 @@ def _run_test(args, test_class, output_filters): traceback.print_exc(exc) return None - test_results = StringIO() + test_results = io.StringIO() run_result = stem.util.test_tools.TimedTestRunner(test_results, verbosity = 2).run(suite) if args.verbose: diff --git a/stem/descriptor/export.py b/stem/descriptor/export.py index fea681be..4b909c97 100644 --- a/stem/descriptor/export.py +++ b/stem/descriptor/export.py @@ -17,13 +17,9 @@ Toolkit for exporting descriptors to other formats. use this modle please `let me know <https://www.atagar.com/contact/>`_. """ +import io import csv -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - import stem.descriptor import stem.prereq @@ -50,7 +46,7 @@ def export_csv(descriptors, included_fields = (), excluded_fields = (), header = :raises: **ValueError** if descriptors contain more than one descriptor type """ - output_buffer = StringIO() + output_buffer = io.StringIO() export_csv_file(output_buffer, descriptors, included_fields, excluded_fields, header) return output_buffer.getvalue() diff --git a/stem/interpreter/commands.py b/stem/interpreter/commands.py index 0f8f333c..6e61fdda 100644 --- a/stem/interpreter/commands.py +++ b/stem/interpreter/commands.py @@ -7,6 +7,7 @@ Handles making requests and formatting the responses. import code import contextlib +import io import socket import sys @@ -21,11 +22,6 @@ import stem.util.tor_tools from stem.interpreter import STANDARD_OUTPUT, BOLD_OUTPUT, ERROR_OUTPUT, uses_settings, msg from stem.util.term import format -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - MAX_EVENTS = 100 @@ -359,7 +355,7 @@ class ControlInterpreter(code.InteractiveConsole): is_tor_command = cmd in config.get('help.usage', {}) and cmd.lower() != 'events' if self._run_python_commands and not is_tor_command: - console_output = StringIO() + console_output = io.StringIO() with redirect(console_output, console_output): self.is_multiline_context = code.InteractiveConsole.push(self, command) diff --git a/test/integ/connection/connect.py b/test/integ/connection/connect.py index bfeefb19..84b4a8fc 100644 --- a/test/integ/connection/connect.py +++ b/test/integ/connection/connect.py @@ -2,6 +2,7 @@ Integration tests for the connect_* convenience functions. """ +import io import unittest import stem.connection @@ -10,15 +11,10 @@ import test.runner from unittest.mock import patch -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - class TestConnect(unittest.TestCase): @test.require.controller - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) def test_connect(self, stdout_mock): """ Basic sanity checks for the connect function. @@ -37,7 +33,7 @@ class TestConnect(unittest.TestCase): self.assertEqual('', stdout_mock.getvalue()) @test.require.controller - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) def test_connect_port(self, stdout_mock): """ Basic sanity checks for the connect_port function. @@ -59,7 +55,7 @@ class TestConnect(unittest.TestCase): self.assertEqual(control_socket, None) @test.require.controller - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) def test_connect_socket_file(self, stdout_mock): """ Basic sanity checks for the connect_socket_file function. @@ -81,7 +77,7 @@ class TestConnect(unittest.TestCase): self.assertEqual(control_socket, None) @test.require.controller - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) def test_connect_to_socks_port(self, stdout_mock): """ Common user gotcha is connecting to the SocksPort or ORPort rather than the diff --git a/test/unit/connection/connect.py b/test/unit/connection/connect.py index ec82f19f..175a1ebd 100644 --- a/test/unit/connection/connect.py +++ b/test/unit/connection/connect.py @@ -2,6 +2,7 @@ Unit tests for the stem.connection.connect function. """ +import io import unittest import stem @@ -10,14 +11,9 @@ import stem.socket from unittest.mock import Mock, patch -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - class TestConnect(unittest.TestCase): - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.util.system.is_running') @patch('os.path.exists', Mock(return_value = True)) @patch('stem.socket.ControlSocketFile', Mock(side_effect = stem.SocketError('failed'))) @@ -30,7 +26,7 @@ class TestConnect(unittest.TestCase): is_running_mock.return_value = True self._assert_connect_fails_with({}, stdout_mock, "Unable to connect to tor. Maybe it's running without a ControlPort?") - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('os.path.exists') @patch('stem.util.system.is_running', Mock(return_value = True)) @patch('stem.socket.ControlSocketFile', Mock(side_effect = stem.SocketError('failed'))) @@ -118,7 +114,7 @@ class TestConnect(unittest.TestCase): authenticate_mock.assert_any_call(control_socket, None, None) authenticate_mock.assert_any_call(control_socket, 'my_password', None) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.connection.authenticate') def test_auth_failure(self, authenticate_mock, stdout_mock): control_socket = stem.socket.ControlPort(connect = False) diff --git a/test/unit/descriptor/export.py b/test/unit/descriptor/export.py index bf7d054a..d27ed241 100644 --- a/test/unit/descriptor/export.py +++ b/test/unit/descriptor/export.py @@ -2,13 +2,9 @@ Unit tests for stem.descriptor.export. """ +import io import unittest -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - import stem.prereq from stem.descriptor.server_descriptor import RelayDescriptor, BridgeDescriptor @@ -59,7 +55,7 @@ class TestExport(unittest.TestCase): desc = RelayDescriptor.create() desc_csv = export_csv(desc) - csv_buffer = StringIO() + csv_buffer = io.StringIO() export_csv_file(csv_buffer, desc) self.assertEqual(desc_csv, csv_buffer.getvalue()) diff --git a/test/unit/tutorial.py b/test/unit/tutorial.py index 58abde9b..91a70a32 100644 --- a/test/unit/tutorial.py +++ b/test/unit/tutorial.py @@ -16,11 +16,6 @@ from stem.descriptor.server_descriptor import RelayDescriptor from stem.exit_policy import ExitPolicy from test.unit import exec_documentation_example -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - OVER_THE_RIVER_OUTPUT = """\ * Connecting to tor @@ -43,7 +38,7 @@ class TestTutorial(unittest.TestCase): stem.descriptor.remote.SINGLETON_DOWNLOADER = None - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.control.Controller.from_port', spec = Controller) def test_the_little_relay_that_could(self, from_port_mock, stdout_mock): controller = from_port_mock().__enter__() @@ -55,7 +50,7 @@ class TestTutorial(unittest.TestCase): exec_documentation_example('hello_world.py') self.assertEqual('My Tor relay has read 33406 bytes and written 29649.\n', stdout_mock.getvalue()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('shutil.rmtree') @patch('stem.control.Controller.from_port', spec = Controller) def test_over_the_river(self, from_port_mock, rmtree_mock, stdout_mock): @@ -121,7 +116,7 @@ class TestTutorial(unittest.TestCase): self.assertEqual(OVER_THE_RIVER_OUTPUT, stdout_mock.getvalue()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.descriptor.remote.DescriptorDownloader') def test_mirror_mirror_on_the_wall_1(self, downloader_mock, stdout_mock): downloader_mock().get_consensus.return_value = [RouterStatusEntryV2.create({ @@ -131,7 +126,7 @@ class TestTutorial(unittest.TestCase): exec_documentation_example('current_descriptors.py') self.assertEqual('found relay caerSidi (A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB)\n', stdout_mock.getvalue()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.control.Controller.from_port', spec = Controller) def test_mirror_mirror_on_the_wall_2(self, from_port_mock, stdout_mock): controller = from_port_mock().__enter__() @@ -142,7 +137,7 @@ class TestTutorial(unittest.TestCase): exec_documentation_example('descriptor_from_tor_control_socket.py') self.assertEqual('found relay caerSidi (A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB)\n', stdout_mock.getvalue()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('%s.open' % __name__, create = True) def test_mirror_mirror_on_the_wall_3(self, open_mock, stdout_mock): def tutorial_example(): @@ -160,7 +155,7 @@ class TestTutorial(unittest.TestCase): tutorial_example() self.assertEqual('found relay caerSidi (A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB)\n', stdout_mock.getvalue()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.descriptor.collector.get_server_descriptors') def test_mirror_mirror_on_the_wall_4(self, get_desc_mock, stdout_mock): get_desc_mock.return_value = iter([RelayDescriptor.create({ @@ -171,7 +166,7 @@ class TestTutorial(unittest.TestCase): exec_documentation_example('collector_reading.py') self.assertEqual('1 relays published an exiting policy today...\n\n caerSidi (2C3C46625698B6D67DF32BC1918AD3EE1F9906B1)\n', stdout_mock.getvalue()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.descriptor.remote.DescriptorDownloader') @patch('stem.prereq.is_crypto_available', Mock(return_value = False)) def test_mirror_mirror_on_the_wall_5(self, downloader_mock, stdout_mock): diff --git a/test/unit/tutorial_examples.py b/test/unit/tutorial_examples.py index c3359d1e..d96ff71b 100644 --- a/test/unit/tutorial_examples.py +++ b/test/unit/tutorial_examples.py @@ -2,15 +2,11 @@ Tests for the examples given in stem's tutorial. """ +import io import itertools import os import unittest -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - import stem.response import stem.descriptor.remote import stem.prereq @@ -125,7 +121,7 @@ def _get_router_status(address = None, port = None, nickname = None, fingerprint class TestTutorialExamples(unittest.TestCase): - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.control.Controller.from_port', spec = Controller) def test_list_circuits(self, from_port_mock, stdout_mock): path_1 = ('B1FA7D51B8B6F0CB585D944F450E7C06EDE7E44C', 'ByTORAndTheSnowDog') @@ -156,7 +152,7 @@ class TestTutorialExamples(unittest.TestCase): exec_documentation_example('list_circuits.py') self.assertCountEqual(LIST_CIRCUITS_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.control.Controller.from_port', spec = Controller) def test_exit_used(self, from_port_mock, stdout_mock): def tutorial_example(mock_event): @@ -207,7 +203,7 @@ class TestTutorialExamples(unittest.TestCase): tutorial_example(event) self.assertCountEqual(EXIT_USED_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.descriptor.remote.DescriptorDownloader') def test_outdated_relays(self, downloader_mock, stdout_mock): downloader_mock().get_server_descriptors.return_value = [ @@ -221,7 +217,7 @@ class TestTutorialExamples(unittest.TestCase): self.assertCountEqual(OUTDATED_RELAYS_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.descriptor.remote.Query') @patch('stem.directory.Authority.from_cache') def test_compare_flags(self, authorities_mock, query_mock, stdout_mock): @@ -262,7 +258,7 @@ class TestTutorialExamples(unittest.TestCase): self.assertCountEqual(COMPARE_FLAGS_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.directory.Authority.from_cache') @patch('stem.descriptor.remote.DescriptorDownloader.query') def test_votes_by_bandwidth_authorities(self, query_mock, authorities_mock, stdout_mock): @@ -295,7 +291,7 @@ class TestTutorialExamples(unittest.TestCase): exec_documentation_example('votes_by_bandwidth_authorities.py') self.assertCountEqual(VOTES_BY_BANDWIDTH_AUTHORITIES_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) - @patch('sys.stdout', new_callable = StringIO) + @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.descriptor.parse_file') @patch('stem.descriptor.remote.Query') def test_persisting_a_consensus(self, query_mock, parse_file_mock, stdout_mock):

1 0

[stem/master] Drop http.client fallback
by atagar＠torproject.org 05 Jan '20

05 Jan '20

commit ab83d8ded24d83005834d4f0d1d9fd40bbd38e96 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 3 16:17:14 2020 -0800 Drop http.client fallback Just a single normalization of HTTPMessage. --- test/unit/descriptor/remote.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 98bb734b..d9893535 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -2,6 +2,7 @@ Unit tests for stem.descriptor.remote. """ +import http.client import io import socket import time @@ -18,11 +19,6 @@ from unittest.mock import patch, Mock, MagicMock from stem.descriptor.remote import Compression from test.unit.descriptor import read_resource -try: - from http.client import HTTPMessage # python3 -except ImportError: - from httplib import HTTPMessage # python2 - TEST_RESOURCE = '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31' # Output from requesting moria1's descriptor from itself... @@ -94,7 +90,7 @@ def _dirport_mock(data, encoding = 'identity'): dirport_mock = Mock() dirport_mock().read.return_value = data - headers = HTTPMessage() + headers = http.client.HTTPMessage() for line in HEADER.splitlines(): key, value = line.split(': ', 1)

1 0

[stem/master] Replace mkdtemp() with TemporaryDirectory()
by atagar＠torproject.org 05 Jan '20

05 Jan '20

commit 85c81eea7fdad016384e734791cccf0e97132fc1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jan 4 15:25:14 2020 -0800 Replace mkdtemp() with TemporaryDirectory() Minor python 3.x simplification now that we can use this builtin. --- stem/descriptor/collector.py | 13 ++--- stem/manual.py | 9 ++-- test/integ/control/controller.py | 105 +++++++++++++++++++-------------------- test/integ/process.py | 31 ++++-------- test/integ/util/system.py | 11 ++-- test/unit/manual.py | 16 +++--- 6 files changed, 82 insertions(+), 103 deletions(-) diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index c2bf8179..7aeb298b 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -55,7 +55,6 @@ import hashlib import json import os import re -import shutil import tempfile import time @@ -264,15 +263,9 @@ class File(object): if self._downloaded_to and os.path.exists(self._downloaded_to): directory = os.path.dirname(self._downloaded_to) else: - # TODO: The following can be replaced with simpler usage of - # tempfile.TemporaryDirectory when we drop python 2.x support. - - tmp_directory = tempfile.mkdtemp() - - for desc in self.read(tmp_directory, descriptor_type, start, end, document_handler, timeout, retries): - yield desc - - shutil.rmtree(tmp_directory) + with tempfile.TemporaryDirectory() as tmp_directory: + for desc in self.read(tmp_directory, descriptor_type, start, end, document_handler, timeout, retries): + yield desc return diff --git a/stem/manual.py b/stem/manual.py index fe49e3de..c1b4bd8f 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -288,11 +288,10 @@ def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, elif not stem.util.system.is_available('a2x'): raise IOError('We require a2x from asciidoc to provide a man page') - dirpath = tempfile.mkdtemp() - asciidoc_path = os.path.join(dirpath, 'tor.1.txt') - manual_path = os.path.join(dirpath, 'tor.1') + with tempfile.TemporaryDirectory() as dirpath: + asciidoc_path = os.path.join(dirpath, 'tor.1.txt') + manual_path = os.path.join(dirpath, 'tor.1') - try: try: with open(asciidoc_path, 'wb') as asciidoc_file: request = urllib.request.urlopen(url, timeout = timeout) @@ -325,8 +324,6 @@ def download_man_page(path = None, file_handle = None, url = GITWEB_MANUAL_URL, with open(manual_path, 'rb') as manual_file: shutil.copyfileobj(manual_file, file_handle) file_handle.flush() - finally: - shutil.rmtree(dirpath) class Manual(object): diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index f87c0817..257d9fbc 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -754,71 +754,70 @@ class TestController(unittest.TestCase): """ runner = test.runner.get_runner() - tmpdir = tempfile.mkdtemp() - with runner.get_tor_controller() as controller: - try: - # successfully set a single option - connlimit = int(controller.get_conf('ConnLimit')) - controller.set_conf('connlimit', str(connlimit - 1)) - self.assertEqual(connlimit - 1, int(controller.get_conf('ConnLimit'))) - - # successfully set a single list option - exit_policy = ['accept *:7777', 'reject *:*'] - controller.set_conf('ExitPolicy', exit_policy) - self.assertEqual(exit_policy, controller.get_conf('ExitPolicy', multiple = True)) + with tempfile.TemporaryDirectory() as tmpdir: - # fail to set a single option + with runner.get_tor_controller() as controller: try: - controller.set_conf('invalidkeyboo', 'abcde') - self.fail() - except stem.InvalidArguments as exc: - self.assertEqual(['invalidkeyboo'], exc.arguments) + # successfully set a single option + connlimit = int(controller.get_conf('ConnLimit')) + controller.set_conf('connlimit', str(connlimit - 1)) + self.assertEqual(connlimit - 1, int(controller.get_conf('ConnLimit'))) - # resets configuration parameters - controller.reset_conf('ConnLimit', 'ExitPolicy') - self.assertEqual(connlimit, int(controller.get_conf('ConnLimit'))) - self.assertEqual(None, controller.get_conf('ExitPolicy')) + # successfully set a single list option + exit_policy = ['accept *:7777', 'reject *:*'] + controller.set_conf('ExitPolicy', exit_policy) + self.assertEqual(exit_policy, controller.get_conf('ExitPolicy', multiple = True)) - # successfully sets multiple config options - controller.set_options({ - 'connlimit': str(connlimit - 2), - 'contactinfo': 'stem@testing', - }) + # fail to set a single option + try: + controller.set_conf('invalidkeyboo', 'abcde') + self.fail() + except stem.InvalidArguments as exc: + self.assertEqual(['invalidkeyboo'], exc.arguments) - self.assertEqual(connlimit - 2, int(controller.get_conf('ConnLimit'))) - self.assertEqual('stem@testing', controller.get_conf('contactinfo')) + # resets configuration parameters + controller.reset_conf('ConnLimit', 'ExitPolicy') + self.assertEqual(connlimit, int(controller.get_conf('ConnLimit'))) + self.assertEqual(None, controller.get_conf('ExitPolicy')) - # fail to set multiple config options - try: + # successfully sets multiple config options controller.set_options({ + 'connlimit': str(connlimit - 2), 'contactinfo': 'stem@testing', - 'bombay': 'vadapav', }) - self.fail() - except stem.InvalidArguments as exc: - self.assertEqual(['bombay'], exc.arguments) - # context-sensitive keys (the only retched things for which order matters) - controller.set_options(( - ('HiddenServiceDir', tmpdir), - ('HiddenServicePort', '17234 127.0.0.1:17235'), - )) + self.assertEqual(connlimit - 2, int(controller.get_conf('ConnLimit'))) + self.assertEqual('stem@testing', controller.get_conf('contactinfo')) - self.assertEqual(tmpdir, controller.get_conf('HiddenServiceDir')) - self.assertEqual('17234 127.0.0.1:17235', controller.get_conf('HiddenServicePort')) - finally: - # reverts configuration changes - - controller.set_options(( - ('ExitPolicy', 'reject *:*'), - ('ConnLimit', None), - ('ContactInfo', None), - ('HiddenServiceDir', None), - ('HiddenServicePort', None), - ), reset = True) - - shutil.rmtree(tmpdir) + # fail to set multiple config options + try: + controller.set_options({ + 'contactinfo': 'stem@testing', + 'bombay': 'vadapav', + }) + self.fail() + except stem.InvalidArguments as exc: + self.assertEqual(['bombay'], exc.arguments) + + # context-sensitive keys (the only retched things for which order matters) + controller.set_options(( + ('HiddenServiceDir', tmpdir), + ('HiddenServicePort', '17234 127.0.0.1:17235'), + )) + + self.assertEqual(tmpdir, controller.get_conf('HiddenServiceDir')) + self.assertEqual('17234 127.0.0.1:17235', controller.get_conf('HiddenServicePort')) + finally: + # reverts configuration changes + + controller.set_options(( + ('ExitPolicy', 'reject *:*'), + ('ConnLimit', None), + ('ContactInfo', None), + ('HiddenServiceDir', None), + ('HiddenServicePort', None), + ), reset = True) @test.require.controller def test_set_conf_for_usebridges(self): diff --git a/test/integ/process.py b/test/integ/process.py index e7d2cae3..e40c501a 100644 --- a/test/integ/process.py +++ b/test/integ/process.py @@ -9,7 +9,6 @@ import hashlib import os import random import re -import shutil import subprocess import tempfile import threading @@ -53,18 +52,8 @@ def random_port(): @contextmanager -def tmp_directory(): - tmp_dir = tempfile.mkdtemp() - - try: - yield tmp_dir - finally: - shutil.rmtree(tmp_dir) - - -@contextmanager def torrc(): - with tmp_directory() as data_directory: + with tempfile.TemporaryDirectory() as data_directory: torrc_path = os.path.join(data_directory, 'torrc') with open(torrc_path, 'w') as torrc_file: @@ -228,7 +217,7 @@ class TestProcess(unittest.TestCase): output = run_tor(tor_cmd, '--list-fingerprint', with_torrc = True, expect_failure = True) assert_in("Clients don't have long-term identity keys. Exiting.", output, 'Should fail to start due to lacking an ORPort') - with tmp_directory() as data_directory: + with tempfile.TemporaryDirectory() as data_directory: torrc_path = os.path.join(data_directory, 'torrc') with open(torrc_path, 'w') as torrc_file: @@ -324,7 +313,7 @@ class TestProcess(unittest.TestCase): if test.tor_version() < stem.version.Requirement.TORRC_VIA_STDIN: skip('(requires %s)' % stem.version.Requirement.TORRC_VIA_STDIN) - with tmp_directory() as data_directory: + with tempfile.TemporaryDirectory() as data_directory: torrc = BASIC_RELAY_TORRC % data_directory output = run_tor(tor_cmd, '-f', '-', '--dump-config', 'short', stdin = torrc) assert_equal(sorted(torrc.splitlines()), sorted(output.splitlines())) @@ -382,7 +371,7 @@ class TestProcess(unittest.TestCase): it isn't. """ - with tmp_directory() as data_directory: + with tempfile.TemporaryDirectory() as data_directory: # Tries running tor in another thread with the given timeout argument. This # issues an invalid torrc so we terminate right away if we get to the point # of actually invoking tor. @@ -435,7 +424,7 @@ class TestProcess(unittest.TestCase): Exercises launch_tor_with_config when we write a torrc to disk. """ - with tmp_directory() as data_directory: + with tempfile.TemporaryDirectory() as data_directory: control_port = random_port() control_socket, tor_process = None, None @@ -479,7 +468,7 @@ class TestProcess(unittest.TestCase): if test.tor_version() < stem.version.Requirement.TORRC_VIA_STDIN: skip('(requires %s)' % stem.version.Requirement.TORRC_VIA_STDIN) - with tmp_directory() as data_directory: + with tempfile.TemporaryDirectory() as data_directory: control_port = random_port() control_socket, tor_process = None, None @@ -521,7 +510,7 @@ class TestProcess(unittest.TestCase): # [warn] Failed to parse/validate config: Failed to bind one of the listener ports. # [err] Reading config failed--see warnings above. - with tmp_directory() as data_directory: + with tempfile.TemporaryDirectory() as data_directory: both_ports = random_port() try: @@ -543,7 +532,7 @@ class TestProcess(unittest.TestCase): Runs launch_tor where it times out before completing. """ - with tmp_directory() as data_directory: + with tempfile.TemporaryDirectory() as data_directory: start_time = time.time() try: @@ -575,7 +564,7 @@ class TestProcess(unittest.TestCase): elif test.tor_version() < stem.version.Requirement.TAKEOWNERSHIP: skip('(requires %s)' % stem.version.Requirement.TAKEOWNERSHIP) - with tmp_directory() as data_directory: + with tempfile.TemporaryDirectory() as data_directory: sleep_process = subprocess.Popen(['sleep', '60']) tor_process = stem.process.launch_tor_with_config( @@ -619,7 +608,7 @@ class TestProcess(unittest.TestCase): if test.tor_version() < stem.version.Requirement.TAKEOWNERSHIP: skip('(requires %s)' % stem.version.Requirement.TAKEOWNERSHIP) - with tmp_directory() as data_directory: + with tempfile.TemporaryDirectory() as data_directory: control_port = random_port() tor_process = stem.process.launch_tor_with_config( diff --git a/test/integ/util/system.py b/test/integ/util/system.py index 4d83d695..2a7ac07a 100644 --- a/test/integ/util/system.py +++ b/test/integ/util/system.py @@ -361,13 +361,14 @@ class TestSystem(unittest.TestCase): Checks the stem.util.system.pid_by_open_file function. """ + # check a directory that doesn't exist + + self.assertEqual(None, stem.util.system.pid_by_open_file('/no/such/path')) + # check a directory that exists, but isn't claimed by any application - tmpdir = tempfile.mkdtemp() - self.assertEqual(None, stem.util.system.pid_by_open_file(tmpdir)) - # check a directory that doesn't exist - os.rmdir(tmpdir) - self.assertEqual(None, stem.util.system.pid_by_open_file(tmpdir)) + with tempfile.TemporaryDirectory() as tmpdir: + self.assertEqual(None, stem.util.system.pid_by_open_file(tmpdir)) @require_path def test_pids_by_user(self): diff --git a/test/unit/manual.py b/test/unit/manual.py index eb76d6e9..e1891ca4 100644 --- a/test/unit/manual.py +++ b/test/unit/manual.py @@ -76,6 +76,10 @@ EXPECTED_CONFIG_OPTIONS['Bridge'] = stem.manual.ConfigOption( CACHED_MANUAL = None +TEMP_DIR_MOCK = Mock() +TEMP_DIR_MOCK.__enter__ = Mock(return_value = '/no/such/path') +TEMP_DIR_MOCK.__exit__ = Mock(return_value = False) + def _cached_manual(): global CACHED_MANUAL @@ -230,16 +234,14 @@ class TestManual(unittest.TestCase): exc_msg = 'We require a2x from asciidoc to provide a man page' self.assertRaisesWith(IOError, exc_msg, stem.manual.download_man_page, '/tmp/no_such_file') - @patch('tempfile.mkdtemp', Mock(return_value = '/no/such/path')) - @patch('shutil.rmtree', Mock()) + @patch('tempfile.TemporaryDirectory', Mock(return_value = TEMP_DIR_MOCK)) @patch('stem.manual.open', Mock(side_effect = IOError('unable to write to file')), create = True) @patch('stem.util.system.is_available', Mock(return_value = True)) def test_download_man_page_when_unable_to_write(self): exc_msg = "Unable to download tor's manual from https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt to /no/such/path/tor.1.txt: unable to write to file" self.assertRaisesWith(IOError, exc_msg, stem.manual.download_man_page, '/tmp/no_such_file') - @patch('tempfile.mkdtemp', Mock(return_value = '/no/such/path')) - @patch('shutil.rmtree', Mock()) + @patch('tempfile.TemporaryDirectory', Mock(return_value = TEMP_DIR_MOCK)) @patch('stem.manual.open', Mock(return_value = io.BytesIO()), create = True) @patch('stem.util.system.is_available', Mock(return_value = True)) @patch('urllib.request.urlopen', Mock(side_effect = urllib.request.URLError('<urlopen error [Errno -2] Name or service not known>'))) @@ -247,8 +249,7 @@ class TestManual(unittest.TestCase): exc_msg = "Unable to download tor's manual from https://www.atagar.com/foo/bar to /no/such/path/tor.1.txt: <urlopen error <urlopen error [Errno -2] Name or service not known>>" self.assertRaisesWith(IOError, exc_msg, stem.manual.download_man_page, '/tmp/no_such_file', url = 'https://www.atagar.com/foo/bar') - @patch('tempfile.mkdtemp', Mock(return_value = '/no/such/path')) - @patch('shutil.rmtree', Mock()) + @patch('tempfile.TemporaryDirectory', Mock(return_value = TEMP_DIR_MOCK)) @patch('stem.manual.open', Mock(return_value = io.BytesIO()), create = True) @patch('stem.util.system.call', Mock(side_effect = stem.util.system.CallError('call failed', 'a2x -f manpage /no/such/path/tor.1.txt', 1, None, None, 'call failed'))) @patch('stem.util.system.is_available', Mock(return_value = True)) @@ -257,8 +258,7 @@ class TestManual(unittest.TestCase): exc_msg = "Unable to run 'a2x -f manpage /no/such/path/tor.1.txt': call failed" self.assertRaisesWith(IOError, exc_msg, stem.manual.download_man_page, '/tmp/no_such_file', url = 'https://www.atagar.com/foo/bar') - @patch('tempfile.mkdtemp', Mock(return_value = '/no/such/path')) - @patch('shutil.rmtree', Mock()) + @patch('tempfile.TemporaryDirectory', Mock(return_value = TEMP_DIR_MOCK)) @patch('stem.manual.open', create = True) @patch('stem.util.system.call') @patch('stem.util.system.is_available', Mock(return_value = True))

1 0

[stem/master] Drop queue fallback
by atagar＠torproject.org 05 Jan '20

05 Jan '20

commit bfafe1062511a62ddf612846e8d9ee1c05d19668 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jan 3 16:28:05 2020 -0800 Drop queue fallback --- stem/control.py | 7 +------ stem/descriptor/reader.py | 6 +----- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/stem/control.py b/stem/control.py index c046755c..4adec330 100644 --- a/stem/control.py +++ b/stem/control.py @@ -253,15 +253,10 @@ import functools import inspect import io import os +import queue import threading import time -try: - # Added in 3.x - import queue -except ImportError: - import Queue as queue - import stem.descriptor.microdescriptor import stem.descriptor.reader import stem.descriptor.router_status_entry diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index ebd41c73..8ef6f22c 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -84,14 +84,10 @@ and picks up where it left off if run again... import mimetypes import os +import queue import tarfile import threading -try: - import queue -except ImportError: - import Queue as queue - import stem.descriptor import stem.prereq import stem.util

1 0