tor-commits February 2013

tor-commits@lists.torproject.org

19 participants
783 discussions

[stem/master] Checking for 2to3 and python3 when needed
by atagar＠torproject.org 02 Feb '13

02 Feb '13

commit 8a3cfb57e1982d26bafe6289f8ed9dea47192604 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 30 08:53:06 2013 -0800 Checking for 2to3 and python3 when needed Warning the user if 2to3 or python3 aren't in our PATH when the user provides the '--python3' testing argument. --- run_tests.py | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diff --git a/run_tests.py b/run_tests.py index 9604731..d18ec9d 100755 --- a/run_tests.py +++ b/run_tests.py @@ -412,6 +412,13 @@ if __name__ == '__main__': load_user_configuration(test_config) + # check that we have 2to3 and python3 available in our PATH + if CONFIG["argument.python3"]: + for required_cmd in ("2to3", "python3"): + if not system.is_available(required_cmd): + test.output.print_line("Unable to test python 3 because %s isn't in your path" % required_cmd, *test.runner.ERROR_ATTR) + sys.exit(1) + if CONFIG["argument.python3"] and sys.version_info.major != 3: python3_destination = os.path.join(CONFIG["integ.test_directory"], "python3")

1 0

[stem/master] Skip universal newline translation in descriptor reader
by atagar＠torproject.org 02 Feb '13

02 Feb '13

commit 0288267be5a325aadd078a487751cf4ce83a2f73 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 30 08:24:59 2013 -0800 Skip universal newline translation in descriptor reader Python 3 introduces universal newline translation, converting '\n', '\r', and '\r\n' into the local system's newline style. This is a really neat feature and will solve many-a-headaches... but not for us. We conform to the tor spec which specifies when CRLF appears verses other newline types. Universal newline translation broke our ability to read the 'cr_in_contact_line' example which has multiple '\r' within a contact line (https://trac.torproject.org/5637). Fixing the reader to disable newline translation and adding a warning to our parse_file() pydocs. --- stem/descriptor/__init__.py | 10 ++++++++++ stem/descriptor/reader.py | 8 +++++++- 2 files changed, 17 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index fe6c03c..69d7db7 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -82,6 +82,16 @@ def parse_file(descriptor_file, descriptor_type = None, path = None, validate = tordnsel 1.0 **unsupported** ===================================== ===== + If you're using python 3 then beware of the open() function's universal + newline translation. By default open() converts all common line endings (NL, + CR, and CRNL) into NL. In some edge cases this can cause us to misparse + content. To disable newline translation set the **newline** to an empty + string. For example... + + :: + + my_descriptor_file = open(descrptor_path, newline='') + :param file descriptor_file: opened file with the descriptor contents :param str descriptor_type: `descriptor type <https://metrics.torproject.org/formats.html#descriptortypes>`_, this is guessed if not provided :param str path: absolute path to the file's location on disk diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 877d230..ef35fc0 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -512,7 +512,13 @@ class DescriptorReader(object): def _handle_descriptor_file(self, target, mime_type): try: self._notify_read_listeners(target) - with open(target) as target_file: + + if stem.prereq.is_python_3(): + target_file = open(target, newline = '') + else: + target_file = open(target) + + with target_file as target_file: for desc in stem.descriptor.parse_file(target_file, validate = self._validate, path = target): if self._is_stopped.isSet(): return

1 0

[stem/master] Providing ASCII bytes to hashlib.sha1()
by atagar＠torproject.org 02 Feb '13

02 Feb '13

commit 6c855996c0daed3b6db222996dd651534c134c0d Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 30 09:32:03 2013 -0800 Providing ASCII bytes to hashlib.sha1() Another unicode/ASCII bytes conversion issue... ====================================================================== ERROR: test_metrics_descriptor ---------------------------------------------------------------------- Traceback: File "/home/atagar/Desktop/stem/test/data/python3/test/integ/descriptor/server_descriptor.py", line 89, in test_metrics_descriptor self.assertEquals("2C7B27BEAB04B4E2459D89CA6D5CD1CC5F95A689", desc.digest()) File "/home/atagar/Desktop/stem/test/data/python3/stem/descriptor/server_descriptor.py", line 666, in digest digest_hash = hashlib.sha1(for_digest) TypeError: Unicode-objects must be encoded before hashing --- stem/descriptor/extrainfo_descriptor.py | 3 ++- stem/descriptor/server_descriptor.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 2c5b6dc..c64550c 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -71,6 +71,7 @@ import re import stem.descriptor import stem.util.connection import stem.util.enum +import stem.util.str_tools # known statuses for dirreq-v2-resp and dirreq-v3-resp... DirResponse = stem.util.enum.Enum( @@ -820,7 +821,7 @@ class RelayExtraInfoDescriptor(ExtraInfoDescriptor): # our digest is calculated from everything except our signature raw_content, ending = str(self), "\nrouter-signature\n" raw_content = raw_content[:raw_content.find(ending) + len(ending)] - self._digest = hashlib.sha1(raw_content).hexdigest().upper() + self._digest = hashlib.sha1(stem.util.str_tools.to_bytes(raw_content)).hexdigest().upper() return self._digest diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 9e5004b..2376a56 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -663,7 +663,7 @@ class RelayDescriptor(ServerDescriptor): if start >= 0 and sig_start > 0 and end > start: for_digest = raw_descriptor[start:end] - digest_hash = hashlib.sha1(for_digest) + digest_hash = hashlib.sha1(stem.util.str_tools.to_bytes(for_digest)) self._digest = digest_hash.hexdigest().upper() else: raise ValueError("unable to calculate digest for descriptor") @@ -684,7 +684,7 @@ class RelayDescriptor(ServerDescriptor): if self.fingerprint: # calculate the signing key hash - key_der_as_hash = hashlib.sha1(key_as_bytes).hexdigest() + key_der_as_hash = hashlib.sha1(stem.util.str_tools.to_bytes(key_as_bytes)).hexdigest() if key_der_as_hash != self.fingerprint.lower(): log.warn("Signing key hash: %s != fingerprint: %s" % (key_der_as_hash, self.fingerprint.lower()))

1 0

[stem/master] Skipping newline translation for descriptor integ tests
by atagar＠torproject.org 02 Feb '13

02 Feb '13

commit 9ff618eebed92e5646090ce3faa802763a5f7720 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 30 09:10:18 2013 -0800 Skipping newline translation for descriptor integ tests Using a custom open() call for python 3's integ tests to prevent newline translation (and the resulting test failures). --- test/integ/descriptor/__init__.py | 23 ++++++++++++++++- test/integ/descriptor/extrainfo_descriptor.py | 12 +++------ test/integ/descriptor/networkstatus.py | 34 +++++++++++-------------- test/integ/descriptor/server_descriptor.py | 33 +++++++----------------- 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/test/integ/descriptor/__init__.py b/test/integ/descriptor/__init__.py index d3d3063..31421f1 100644 --- a/test/integ/descriptor/__init__.py +++ b/test/integ/descriptor/__init__.py @@ -2,10 +2,18 @@ Integration tests for stem.descriptor.* contents. """ -__all__ = ["reader", "extrainfo_descriptor", "server_descriptor"] +__all__ = [ + "reader", + "extrainfo_descriptor", + "server_descriptor", + "get_resource", + "open_desc", +] import os +import stem.prereq + DESCRIPTOR_TEST_DATA = os.path.join(os.path.dirname(__file__), "data") @@ -15,3 +23,16 @@ def get_resource(filename): """ return os.path.join(DESCRIPTOR_TEST_DATA, filename) + + +def open_desc(filename, absolute = False): + """ + Provides the file for a given descriptor in our data directory. + """ + + path = filename if absolute else get_resource(filename) + + if stem.prereq.is_python_3(): + return open(path, newline = '') + else: + return open(path) diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 5fc9b5a..0bcc23f 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -9,10 +9,10 @@ import os import unittest import stem.descriptor.extrainfo_descriptor -import test.integ.descriptor import test.runner from stem.descriptor.extrainfo_descriptor import DirResponse +from test.integ.descriptor import open_desc class TestExtraInfoDescriptor(unittest.TestCase): @@ -21,9 +21,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): Parses and checks our results against an extrainfo descriptor from metrics. """ - descriptor_path = test.integ.descriptor.get_resource("extrainfo_relay_descriptor") - - descriptor_file = open(descriptor_path) + descriptor_file = open_desc("extrainfo_relay_descriptor") descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close() @@ -72,9 +70,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw metrics. """ - descriptor_path = test.integ.descriptor.get_resource("extrainfo_bridge_descriptor") - - descriptor_file = open(descriptor_path) + descriptor_file = open_desc("extrainfo_bridge_descriptor") descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close() @@ -150,7 +146,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw test.runner.skip(self, "(no cached descriptors)") return - with open(descriptor_path) as descriptor_file: + with open_desc(descriptor_path, absolute = True) as descriptor_file: for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file): unrecognized_lines = desc.get_unrecognized_lines() diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index 035289d..19c3d8f 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -13,7 +13,9 @@ import stem import stem.descriptor import stem.descriptor.networkstatus import stem.version -import test.integ.descriptor +import test.runner + +from test.integ.descriptor import get_resource, open_desc class TestNetworkStatus(unittest.TestCase): @@ -40,7 +42,7 @@ class TestNetworkStatus(unittest.TestCase): return count = 0 - with open(consensus_path) as descriptor_file: + with open_desc(consensus_path, absolute = True) as descriptor_file: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV3 for router in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type): @@ -87,7 +89,7 @@ class TestNetworkStatus(unittest.TestCase): return count = 0 - with open(consensus_path) as descriptor_file: + with open_desc(consensus_path, absolute = True) as descriptor_file: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV3 for router in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, is_microdescriptor = True): @@ -115,10 +117,10 @@ class TestNetworkStatus(unittest.TestCase): Checks if consensus documents from Metrics are parsed properly. """ - consensus_path = test.integ.descriptor.get_resource("metrics_consensus") + consensus_path = get_resource("metrics_consensus") for specify_type in (True, False): - with open(consensus_path) as descriptor_file: + with open_desc(consensus_path, absolute = True) as descriptor_file: if specify_type: descriptors = stem.descriptor.parse_file(descriptor_file, "network-status-consensus-3 1.0", path = consensus_path) else: @@ -138,9 +140,9 @@ class TestNetworkStatus(unittest.TestCase): Checks if the bridge documents from Metrics are parsed properly. """ - consensus_path = test.integ.descriptor.get_resource("bridge_network_status") + consensus_path = get_resource("bridge_network_status") - with open(consensus_path) as descriptor_file: + with open_desc(consensus_path, absolute = True) as descriptor_file: descriptors = stem.descriptor.parse_file(descriptor_file, path = consensus_path) router = next(descriptors) @@ -186,7 +188,7 @@ RY22NXCwrJvSMEwiy7acC8FGysqwHRyE356+Rw6TB43g3Tno9KaHEK7MHXjSHwNs GM9hAsAMRX9Ogqhq5UjDNqEsvDKuyVeyh7unSZEOip9Zr6K/+7VsVPNb8vfBRBjo -----END SIGNATURE-----""" - cert_path = test.integ.descriptor.get_resource("metrics_cert") + cert_path = get_resource("metrics_cert") with open(cert_path) as cert_file: certs = stem.descriptor.parse_file(cert_file, path = cert_path) @@ -243,9 +245,7 @@ mfWcW0b+jsrXcJoCxV5IrwCDF3u1aC3diwZY6yiG186pwWbOwE41188XI2DeYPwE I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= -----END SIGNATURE-----""" - consensus_path = test.integ.descriptor.get_resource("cached-consensus") - - with open(consensus_path) as descriptor_file: + with open_desc("cached-consensus") as descriptor_file: document = stem.descriptor.networkstatus.NetworkStatusDocumentV3(descriptor_file.read(), default_params = False) self.assertEquals(3, document.version) @@ -316,9 +316,7 @@ nTA+fD8JQqpPtb8b0SnG9kwy75eS//sRu7TErie2PzGMxrf9LH0LAgMBAAE= TpQQk3nNQF8z6UIvdlvP+DnJV4izWVkQEZgUZgIVM0E= -----END SIGNATURE-----""" - consensus_path = test.integ.descriptor.get_resource("cached-consensus-v2") - - with open(consensus_path) as descriptor_file: + with open_desc("cached-consensus-v2") as descriptor_file: descriptor_file.readline() # strip header document = stem.descriptor.networkstatus.NetworkStatusDocumentV2(descriptor_file.read()) @@ -381,9 +379,9 @@ TpQQk3nNQF8z6UIvdlvP+DnJV4izWVkQEZgUZgIVM0E= Checks if vote documents from Metrics are parsed properly. """ - vote_path = test.integ.descriptor.get_resource("metrics_vote") + vote_path = get_resource("metrics_vote") - with open(vote_path) as descriptor_file: + with open_desc(vote_path, absolute = True) as descriptor_file: descriptors = stem.descriptor.parse_file(descriptor_file, path = vote_path) router = next(descriptors) @@ -445,9 +443,7 @@ JZ/1HL9sHyZfo6bwaC6YSM9PNiiY6L7rnGpS7UkHiFI+M96VCMorvjm5YPs3FioJ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= -----END SIGNATURE-----""" - vote_path = test.integ.descriptor.get_resource("vote") - - with open(vote_path) as descriptor_file: + with open_desc("vote") as descriptor_file: document = stem.descriptor.networkstatus.NetworkStatusDocumentV3(descriptor_file.read(), default_params = False) self.assertEquals(3, document.version) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 0258dfa..8eb262a 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -13,9 +13,10 @@ import stem.descriptor import stem.descriptor.server_descriptor import stem.exit_policy import stem.version -import test.integ.descriptor import test.runner +from test.integ.descriptor import open_desc + class TestServerDescriptor(unittest.TestCase): def test_metrics_descriptor(self): @@ -23,9 +24,7 @@ class TestServerDescriptor(unittest.TestCase): Parses and checks our results against a server descriptor from metrics. """ - descriptor_path = test.integ.descriptor.get_resource("example_descriptor") - - descriptor_file = open(descriptor_path) + descriptor_file = open_desc("example_descriptor") descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close() @@ -95,9 +94,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses and checks our results against a server descriptor from metrics. """ - descriptor_path = test.integ.descriptor.get_resource("metrics_server_desc_multiple") - - with open(descriptor_path) as descriptor_file: + with open_desc("metrics_server_desc_multiple") as descriptor_file: descriptors = list(stem.descriptor.parse_file(descriptor_file, "server-descriptor 1.0")) self.assertEquals(2, len(descriptors)) @@ -113,9 +110,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses a relay server descriptor from 2005. """ - descriptor_path = test.integ.descriptor.get_resource("old_descriptor") - - descriptor_file = open(descriptor_path) + descriptor_file = open_desc("old_descriptor") descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close() @@ -177,7 +172,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= test.runner.skip(self, "(no cached descriptors)") return - with open(descriptor_path) as descriptor_file: + with open_desc(descriptor_path, absolute = True) as descriptor_file: for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file): # the following attributes should be deprecated, and not appear in the wild self.assertEquals(None, desc.read_history_end) @@ -201,9 +196,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses a descriptor with non-ascii content. """ - descriptor_path = test.integ.descriptor.get_resource("non-ascii_descriptor") - - descriptor_file = open(descriptor_path) + descriptor_file = open_desc("non-ascii_descriptor") descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close() @@ -243,9 +236,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= returns ('\r' entries). """ - descriptor_path = test.integ.descriptor.get_resource("cr_in_contact_line") - - descriptor_file = open(descriptor_path) + descriptor_file = open_desc("cr_in_contact_line") descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close() @@ -271,9 +262,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= where we shouldn't be. """ - descriptor_path = test.integ.descriptor.get_resource("negative_uptime") - - descriptor_file = open(descriptor_path) + descriptor_file = open_desc("negative_uptime") descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close() @@ -295,9 +284,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses a bridge descriptor. """ - descriptor_path = test.integ.descriptor.get_resource("bridge_descriptor") - - descriptor_file = open(descriptor_path) + descriptor_file = open_desc("bridge_descriptor") descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close()

1 0

[stem/master] Changing is_python_2* prereq checks to include python 3
by atagar＠torproject.org 02 Feb '13

02 Feb '13

commit 46f1bfb98cee79f9a00b76513d2c70df1824bb33 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 30 09:36:09 2013 -0800 Changing is_python_2* prereq checks to include python 3 The is_python_26 and is_python_27 were checking if we were 2.6-2.x or 2.7-2.x. On reflection it makes more sense for these to be '2.y and above' checks rather than '2.y and above in the 2.x series'. --- stem/prereq.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stem/prereq.py b/stem/prereq.py index 46913e4..a6eab48 100644 --- a/stem/prereq.py +++ b/stem/prereq.py @@ -42,7 +42,7 @@ def check_requirements(): def is_python_26(): """ - Checks if we're in the 2.6 - 2.x range. + Checks if we're running python 2.6 or above. :returns: bool that is True if we meet this requirement and False otherwise """ @@ -52,7 +52,7 @@ def is_python_26(): def is_python_27(): """ - Checks if we're in the 2.7 - 2.x range. + Checks if we're running python 2.7 or above. :returns: bool that is True if we meet this requirement and False otherwise """ @@ -95,7 +95,7 @@ def _check_version(minor_req): major_version, minor_version = sys.version_info[0:2] if major_version > 2: - return False + return True elif major_version < 2 or minor_version < minor_req: return False

1 0

[stem/master] Fixing server descriptor test expecting unicode
by atagar＠torproject.org 02 Feb '13

02 Feb '13

commit bec2e9721e228f73c8b1438f1d398182893df27f Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jan 31 08:22:38 2013 -0800 Fixing server descriptor test expecting unicode One of the server descriptor integ tests had a failing assertion because the expected text was ASCII bytes and the descriptor content was unicode. Fixing the test and moving the to_unicode helper to str_tools where it belongs. ====================================================================== FAIL: test_non_ascii_descriptor ---------------------------------------------------------------------- Traceback: File "/home/atagar/Desktop/stem/test/data/python3/test/integ/descriptor/server_descriptor.py", line 221, in test_non_ascii_descriptor self.assertEquals(expected_contact, desc.contact) AssertionError: '2048R/F171EC1F Johan BlÃ¥bÃ¤ck ã\x81\x93ã\x82\x93ã\x81«ã\x81¡ã\x81¯' != '2048R/F171EC1F Johan Blåbäck こんにちは' - 2048R/F171EC1F Johan BlÃ¥bÃ¤ck ããã«ã¡ã¯ + 2048R/F171EC1F Johan Blåbäck こんにちは --- stem/descriptor/__init__.py | 21 ++++----------------- stem/util/str_tools.py | 26 ++++++++++++++++++++++++++ test/integ/descriptor/server_descriptor.py | 5 +++-- test/settings.cfg | 5 +++-- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 69d7db7..bebcdcc 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -28,6 +28,7 @@ import os import re import stem.prereq +import stem.util.str_tools try: # added in python 2.7 @@ -274,10 +275,10 @@ class _UnicodeReader(object): return self.wrapped_file.next() def read(self, n = -1): - return self._to_unicode(self.wrapped_file.read(n)) + return stem.util.str_tools.to_unicode(self.wrapped_file.read(n)) def readline(self): - return self._to_unicode(self.wrapped_file.readline()) + return stem.util.str_tools.to_unicode(self.wrapped_file.readline()) def readlines(self, sizehint = 0): # being careful to do in-place conversion so we don't accidently double our @@ -286,7 +287,7 @@ class _UnicodeReader(object): results = self.wrapped_file.readlines(sizehint) for i in xrange(len(results)): - results[i] = self._to_unicode(results[i]) + results[i] = stem.util.str_tools.to_unicode(results[i]) return results @@ -296,20 +297,6 @@ class _UnicodeReader(object): def tell(self): return self.wrapped_file.tell() - def _to_unicode(self, msg): - if msg is None: - return msg - - if stem.prereq.is_python_3(): - is_unicode = isinstance(msg, str) - else: - is_unicode = isinstance(msg, unicode) - - if is_unicode: - return msg - else: - return msg.decode("utf-8", "replace") - def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False, skip = False, end_position = None, include_ending_keyword = False): """ diff --git a/stem/util/str_tools.py b/stem/util/str_tools.py index 2956419..37756ed 100644 --- a/stem/util/str_tools.py +++ b/stem/util/str_tools.py @@ -6,6 +6,7 @@ Toolkit for various string activity. :: to_bytes - normalizes string ASCII bytes + to_unicode - normalizes string to unicode to_camel_case - converts a string to camel case get_size_label - human readable label for a number of bytes get_time_label - human readable label for a number of seconds @@ -55,10 +56,21 @@ if stem.prereq.is_python_3(): return codecs.latin_1_encode(msg)[0] else: return msg + + def _to_unicode(msg): + if msg is not None and not isinstance(msg, str): + return msg.decode("utf-8", "replace") + else: + return msg else: def _to_bytes(msg): return msg + def _to_unicode(msg): + if msg is not None and not isinstance(msg, unicode): + return msg.decode("utf-8", "replace") + else: + return msg def to_bytes(msg): """ @@ -76,6 +88,20 @@ def to_bytes(msg): return _to_bytes(msg) +def to_unicode(msg): + """ + Provides the unicode string for the given ASCII bytes. This is purely to + provide python 3 compatability, normalizing the unicode/ASCII change in the + version bump. + + :param msg label: string to be converted + + :returns: unicode conversion + """ + + return _to_unicode(msg) + + def to_camel_case(label, divider = "_", joiner = " "): """ Converts the given string to camel case, ie: diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 8eb262a..549cea8 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -12,6 +12,7 @@ import stem.control import stem.descriptor import stem.descriptor.server_descriptor import stem.exit_policy +import stem.util.str_tools import stem.version import test.runner @@ -198,10 +199,10 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_file = open_desc("non-ascii_descriptor") descriptor_file.readline() # strip header - descriptor_contents = descriptor_file.read() + descriptor_contents = stem.util.str_tools.to_unicode(descriptor_file.read()) descriptor_file.close() - expected_contact = "2048R/F171EC1F Johan Bl\xc3\xa5b\xc3\xa4ck \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf" + expected_contact = b"2048R/F171EC1F Johan Bl\xc3\xa5b\xc3\xa4ck \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf".decode("utf-8", "replace") desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) self.assertEquals("torrelay389752132", desc.nickname) diff --git a/test/settings.cfg b/test/settings.cfg index e7e659b..6ad4b59 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -157,8 +157,9 @@ target.torrc RUN_PTRACE => PORT, PTRACE pyflakes.ignore stem/prereq.py => 'RSA' imported but unused pyflakes.ignore stem/prereq.py => 'asn1' imported but unused pyflakes.ignore stem/prereq.py => 'long_to_bytes' imported but unused -pyflakes.ignore stem/descriptor/__init__.py => redefinition of unused 'OrderedDict' from line 34 -pyflakes.ignore stem/util/str_tools.py => redefinition of function '_to_bytes' from line 53 +pyflakes.ignore stem/descriptor/__init__.py => redefinition of unused 'OrderedDict' from line 35 +pyflakes.ignore stem/util/str_tools.py => redefinition of function '_to_bytes' from line 54 +pyflakes.ignore stem/util/str_tools.py => redefinition of function '_to_unicode' from line 60 pyflakes.ignore test/mocking.py => undefined name 'builtins' pyflakes.ignore test/unit/response/events.py => 'from stem import *' used; unable to detect undefined names

1 0

[stem/master] Checking that to_bytes has unicode before converting
by atagar＠torproject.org 02 Feb '13

02 Feb '13

commit f4ee5ab36ee512f7cb904bb371f593f7e2f1f8a6 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 30 09:39:45 2013 -0800 Checking that to_bytes has unicode before converting Adding a check to the to_bytes() helper so we don't attempt to convert ASCII bytes to ASCII bytes (which doesn't work so well). ====================================================================== ERROR: test_old_descriptor ---------------------------------------------------------------------- Traceback: File "/home/atagar/Desktop/stem/test/data/python3/test/integ/descriptor/server_descriptor.py", line 120, in test_old_descriptor desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) File "/home/atagar/Desktop/stem/test/data/python3/stem/descriptor/server_descriptor.py", line 642, in __init__ self._validate_content() File "/home/atagar/Desktop/stem/test/data/python3/stem/descriptor/server_descriptor.py", line 687, in _validate_content key_der_as_hash = hashlib.sha1(stem.util.str_tools.to_bytes(key_as_bytes)).hexdigest() File "/home/atagar/Desktop/stem/test/data/python3/stem/util/str_tools.py", line 73, in to_bytes return _to_bytes(msg) File "/home/atagar/Desktop/stem/test/data/python3/stem/util/str_tools.py", line 54, in _to_bytes return codecs.latin_1_encode(msg)[0] TypeError: Can't convert 'bytes' object to str implicitly --- stem/util/str_tools.py | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/stem/util/str_tools.py b/stem/util/str_tools.py index 022874e..2956419 100644 --- a/stem/util/str_tools.py +++ b/stem/util/str_tools.py @@ -51,7 +51,10 @@ if stem.prereq.is_python_3(): import codecs def _to_bytes(msg): - return codecs.latin_1_encode(msg)[0] + if isinstance(msg, str): + return codecs.latin_1_encode(msg)[0] + else: + return msg else: def _to_bytes(msg): return msg

1 0

[stem/master] Skipping deletion of pyc in __pycache__
by atagar＠torproject.org 02 Feb '13

02 Feb '13

commit d9d46cb5b35f586bcf8025cb7682c4ad79e57e15 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jan 31 08:31:53 2013 -0800 Skipping deletion of pyc in __pycache__ I disabled the deletion of orphaned pyc files when testing python 3 but on reflection that wasn't enough. Python 2.x test runs still delete the python 3 bytecode. Changing the orphaned check to skip those files. --- run_tests.py | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) diff --git a/run_tests.py b/run_tests.py index d18ec9d..feab3a7 100755 --- a/run_tests.py +++ b/run_tests.py @@ -271,22 +271,22 @@ def load_user_configuration(test_config): def _clean_orphaned_pyc(): - # If we're running python 3 then the *.pyc files are no longer bundled with - # the *.py. Rather, they're in a __pycache__ directory. - # - # At the moment there's no point in checking for orphaned bytecode with - # python 3 because it's an exported copy of the python 2 codebase, so - # skipping. - - if CONFIG["argument.python3"]: - return - test.output.print_noline(" checking for orphaned .pyc files... ", *test.runner.STATUS_ATTR) orphaned_pyc = [] for base_dir in ('stem', 'test', 'run_tests.py'): for pyc_path in test.check_whitespace._get_files_with_suffix(base_dir, ".pyc"): + # If we're running python 3 then the *.pyc files are no longer bundled + # with the *.py. Rather, they're in a __pycache__ directory. + # + # At the moment there's no point in checking for orphaned bytecode with + # python 3 because it's an exported copy of the python 2 codebase, so + # skipping. + + if "__pycache__" in pyc_path: + continue + if not os.path.exists(pyc_path[:-1]): orphaned_pyc.append(pyc_path)

1 0

[stem/master] Using stem.descriptor.parse_file() for extrainfo integ tests
by atagar＠torproject.org 02 Feb '13

02 Feb '13

commit 4815afe52c4af2d4cb3d687b35b7012239d7471b Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jan 31 09:54:27 2013 -0800 Using stem.descriptor.parse_file() for extrainfo integ tests Going through parse_file() so we do the proper unicode conversion. ====================================================================== ERROR: test_cached_descriptor ---------------------------------------------------------------------- Traceback: File "/home/atagar/Desktop/stem/test/data/python3/test/integ/descriptor/extrainfo_descriptor.py", line 150, in test_cached_descriptor for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file): File "/home/atagar/Desktop/stem/test/data/python3/stem/descriptor/extrainfo_descriptor.py", line 155, in _parse_file extrainfo_content = stem.descriptor._read_until_keywords("router-signature", descriptor_file) File "/home/atagar/Desktop/stem/test/data/python3/stem/descriptor/__init__.py", line 350, in _read_until_keywords line_match = KEYWORD_LINE.match(line) TypeError: can't use a string pattern on a bytes-like object --- test/integ/descriptor/extrainfo_descriptor.py | 14 ++++---------- 1 files changed, 4 insertions(+), 10 deletions(-) diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 4cb9803..ab47a2b 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -8,7 +8,7 @@ import datetime import os import unittest -import stem.descriptor.extrainfo_descriptor +import stem.descriptor import test.runner from stem.descriptor.extrainfo_descriptor import DirResponse @@ -22,9 +22,6 @@ class TestExtraInfoDescriptor(unittest.TestCase): """ descriptor_file = open(get_resource("extrainfo_relay_descriptor"), 'rb') - descriptor_file.readline() # strip header - descriptor_contents = descriptor_file.read() - descriptor_file.close() expected_signature = """-----BEGIN SIGNATURE----- K5FSywk7qvw/boA4DQcqkls6Ize5vcBYfhQ8JnOeRQC9+uDxbnpm3qaYN9jZ8myj @@ -32,7 +29,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw 7LZqklu+gVvhMKREpchVqlAwXkWR44VENm24Hs+mT3M= -----END SIGNATURE-----""" - desc = stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor(descriptor_contents) + desc = next(stem.descriptor.parse_file(descriptor_file, "extra-info 1.0")) self.assertEquals("NINJA", desc.nickname) self.assertEquals("B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48", desc.fingerprint) self.assertEquals(datetime.datetime(2012, 5, 5, 17, 3, 50), desc.published) @@ -71,9 +68,6 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw """ descriptor_file = open(get_resource("extrainfo_bridge_descriptor"), 'rb') - descriptor_file.readline() # strip header - descriptor_contents = descriptor_file.read() - descriptor_file.close() expected_dir_v2_responses = { DirResponse.OK: 0, @@ -92,7 +86,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw DirResponse.BUSY: 0, } - desc = stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(descriptor_contents) + desc = next(stem.descriptor.parse_file(descriptor_file, "bridge-extra-info 1.0")) self.assertEquals("ec2bridgereaac65a3", desc.nickname) self.assertEquals("1EC248422B57D9C0BD751892FE787585407479A4", desc.fingerprint) self.assertEquals(datetime.datetime(2012, 6, 8, 2, 21, 27), desc.published) @@ -147,7 +141,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw return with open(descriptor_path, 'rb') as descriptor_file: - for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file): + for desc in stem.descriptor.parse_file(descriptor_file, "extra-info 1.0"): unrecognized_lines = desc.get_unrecognized_lines() if desc.dir_v2_responses_unknown:

1 0

[stem/master] Using binary mode when reading descriptors
by atagar＠torproject.org 02 Feb '13

02 Feb '13

commit 3afa4346d102d99eab93929be738cdb594105d9b Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jan 31 09:44:18 2013 -0800 Using binary mode when reading descriptors Now Damian, repleat after me: text mode is bad. In python 2.x text mode and binary mode seem to be indistinguishable, but in python 3 there's one tiny little difference: text mode is around 33x slower. The integ test that read the cached-consensus took over five minutes (by comparison to ten seconds with python 2.7), and in one case simply hung for twenty minutes before I killed it. I'm not aware of any disadvantage to using binary mode, so opting for that. --- stem/descriptor/__init__.py | 11 +++++- stem/descriptor/reader.py | 7 +--- stem/descriptor/server_descriptor.py | 2 +- stem/util/str_tools.py | 1 + test/integ/descriptor/__init__.py | 15 ------- test/integ/descriptor/extrainfo_descriptor.py | 8 ++-- test/integ/descriptor/networkstatus.py | 18 ++++---- test/integ/descriptor/server_descriptor.py | 55 ++++++++----------------- test/unit/descriptor/server_descriptor.py | 3 +- 9 files changed, 45 insertions(+), 75 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index bebcdcc..788b442 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -83,7 +83,7 @@ def parse_file(descriptor_file, descriptor_type = None, path = None, validate = tordnsel 1.0 **unsupported** ===================================== ===== - If you're using python 3 then beware of the open() function's universal + If you're using **python 3** then beware of the open() function's universal newline translation. By default open() converts all common line endings (NL, CR, and CRNL) into NL. In some edge cases this can cause us to misparse content. To disable newline translation set the **newline** to an empty @@ -93,6 +93,15 @@ def parse_file(descriptor_file, descriptor_type = None, path = None, validate = my_descriptor_file = open(descrptor_path, newline='') + What's more, python 3's read performance in **text mode** is deplorably bad + (my testing with python 3.2 shows it to be 33x slower). Using **binary mode** + is strongly suggested. If you do this then newline translation is + automatically disabled... + + :: + + my_descriptor_file = open(descriptor_path, 'rb') + :param file descriptor_file: opened file with the descriptor contents :param str descriptor_type: `descriptor type <https://metrics.torproject.org/formats.html#descriptortypes>`_, this is guessed if not provided :param str path: absolute path to the file's location on disk diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index ef35fc0..f00e0aa 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -513,12 +513,7 @@ class DescriptorReader(object): try: self._notify_read_listeners(target) - if stem.prereq.is_python_3(): - target_file = open(target, newline = '') - else: - target_file = open(target) - - with target_file as target_file: + with open(target, 'rb') as target_file: for desc in stem.descriptor.parse_file(target_file, validate = self._validate, path = target): if self._is_stopped.isSet(): return diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 2376a56..95a4ef0 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -124,7 +124,7 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True): if descriptor_content: # strip newlines from annotations - annotations = map(str.strip, annotations) + annotations = map(unicode.strip, annotations) descriptor_text = "".join(descriptor_content) diff --git a/stem/util/str_tools.py b/stem/util/str_tools.py index 37756ed..2733875 100644 --- a/stem/util/str_tools.py +++ b/stem/util/str_tools.py @@ -72,6 +72,7 @@ else: else: return msg + def to_bytes(msg): """ Provides the ASCII bytes for the given string. This is purely to provide diff --git a/test/integ/descriptor/__init__.py b/test/integ/descriptor/__init__.py index 31421f1..58f9f81 100644 --- a/test/integ/descriptor/__init__.py +++ b/test/integ/descriptor/__init__.py @@ -12,8 +12,6 @@ __all__ = [ import os -import stem.prereq - DESCRIPTOR_TEST_DATA = os.path.join(os.path.dirname(__file__), "data") @@ -23,16 +21,3 @@ def get_resource(filename): """ return os.path.join(DESCRIPTOR_TEST_DATA, filename) - - -def open_desc(filename, absolute = False): - """ - Provides the file for a given descriptor in our data directory. - """ - - path = filename if absolute else get_resource(filename) - - if stem.prereq.is_python_3(): - return open(path, newline = '') - else: - return open(path) diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 0bcc23f..4cb9803 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -12,7 +12,7 @@ import stem.descriptor.extrainfo_descriptor import test.runner from stem.descriptor.extrainfo_descriptor import DirResponse -from test.integ.descriptor import open_desc +from test.integ.descriptor import get_resource class TestExtraInfoDescriptor(unittest.TestCase): @@ -21,7 +21,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): Parses and checks our results against an extrainfo descriptor from metrics. """ - descriptor_file = open_desc("extrainfo_relay_descriptor") + descriptor_file = open(get_resource("extrainfo_relay_descriptor"), 'rb') descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close() @@ -70,7 +70,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw metrics. """ - descriptor_file = open_desc("extrainfo_bridge_descriptor") + descriptor_file = open(get_resource("extrainfo_bridge_descriptor"), 'rb') descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close() @@ -146,7 +146,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw test.runner.skip(self, "(no cached descriptors)") return - with open_desc(descriptor_path, absolute = True) as descriptor_file: + with open(descriptor_path, 'rb') as descriptor_file: for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file): unrecognized_lines = desc.get_unrecognized_lines() diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index 19c3d8f..df676b4 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -15,7 +15,7 @@ import stem.descriptor.networkstatus import stem.version import test.runner -from test.integ.descriptor import get_resource, open_desc +from test.integ.descriptor import get_resource class TestNetworkStatus(unittest.TestCase): @@ -42,7 +42,7 @@ class TestNetworkStatus(unittest.TestCase): return count = 0 - with open_desc(consensus_path, absolute = True) as descriptor_file: + with open(consensus_path, 'rb') as descriptor_file: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV3 for router in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type): @@ -89,7 +89,7 @@ class TestNetworkStatus(unittest.TestCase): return count = 0 - with open_desc(consensus_path, absolute = True) as descriptor_file: + with open(consensus_path, 'rb') as descriptor_file: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV3 for router in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, is_microdescriptor = True): @@ -120,7 +120,7 @@ class TestNetworkStatus(unittest.TestCase): consensus_path = get_resource("metrics_consensus") for specify_type in (True, False): - with open_desc(consensus_path, absolute = True) as descriptor_file: + with open(consensus_path, 'rb') as descriptor_file: if specify_type: descriptors = stem.descriptor.parse_file(descriptor_file, "network-status-consensus-3 1.0", path = consensus_path) else: @@ -142,7 +142,7 @@ class TestNetworkStatus(unittest.TestCase): consensus_path = get_resource("bridge_network_status") - with open_desc(consensus_path, absolute = True) as descriptor_file: + with open(consensus_path, 'rb') as descriptor_file: descriptors = stem.descriptor.parse_file(descriptor_file, path = consensus_path) router = next(descriptors) @@ -245,7 +245,7 @@ mfWcW0b+jsrXcJoCxV5IrwCDF3u1aC3diwZY6yiG186pwWbOwE41188XI2DeYPwE I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= -----END SIGNATURE-----""" - with open_desc("cached-consensus") as descriptor_file: + with open(get_resource("cached-consensus"), 'rb') as descriptor_file: document = stem.descriptor.networkstatus.NetworkStatusDocumentV3(descriptor_file.read(), default_params = False) self.assertEquals(3, document.version) @@ -316,7 +316,7 @@ nTA+fD8JQqpPtb8b0SnG9kwy75eS//sRu7TErie2PzGMxrf9LH0LAgMBAAE= TpQQk3nNQF8z6UIvdlvP+DnJV4izWVkQEZgUZgIVM0E= -----END SIGNATURE-----""" - with open_desc("cached-consensus-v2") as descriptor_file: + with open(get_resource("cached-consensus-v2"), 'rb') as descriptor_file: descriptor_file.readline() # strip header document = stem.descriptor.networkstatus.NetworkStatusDocumentV2(descriptor_file.read()) @@ -381,7 +381,7 @@ TpQQk3nNQF8z6UIvdlvP+DnJV4izWVkQEZgUZgIVM0E= vote_path = get_resource("metrics_vote") - with open_desc(vote_path, absolute = True) as descriptor_file: + with open(vote_path, 'rb') as descriptor_file: descriptors = stem.descriptor.parse_file(descriptor_file, path = vote_path) router = next(descriptors) @@ -443,7 +443,7 @@ JZ/1HL9sHyZfo6bwaC6YSM9PNiiY6L7rnGpS7UkHiFI+M96VCMorvjm5YPs3FioJ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= -----END SIGNATURE-----""" - with open_desc("vote") as descriptor_file: + with open(get_resource("vote"), 'rb') as descriptor_file: document = stem.descriptor.networkstatus.NetworkStatusDocumentV3(descriptor_file.read(), default_params = False) self.assertEquals(3, document.version) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 549cea8..c1cd742 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -12,11 +12,10 @@ import stem.control import stem.descriptor import stem.descriptor.server_descriptor import stem.exit_policy -import stem.util.str_tools import stem.version import test.runner -from test.integ.descriptor import open_desc +from test.integ.descriptor import get_resource class TestServerDescriptor(unittest.TestCase): @@ -25,10 +24,7 @@ class TestServerDescriptor(unittest.TestCase): Parses and checks our results against a server descriptor from metrics. """ - descriptor_file = open_desc("example_descriptor") - descriptor_file.readline() # strip header - descriptor_contents = descriptor_file.read() - descriptor_file.close() + descriptor_file = open(get_resource("example_descriptor"), 'rb') expected_family = [ "$0CE3CFB1E9CC47B63EA8869813BF6FAB7D4540C1", @@ -59,7 +55,7 @@ dskLSPz8beUW7bzwDjR6EVNGpyoZde83Ejvau+5F2c6cGnlu91fiZN3suE88iE6e Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= -----END SIGNATURE-----""" - desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) + desc = next(stem.descriptor.parse_file(descriptor_file, "server-descriptor 1.0")) self.assertEquals("caerSidi", desc.nickname) self.assertEquals("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", desc.fingerprint) self.assertEquals("71.35.133.197", desc.address) @@ -95,7 +91,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses and checks our results against a server descriptor from metrics. """ - with open_desc("metrics_server_desc_multiple") as descriptor_file: + with open(get_resource("metrics_server_desc_multiple"), 'rb') as descriptor_file: descriptors = list(stem.descriptor.parse_file(descriptor_file, "server-descriptor 1.0")) self.assertEquals(2, len(descriptors)) @@ -111,12 +107,9 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses a relay server descriptor from 2005. """ - descriptor_file = open_desc("old_descriptor") - descriptor_file.readline() # strip header - descriptor_contents = descriptor_file.read() - descriptor_file.close() + descriptor_file = open(get_resource("old_descriptor"), 'rb') - desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) + desc = next(stem.descriptor.parse_file(descriptor_file, "server-descriptor 1.0")) self.assertEquals("krypton", desc.nickname) self.assertEquals("3E2F63E2356F52318B536A12B6445373808A5D6C", desc.fingerprint) self.assertEquals("212.37.39.59", desc.address) @@ -173,8 +166,8 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= test.runner.skip(self, "(no cached descriptors)") return - with open_desc(descriptor_path, absolute = True) as descriptor_file: - for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file): + with open(descriptor_path, 'rb') as descriptor_file: + for desc in stem.descriptor.parse_file(descriptor_file, "server-descriptor 1.0"): # the following attributes should be deprecated, and not appear in the wild self.assertEquals(None, desc.read_history_end) self.assertEquals(None, desc.write_history_end) @@ -197,14 +190,11 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses a descriptor with non-ascii content. """ - descriptor_file = open_desc("non-ascii_descriptor") - descriptor_file.readline() # strip header - descriptor_contents = stem.util.str_tools.to_unicode(descriptor_file.read()) - descriptor_file.close() + descriptor_file = open(get_resource("non-ascii_descriptor"), 'rb') expected_contact = b"2048R/F171EC1F Johan Bl\xc3\xa5b\xc3\xa4ck \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf".decode("utf-8", "replace") - desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) + desc = next(stem.descriptor.parse_file(descriptor_file, "server-descriptor 1.0")) self.assertEquals("torrelay389752132", desc.nickname) self.assertEquals("5D47E91A1F7421A4E3255F4D04E534E9A21407BB", desc.fingerprint) self.assertEquals("130.243.230.116", desc.address) @@ -237,12 +227,8 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= returns ('\r' entries). """ - descriptor_file = open_desc("cr_in_contact_line") - descriptor_file.readline() # strip header - descriptor_contents = descriptor_file.read() - descriptor_file.close() - - desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) + descriptor_file = open(get_resource("cr_in_contact_line"), 'rb') + desc = next(stem.descriptor.parse_file(descriptor_file, "server-descriptor 1.0")) self.assertEquals("pogonip", desc.nickname) self.assertEquals("6DABD62BC65D4E6FE620293157FC76968DAB9C9B", desc.fingerprint) @@ -263,12 +249,8 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= where we shouldn't be. """ - descriptor_file = open_desc("negative_uptime") - descriptor_file.readline() # strip header - descriptor_contents = descriptor_file.read() - descriptor_file.close() - - desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) + descriptor_file = open(get_resource("negative_uptime"), 'rb') + desc = next(stem.descriptor.parse_file(descriptor_file, "server-descriptor 1.0")) self.assertEquals("TipTor", desc.nickname) self.assertEquals("137962D4931DBF08A24E843288B8A155D6D2AEDD", desc.fingerprint) @@ -277,7 +259,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= # modify the relay version so it's after when the negative uptime bug # should appear - descriptor_contents = descriptor_contents.replace("Tor 0.1.1.25", "Tor 0.1.2.7") + descriptor_contents = str(desc).replace("Tor 0.1.1.25", "Tor 0.1.2.7") self.assertRaises(ValueError, stem.descriptor.server_descriptor.RelayDescriptor, descriptor_contents) def test_bridge_descriptor(self): @@ -285,10 +267,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses a bridge descriptor. """ - descriptor_file = open_desc("bridge_descriptor") - descriptor_file.readline() # strip header - descriptor_contents = descriptor_file.read() - descriptor_file.close() + descriptor_file = open(get_resource("bridge_descriptor"), 'rb') expected_family = [ "$CE396C72A3D0880F74C064FEA79D68C15BD380B9", @@ -296,7 +275,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= "$8C8A470D7C23151665A7B84E75E89FCC205A3304", ] - desc = stem.descriptor.server_descriptor.BridgeDescriptor(descriptor_contents) + desc = next(stem.descriptor.parse_file(descriptor_file, "bridge-server-descriptor 1.0")) self.assertEquals("Unnamed", desc.nickname) self.assertEquals("AE54E28ED069CDF45F3009F963EE3B3D6FA26A2E", desc.fingerprint) self.assertEquals("10.45.227.253", desc.address) diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 00fd96f..66ec460 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -9,6 +9,7 @@ import unittest import stem.descriptor.server_descriptor import stem.exit_policy import stem.prereq +import stem.util.str_tools from stem.descriptor.server_descriptor import RelayDescriptor, BridgeDescriptor @@ -209,7 +210,7 @@ class TestServerDescriptor(unittest.TestCase): desc_text += "\ntrailing text that should be ignored, ho hum" # running _parse_file should provide an iterator with a single descriptor - desc_iter = stem.descriptor.server_descriptor._parse_file(StringIO.StringIO(desc_text)) + desc_iter = stem.descriptor.server_descriptor._parse_file(StringIO.StringIO(stem.util.str_tools.to_unicode(desc_text))) desc_entries = list(desc_iter) self.assertEquals(1, len(desc_entries)) desc = desc_entries[0]

1 0

← Newer
1
...
67
68
69
70
71
72
73
...
79
Older →

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits February 2013