May 2012 - tor-commits - lists.torproject.org

[stem/master] Helper function for parsing timestamp/intervals
by atagar＠torproject.org 14 May '12

14 May '12

commit b8e08ce69d30d10cfa6879b7205f62e988fea0e1 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue May 8 09:53:13 2012 -0700 Helper function for parsing timestamp/intervals Several extra-info descriptor lines are of the form... 'YYYY-MM-DD HH:MM:SS (NSEC s) *' So making a helper function to parse it. Using it for the server descriptor's read/write-history entries and it improves code readability quite a bit... --- stem/descriptor/extrainfo_descriptor.py | 35 +++++++++++++++++++ stem/descriptor/server_descriptor.py | 57 ++++++++++++------------------ 2 files changed, 58 insertions(+), 34 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index e13787f..ca34ca1 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -21,6 +21,7 @@ ExtraInfoDescriptor - Tor extra-info descriptor. +- get_unrecognized_lines - lines with unrecognized content """ +import re import datetime import stem.descriptor @@ -99,6 +100,40 @@ def parse_file(descriptor_file, validate = True): yield ExtraInfoDescriptor("".join(extrainfo_content), validate) else: break # done parsing file +def _parse_timestamp_and_interval(keyword, content): + """ + Parses a 'YYYY-MM-DD HH:MM:SS (NSEC s) *' entry. + + Arguments: + keyword (str) - line's keyword + content (str) - line content to be parsed + + Returns: + tuple of the form... + (timestamp (datetime), interval (int), remaining content (str)) + + Raises: + ValueError if the content is malformed + """ + + line = "%s %s" % (keyword, content) + content_match = re.match("^(.*) $([0-9]+) s$( .*)?$", content) + + if not content_match: + raise ValueError("Malformed %s line: %s" % (keyword, line)) + + timestamp_str, interval, remainder = content_match.groups() + if remainder: remainder = remainder[1:] # remove leading space + + if not interval.isdigit(): + raise ValueError("%s line's interval wasn't a number: %s" % (keyword, line)) + + try: + timestamp = datetime.datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S") + return timestamp, int(interval), remainder + except ValueError: + raise ValueError("%s line's timestamp wasn't parseable: %s" % (keyword, line)) + class ExtraInfoDescriptor(stem.descriptor.Descriptor): """ Extra-info descriptor document. diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 8513b86..2a9d72f 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -29,6 +29,7 @@ import hashlib import datetime import stem.descriptor +import stem.descriptor.extrainfo_descriptor import stem.version import stem.util.connection import stem.util.tor_tools @@ -467,43 +468,31 @@ class ServerDescriptor(stem.descriptor.Descriptor): elif keyword == "eventdns": self.eventdns = value == "1" elif keyword in ("read-history", "write-history"): - is_read = keyword == "read-history" - - if is_read: self.read_history = value - else: self.write_history = value - - value_match = re.match("^(.*) $([0-9]+) s$ (.*)$", value) - - if not value_match: - if not validate: continue - raise ValueError("Malformed %s line: %s" % (keyword, line)) - - end_value, interval_value, history_values = value_match.groups() - try: - end_datetime = datetime.datetime.strptime(end_value, "%Y-%m-%d %H:%M:%S") + timestamp, interval, remainder = \ + stem.descriptor.extrainfo_descriptor._parse_timestamp_and_interval(keyword, value) - if is_read: self.read_history_end = end_datetime - else: self.write_history_end = end_datetime - except ValueError: - if validate: - raise ValueError("%s line's time wasn't parseable: %s" % (keyword, line)) - - if interval_value.isdigit(): - if is_read: self.read_history_interval = int(interval_value) - else: self.write_history_interval = int(interval_value) - elif validate: - raise ValueError("%s line's interval wasn't a number: %s" % (keyword, line)) - - if history_values != '': - for sampling in history_values.split(","): - if sampling.isdigit(): - if is_read: self.read_history_values.append(int(sampling)) - else: self.write_history_values.append(int(sampling)) + try: + if remainder: + history_values = [int(entry) for entry in remainder.split(",")] else: - if validate: - raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) - else: break + history_values = [] + except ValueError: + raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) + + if keyword == "read-history": + self.read_history = value + self.read_history_end = timestamp + self.read_history_interval = interval + self.read_history_values = history_values + else: + self.write_history = value + self.write_history_end = timestamp + self.write_history_interval = interval + self.write_history_values = history_values + except ValueError, exc: + if not validate: continue + else: raise exc else: self._unrecognized_lines.append(line)

1 0

[stem/master] Parsing read/write-history lines in extrainfo descriptors
by atagar＠torproject.org 14 May '12

14 May '12

commit fa6d81cf69d5590f6bcb0ad94cf34a7b313f02c2 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed May 9 08:09:00 2012 -0700 Parsing read/write-history lines in extrainfo descriptors Copying the code for parsing from the server descriptors. I suspect that this isn't yet exercised by the tests - I'm next gonna improve our integ tests then check if this would be appropriate for a unit test too. --- stem/descriptor/extrainfo_descriptor.py | 25 +++++++++++++++++++++++++ 1 files changed, 25 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index ca34ca1..2ea8829 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -263,6 +263,31 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): except ValueError: if validate: raise ValueError("Published line's time wasn't parseable: %s" % line) + elif keyword in ("read-history", "write-history"): + try: + timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value) + + try: + if remainder: + history_values = [int(entry) for entry in remainder.split(",")] + else: + history_values = [] + except ValueError: + raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) + + if keyword == "read-history": + self.read_history = value + self.read_history_end = timestamp + self.read_history_interval = interval + self.read_history_values = history_values + else: + self.write_history = value + self.write_history_end = timestamp + self.write_history_interval = interval + self.write_history_values = history_values + except ValueError, exc: + if not validate: continue + else: raise exc elif keyword == "router-signature": if validate and not block_contents: raise ValueError("Router signature line must be followed by a signature block: %s" % line)

1 0

[stem/master] Extra-info integraion tests
by atagar＠torproject.org 14 May '12

14 May '12

commit 406205c2549e855140723bc42af874248a6b3026 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed May 9 08:42:48 2012 -0700 Extra-info integraion tests ExtraInfo counterparts for parsing a metrics descriptor and tor's cached file. There's some common bits with the server descriptor tests that I plan to move out. --- run_tests.py | 2 + test/integ/descriptor/__init__.py | 2 +- test/integ/descriptor/extrainfo_descriptor.py | 108 +++++++++++++++++++++++++ test/integ/descriptor/server_descriptor.py | 19 ++--- test/runner.py | 1 + 5 files changed, 119 insertions(+), 13 deletions(-) diff --git a/run_tests.py b/run_tests.py index bf2c2bc..85a4f65 100755 --- a/run_tests.py +++ b/run_tests.py @@ -36,6 +36,7 @@ import test.integ.socket.control_message import test.integ.socket.control_socket import test.integ.descriptor.reader import test.integ.descriptor.server_descriptor +import test.integ.descriptor.extrainfo_descriptor import test.integ.util.conf import test.integ.util.system import test.integ.process @@ -108,6 +109,7 @@ INTEG_TESTS = ( test.integ.util.system.TestSystem, test.integ.descriptor.reader.TestDescriptorReader, test.integ.descriptor.server_descriptor.TestServerDescriptor, + test.integ.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, test.integ.version.TestVersion, test.integ.process.TestProcess, test.integ.socket.control_socket.TestControlSocket, diff --git a/test/integ/descriptor/__init__.py b/test/integ/descriptor/__init__.py index b143c2a..ec61ba7 100644 --- a/test/integ/descriptor/__init__.py +++ b/test/integ/descriptor/__init__.py @@ -2,5 +2,5 @@ Integration tests for stem.descriptor.* contents. """ -__all__ = ["reader", "server_descriptor"] +__all__ = ["reader", "extrainfo_descriptor", "server_descriptor"] diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py new file mode 100644 index 0000000..74ac7fb --- /dev/null +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -0,0 +1,108 @@ +""" +Integration tests for stem.descriptor.extrainfo_descriptor. +""" + +import os +import datetime +import unittest + +import stem.descriptor.extrainfo_descriptor +import test.runner + +my_dir = os.path.dirname(__file__) +DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") + +# 'test_cached_descriptor' is a lengthy test and uneffected by testing targets, +# so including a flag to prevent it from being ran multiple times + +RAN_CACHED_DESCRIPTOR_TEST = False + +class TestExtraInfoDescriptor(unittest.TestCase): + is_cached_descriptors_available = None + + def setUp(self): + if self.is_cached_descriptors_available is None: + test_dir = test.runner.get_runner().get_test_dir() + descriptor_path = os.path.join(test_dir, "cached-extrainfo") + self.is_cached_descriptors_available = os.path.exists(descriptor_path) + + def test_metrics_descriptor(self): + """ + Parses and checks our results against an extrainfo descriptor from metrics. + """ + + descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "extrainfo_descriptor") + + descriptor_file = open(descriptor_path) + descriptor_contents = descriptor_file.read() + descriptor_file.close() + + expected_signature = """-----BEGIN SIGNATURE----- +K5FSywk7qvw/boA4DQcqkls6Ize5vcBYfhQ8JnOeRQC9+uDxbnpm3qaYN9jZ8myj +k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw +7LZqklu+gVvhMKREpchVqlAwXkWR44VENm24Hs+mT3M= +-----END SIGNATURE-----""" + + desc = stem.descriptor.extrainfo_descriptor.ExtraInfoDescriptor(descriptor_contents) + self.assertEquals("NINJA", desc.nickname) + self.assertEquals("B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48", desc.fingerprint) + self.assertEquals(datetime.datetime(2012, 5, 5, 17, 3, 50), desc.published) + self.assertEquals(datetime.datetime(2012, 5, 5, 17, 2, 45), desc.read_history_end) + self.assertEquals(900, desc.read_history_interval) + self.assertEquals(datetime.datetime(2012, 5, 5, 17, 2, 45), desc.write_history_end) + self.assertEquals(900, desc.write_history_interval) + self.assertEquals(expected_signature, desc.signature) + + # TODO: still missing dirreq-read-history and dirreq-write-history + #self.assertEquals([], desc.get_unrecognized_lines()) + + # The read-history and write-history lines are pretty long so just checking + # the initial contents for the line and parsed values. + + read_start = "2012-05-05 17:02:45 (900 s) 3309568,9216,41984" + self.assertTrue(desc.read_history.startswith(read_start)) + + read_values_start = [3309568, 9216, 41984, 27648, 123904] + self.assertEquals(read_values_start, desc.read_history_values[:5]) + + write_start = "2012-05-05 17:02:45 (900 s) 1082368,19456,50176,272384" + self.assertTrue(desc.write_history.startswith(write_start)) + + write_values_start = [1082368, 19456, 50176, 272384, 485376] + self.assertEquals(write_values_start, desc.write_history_values[:5]) + + def test_cached_descriptor(self): + """ + Parses the cached descriptor file in our data directory, checking that it + doesn't raise any validation issues and looking for unrecognized descriptor + additions. + """ + + descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-extrainfo") + + if not self.is_cached_descriptors_available: + self.skipTest("(no cached descriptors)") + + global RAN_CACHED_DESCRIPTOR_TEST + + if RAN_CACHED_DESCRIPTOR_TEST: + self.skipTest("(already ran)") + else: + RAN_CACHED_DESCRIPTOR_TEST = True + + with open(descriptor_path) as descriptor_file: + for desc in stem.descriptor.extrainfo_descriptor.parse_file(descriptor_file): + # TODO: uncomment when we're done implementing the ExtraInfoDescriptor class + #unrecognized_lines = desc.get_unrecognized_lines() + unrecognized_lines = [] + + if unrecognized_lines: + # TODO: This isn't actually a problem, and rather than failing we + # should alert the user about these entries at the end of the tests + # (along with new events, getinfo options, and such). For now though + # there doesn't seem to be anything in practice to trigger this so + # failing to get our attention if it does. + + print "Unrecognized descriptor content: %s" % unrecognized_lines + self.fail() + diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 5ad700f..a74e4eb 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -20,17 +20,17 @@ DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") RAN_CACHED_DESCRIPTOR_TEST = False class TestServerDescriptor(unittest.TestCase): - is_descriptors_available = None + is_cached_descriptors_available = None def setUp(self): # If this is our first time running the integ tests and we didn't wait for # a full tor initialization then the cached descriptors won't exist yet. # Noting if they exist or not since some tests need them. - if self.is_descriptors_available is None: + if self.is_cached_descriptors_available is None: test_dir = test.runner.get_runner().get_test_dir() descriptor_path = os.path.join(test_dir, "cached-descriptors") - self.is_descriptors_available = os.path.exists(descriptor_path) + self.is_cached_descriptors_available = os.path.exists(descriptor_path) def test_metrics_descriptor(self): """ @@ -43,8 +43,6 @@ class TestServerDescriptor(unittest.TestCase): descriptor_contents = descriptor_file.read() descriptor_file.close() - expected_published = datetime.datetime(2012, 3, 1, 17, 15, 27) - expected_family = [ "$0CE3CFB1E9CC47B63EA8869813BF6FAB7D4540C1", "$1FD187E8F69A9B74C9202DC16A25B9E7744AB9F6", @@ -85,7 +83,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(stem.version.Version("0.2.1.30"), desc.tor_version) self.assertEquals("Linux x86_64", desc.operating_system) self.assertEquals(588217, desc.uptime) - self.assertEquals(expected_published, desc.published) + self.assertEquals(datetime.datetime(2012, 3, 1, 17, 15, 27), desc.published) self.assertEquals("www.atagar.com/contact", desc.contact) self.assertEquals(["1", "2"], desc.link_protocols) self.assertEquals(["1"], desc.circuit_protocols) @@ -170,7 +168,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-descriptors") - if not self.is_descriptors_available: + if not self.is_cached_descriptors_available: self.skipTest("(no cached descriptors)") global RAN_CACHED_DESCRIPTOR_TEST @@ -211,7 +209,6 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_contents = descriptor_file.read() descriptor_file.close() - expected_published = datetime.datetime(2012, 3, 21, 16, 28, 14) expected_contact = "2048R/F171EC1F Johan Bl\xc3\xa5b\xc3\xa4ck \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf" desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) @@ -225,7 +222,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(stem.version.Version("0.2.2.35"), desc.tor_version) self.assertEquals("Linux x86_64", desc.operating_system) self.assertEquals(3103848, desc.uptime) - self.assertEquals(expected_published, desc.published) + self.assertEquals(datetime.datetime(2012, 3, 21, 16, 28, 14), desc.published) self.assertEquals(expected_contact, desc.contact) self.assertEquals(["1", "2"], desc.link_protocols) self.assertEquals(["1"], desc.circuit_protocols) @@ -303,8 +300,6 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_contents = descriptor_file.read() descriptor_file.close() - expected_published = datetime.datetime(2012, 3, 22, 17, 34, 38) - expected_family = [ "$CE396C72A3D0880F74C064FEA79D68C15BD380B9", "$AB8B00C00B1347BA80A88E548FAC9EDF701D7D0E", @@ -322,7 +317,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(stem.version.Version("0.2.3.12-alpha"), desc.tor_version) self.assertEquals("Linux x86_64", desc.operating_system) self.assertEquals(186, desc.uptime) - self.assertEquals(expected_published, desc.published) + self.assertEquals(datetime.datetime(2012, 3, 22, 17, 34, 38), desc.published) self.assertEquals("somebody", desc.contact) self.assertEquals(["1", "2"], desc.link_protocols) self.assertEquals(["1"], desc.circuit_protocols) diff --git a/test/runner.py b/test/runner.py index 6acc68e..034f1e5 100644 --- a/test/runner.py +++ b/test/runner.py @@ -63,6 +63,7 @@ ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) BASE_TORRC = """# configuration for stem integration tests DataDirectory %s SocksPort 0 +DownloadExtraInfo 1 """ # We make some paths relative to stem's base directory (the one above us)

1 0

[stem/master] Letting runner's get_test_dir() provide resources
by atagar＠torproject.org 14 May '12

14 May '12

commit 46fb6b272e45ab8c569e1439a53a985c5a8d827d Author: Damian Johnson <atagar(a)torproject.org> Date: Wed May 9 08:54:26 2012 -0700 Letting runner's get_test_dir() provide resources The most common use of the integ runner's get_test_dir() method is to get a path for resources in that directory. Providing an optional argument to get that path rather than making the caller use os.join(). --- test/integ/connection/authentication.py | 2 +- test/integ/descriptor/extrainfo_descriptor.py | 5 ++--- test/integ/descriptor/reader.py | 6 +++--- test/integ/descriptor/server_descriptor.py | 5 ++--- test/integ/util/conf.py | 2 +- test/runner.py | 14 ++++++++++---- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/test/integ/connection/authentication.py b/test/integ/connection/authentication.py index c209d2b..9d912b3 100644 --- a/test/integ/connection/authentication.py +++ b/test/integ/connection/authentication.py @@ -236,7 +236,7 @@ class TestAuthenticate(unittest.TestCase): """ auth_type = stem.connection.AuthMethod.COOKIE - auth_value = os.path.join(test.runner.get_runner().get_test_dir(), "fake_cookie") + auth_value = test.runner.get_runner().get_test_dir("fake_cookie") # we need to create a 32 byte cookie file to load from fake_cookie = open(auth_value, "w") diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 74ac7fb..8b59844 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -22,8 +22,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): def setUp(self): if self.is_cached_descriptors_available is None: - test_dir = test.runner.get_runner().get_test_dir() - descriptor_path = os.path.join(test_dir, "cached-extrainfo") + descriptor_path = test.runner.get_runner().get_test_dir("cached-extrainfo") self.is_cached_descriptors_available = os.path.exists(descriptor_path) def test_metrics_descriptor(self): @@ -78,7 +77,7 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw additions. """ - descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-extrainfo") + descriptor_path = test.runner.get_runner().get_test_dir("cached-extrainfo") if not self.is_cached_descriptors_available: self.skipTest("(no cached descriptors)") diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index 0d4a638..81635e0 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -24,7 +24,7 @@ DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") TAR_DESCRIPTORS = None def _get_processed_files_path(): - return os.path.join(test.runner.get_runner().get_test_dir(), "descriptor_processed_files") + return test.runner.get_runner().get_test_dir("descriptor_processed_files") def _make_processed_files_listing(contents): """ @@ -372,7 +372,7 @@ class TestDescriptorReader(unittest.TestCase): # types are solely based on file extensions so making something that looks # like an png image - test_path = os.path.join(test.runner.get_runner().get_test_dir(), "test.png") + test_path = test.runner.get_runner().get_test_dir("test.png") try: test_file = open(test_path, "w") @@ -399,7 +399,7 @@ class TestDescriptorReader(unittest.TestCase): Listens for a file that's skipped because we lack read permissions. """ - test_path = os.path.join(test.runner.get_runner().get_test_dir(), "secret_file") + test_path = test.runner.get_runner().get_test_dir("secret_file") try: test_file = open(test_path, "w") diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index a74e4eb..c94c709 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -28,8 +28,7 @@ class TestServerDescriptor(unittest.TestCase): # Noting if they exist or not since some tests need them. if self.is_cached_descriptors_available is None: - test_dir = test.runner.get_runner().get_test_dir() - descriptor_path = os.path.join(test_dir, "cached-descriptors") + descriptor_path = test.runner.get_runner().get_test_dir("cached-descriptors") self.is_cached_descriptors_available = os.path.exists(descriptor_path) def test_metrics_descriptor(self): @@ -166,7 +165,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= additions. """ - descriptor_path = os.path.join(test.runner.get_runner().get_test_dir(), "cached-descriptors") + descriptor_path = test.runner.get_runner().get_test_dir("cached-descriptors") if not self.is_cached_descriptors_available: self.skipTest("(no cached descriptors)") diff --git a/test/integ/util/conf.py b/test/integ/util/conf.py index 13d0579..a429f2a 100644 --- a/test/integ/util/conf.py +++ b/test/integ/util/conf.py @@ -46,7 +46,7 @@ Why are those arrows", coming my way?!?""" def _get_test_config_path(): - return os.path.join(test.runner.get_runner().get_test_dir(), "integ_test_cfg") + return test.runner.get_runner().get_test_dir("integ_test_cfg") def _make_config(contents): """ diff --git a/test/runner.py b/test/runner.py index 034f1e5..5ee9098 100644 --- a/test/runner.py +++ b/test/runner.py @@ -346,18 +346,24 @@ class Runner: return self._custom_opts - def get_test_dir(self): + def get_test_dir(self, resource = None): """ - Provides the absolute path for our testing directory. + Provides the absolute path for our testing directory or a file within it. + + Arguments: + resource (str) - file within our test directory to provide the path for Returns: - str with our test direcectory path + str with our test direcectory's absolute path or that of a file within it Raises: RunnerStopped if we aren't running """ - return self._get("_test_dir") + if resource: + return os.path.join(self._get("_test_dir"), resource) + else: + return self._get("_test_dir") def get_torrc_path(self): """

1 0

[stem/master] Parsing extrainfo published and router-signature lines
by atagar＠torproject.org 14 May '12

14 May '12

commit 7a91e7d85bbc24df130147cc8789e87a5c0f8911 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 7 08:39:34 2012 -0700 Parsing extrainfo published and router-signature lines Parsing the remaining two mandatory fields for extrainfo descriptors. These fields are identical to what's in server descriptors. --- stem/descriptor/extrainfo_descriptor.py | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 04530c0..fceaf60 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -20,6 +20,8 @@ parse_file - Iterates over the extra-info descriptors in a file. ExtraInfoDescriptor - Tor extra-info descriptor. """ +import datetime + import stem.descriptor # relay descriptors must have exactly one of the following @@ -105,6 +107,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): fingerprint (str) - fourty hex digits that make up the relay's fingerprint (*) published (datetime.datetime) - time in GMT when the descriptor was generated (*) geoip_db_digest (str) - sha1 of geoIP database file + signature (str) - signature for this extrainfo descriptor (*) read_history (str) - read-history line, always unset read_history_end (datetime.datetime) - end of the sampling interval @@ -144,6 +147,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.fingerprint = None self.published = None self.geoip_db_digest = None + self.signature = None self.read_history = None self.read_history_end = None @@ -210,4 +214,17 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.nickname = extra_info_comp[0] self.fingerprint = extra_info_comp[1] + elif keyword == "published": + # "published" YYYY-MM-DD HH:MM:SS + + try: + self.published = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + except ValueError: + if validate: + raise ValueError("Published line's time wasn't parseable: %s" % line) + elif keyword == "router-signature": + if validate and not block_contents: + raise ValueError("Router signature line must be followed by a signature block: %s" % line) + + self.signature = block_contents

1 0

[stem/master] Bumping runtime of async event test by a second
by atagar＠torproject.org 14 May '12

14 May '12

commit bd1d3f345f25835a52813449f4bb45bdddf32eaf Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 7 08:37:34 2012 -0700 Bumping runtime of async event test by a second Testing for asynchronous events relies on BW events happening every second by checking that we get two of them. This is not reliable since we don't have a hard assurance about its rate. Giving the test an extra second to make it more reliable. --- test/integ/control/base_controller.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/test/integ/control/base_controller.py b/test/integ/control/base_controller.py index cb5eaf2..4a8033a 100644 --- a/test/integ/control/base_controller.py +++ b/test/integ/control/base_controller.py @@ -167,7 +167,7 @@ class TestBaseController(unittest.TestCase): start_time = time.time() - while (time.time() - start_time) < 2: + while (time.time() - start_time) < 3: test.runner.exercise_controller(self, controller) # Concurrently shut down the controller. We need to do this in another

1 0

[stem/master] Helper function for getting descriptor resource paths
by atagar＠torproject.org 14 May '12

14 May '12

commit 02451060cf8d5fcc8834d455e0c12a24b08b876b Author: Damian Johnson <atagar(a)torproject.org> Date: Wed May 9 09:02:51 2012 -0700 Helper function for getting descriptor resource paths Making a common helper function for getting the paths of resources in the descriptor data directory rather than making everyone figure out a DESCRIPTOR_TEST_DATA constant. --- test/integ/descriptor/__init__.py | 11 +++++++++++ test/integ/descriptor/extrainfo_descriptor.py | 6 ++---- test/integ/descriptor/server_descriptor.py | 16 +++++++--------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/test/integ/descriptor/__init__.py b/test/integ/descriptor/__init__.py index ec61ba7..eb8da22 100644 --- a/test/integ/descriptor/__init__.py +++ b/test/integ/descriptor/__init__.py @@ -4,3 +4,14 @@ Integration tests for stem.descriptor.* contents. __all__ = ["reader", "extrainfo_descriptor", "server_descriptor"] +import os + +DESCRIPTOR_TEST_DATA = os.path.join(os.path.dirname(__file__), "data") + +def get_resource(filename): + """ + Provides the path for a file in our descriptor data directory. + """ + + return os.path.join(DESCRIPTOR_TEST_DATA, filename) + diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 8b59844..ec94d7c 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -8,9 +8,7 @@ import unittest import stem.descriptor.extrainfo_descriptor import test.runner - -my_dir = os.path.dirname(__file__) -DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") +import test.integ.descriptor # 'test_cached_descriptor' is a lengthy test and uneffected by testing targets, # so including a flag to prevent it from being ran multiple times @@ -30,7 +28,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): Parses and checks our results against an extrainfo descriptor from metrics. """ - descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "extrainfo_descriptor") + descriptor_path = test.integ.descriptor.get_resource("extrainfo_descriptor") descriptor_file = open(descriptor_path) descriptor_contents = descriptor_file.read() diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index c94c709..b43d1ee 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -10,9 +10,7 @@ import stem.control import stem.version import stem.descriptor.server_descriptor import test.runner - -my_dir = os.path.dirname(__file__) -DESCRIPTOR_TEST_DATA = os.path.join(my_dir, "data") +import test.integ.descriptor # 'test_cached_descriptor' is a lengthy test and uneffected by testing targets, # so including a flag to prevent it from being ran multiple times @@ -36,7 +34,7 @@ class TestServerDescriptor(unittest.TestCase): Parses and checks our results against a server descriptor from metrics. """ - descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor") + descriptor_path = test.integ.descriptor.get_resource("example_descriptor") descriptor_file = open(descriptor_path) descriptor_contents = descriptor_file.read() @@ -107,7 +105,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses a relay server descriptor from 2005. """ - descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "old_descriptor") + descriptor_path = test.integ.descriptor.get_resource("old_descriptor") descriptor_file = open(descriptor_path) descriptor_contents = descriptor_file.read() @@ -202,7 +200,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses a descriptor with non-ascii content. """ - descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "non-ascii_descriptor") + descriptor_path = test.integ.descriptor.get_resource("non-ascii_descriptor") descriptor_file = open(descriptor_path) descriptor_contents = descriptor_file.read() @@ -243,7 +241,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= returns ('\r' entries). """ - descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "cr_in_contact_line") + descriptor_path = test.integ.descriptor.get_resource("cr_in_contact_line") descriptor_file = open(descriptor_path) descriptor_contents = descriptor_file.read() @@ -270,7 +268,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= where we shouldn't be. """ - descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "negative_uptime") + descriptor_path = test.integ.descriptor.get_resource("negative_uptime") descriptor_file = open(descriptor_path) descriptor_contents = descriptor_file.read() @@ -293,7 +291,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= Parses a bridge descriptor. """ - descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "bridge_descriptor") + descriptor_path = test.integ.descriptor.get_resource("bridge_descriptor") descriptor_file = open(descriptor_path) descriptor_contents = descriptor_file.read()

1 0

[stem/master] Only checking cached descriptor existance when needed
by atagar＠torproject.org 14 May '12

14 May '12

commit 9d24a9090a00a8686b6eae3258bb6af84cd06087 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed May 9 09:11:46 2012 -0700 Only checking cached descriptor existance when needed Maybe once upon a time we had multiple tests that ran over the cached descriptors? I don't recall it, and we definitely don't right now so only checking for the existance of the cached-descriptor and cached-extrainfo files when we want to test against them. --- test/integ/descriptor/extrainfo_descriptor.py | 21 ++++----------------- test/integ/descriptor/server_descriptor.py | 22 ++++------------------ 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index ec94d7c..d62a164 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -10,19 +10,9 @@ import stem.descriptor.extrainfo_descriptor import test.runner import test.integ.descriptor -# 'test_cached_descriptor' is a lengthy test and uneffected by testing targets, -# so including a flag to prevent it from being ran multiple times - RAN_CACHED_DESCRIPTOR_TEST = False class TestExtraInfoDescriptor(unittest.TestCase): - is_cached_descriptors_available = None - - def setUp(self): - if self.is_cached_descriptors_available is None: - descriptor_path = test.runner.get_runner().get_test_dir("cached-extrainfo") - self.is_cached_descriptors_available = os.path.exists(descriptor_path) - def test_metrics_descriptor(self): """ Parses and checks our results against an extrainfo descriptor from metrics. @@ -75,18 +65,15 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw additions. """ - descriptor_path = test.runner.get_runner().get_test_dir("cached-extrainfo") - - if not self.is_cached_descriptors_available: - self.skipTest("(no cached descriptors)") - global RAN_CACHED_DESCRIPTOR_TEST + descriptor_path = test.runner.get_runner().get_test_dir("cached-extrainfo") if RAN_CACHED_DESCRIPTOR_TEST: self.skipTest("(already ran)") - else: - RAN_CACHED_DESCRIPTOR_TEST = True + elif not os.path.exists(descriptor_path): + self.skipTest("(no cached descriptors)") + RAN_CACHED_DESCRIPTOR_TEST = True with open(descriptor_path) as descriptor_file: for desc in stem.descriptor.extrainfo_descriptor.parse_file(descriptor_file): # TODO: uncomment when we're done implementing the ExtraInfoDescriptor class diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index b43d1ee..57f6879 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -18,17 +18,6 @@ import test.integ.descriptor RAN_CACHED_DESCRIPTOR_TEST = False class TestServerDescriptor(unittest.TestCase): - is_cached_descriptors_available = None - - def setUp(self): - # If this is our first time running the integ tests and we didn't wait for - # a full tor initialization then the cached descriptors won't exist yet. - # Noting if they exist or not since some tests need them. - - if self.is_cached_descriptors_available is None: - descriptor_path = test.runner.get_runner().get_test_dir("cached-descriptors") - self.is_cached_descriptors_available = os.path.exists(descriptor_path) - def test_metrics_descriptor(self): """ Parses and checks our results against a server descriptor from metrics. @@ -163,18 +152,15 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= additions. """ - descriptor_path = test.runner.get_runner().get_test_dir("cached-descriptors") - - if not self.is_cached_descriptors_available: - self.skipTest("(no cached descriptors)") - global RAN_CACHED_DESCRIPTOR_TEST + descriptor_path = test.runner.get_runner().get_test_dir("cached-descriptors") if RAN_CACHED_DESCRIPTOR_TEST: self.skipTest("(already ran)") - else: - RAN_CACHED_DESCRIPTOR_TEST = True + elif not os.path.exists(descriptor_path): + self.skipTest("(no cached descriptors)") + RAN_CACHED_DESCRIPTOR_TEST = True with open(descriptor_path) as descriptor_file: for desc in stem.descriptor.server_descriptor.parse_file(descriptor_file): # the following attributes should be deprecated, and not appear in the wild

1 0

[stem/master] Renaming read/write_history to read/write_history_line
by atagar＠torproject.org 14 May '12

14 May '12

commit 7178c5cb0a831d217c969ad38453dfdc85630983 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed May 9 09:27:16 2012 -0700 Renaming read/write_history to read/write_history_line The 'read_history' variable name was a bit vague, and generally the attribute isn't useful unlike its read_history_* counterparts. Renaming the variables to make it clear that read_history_line is the raw content for the read-history line. --- stem/descriptor/extrainfo_descriptor.py | 12 ++++++------ stem/descriptor/server_descriptor.py | 12 ++++++------ test/integ/descriptor/extrainfo_descriptor.py | 4 ++-- test/integ/descriptor/server_descriptor.py | 8 ++++---- test/unit/descriptor/server_descriptor.py | 6 +++--- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 2ea8829..1def09c 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -145,12 +145,12 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): geoip_db_digest (str) - sha1 of geoIP database file signature (str) - signature for this extrainfo descriptor (*) - read_history (str) - read-history line, always unset + read_history_line (str) - bytes read for relayed traffic read_history_end (datetime.datetime) - end of the sampling interval read_history_interval (int) - seconds per interval read_history_values (list) - bytes read during each interval (*) - write_history (str) - write-history line, always unset + write_history_line (str) - bytes written for relayed traffic write_history_end (datetime.datetime) - end of the sampling interval write_history_interval (int) - seconds per interval write_history_values (list) - bytes written during each interval (*) @@ -185,12 +185,12 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.geoip_db_digest = None self.signature = None - self.read_history = None + self.read_history_line = None self.read_history_end = None self.read_history_interval = None self.read_history_values = [] - self.write_history = None + self.write_history_line = None self.write_history_end = None self.write_history_interval = None self.write_history_values = [] @@ -276,12 +276,12 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) if keyword == "read-history": - self.read_history = value + self.read_history_line = value self.read_history_end = timestamp self.read_history_interval = interval self.read_history_values = history_values else: - self.write_history = value + self.write_history_line = value self.write_history_end = timestamp self.write_history_interval = interval self.write_history_values = history_values diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 2a9d72f..80f1538 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -167,12 +167,12 @@ class ServerDescriptor(stem.descriptor.Descriptor): observed_bandwidth (int) - estimated capacity of the relay based on usage in bytes/s (*) eventdns (bool) - (deprecated) always unset (*) - read_history (str) - (deprecated) read-history line, always unset + read_history_line (str) - (deprecated) bytes read for relayed traffic read_history_end (datetime.datetime) - (deprecated) end of the sampling interval read_history_interval (int) - (deprecated) seconds per interval read_history_values (list) - (deprecated) bytes read during each interval (*) - write_history (str) - (deprecated) write-history line, always unset + write_history_line (str) - (deprecated) bytes written for relayed traffic write_history_end (datetime.datetime) - (deprecated) end of the sampling interval write_history_interval (int) - (deprecated) seconds per interval write_history_values (list) - (deprecated) bytes written during each interval (*) @@ -227,12 +227,12 @@ class ServerDescriptor(stem.descriptor.Descriptor): self.observed_bandwidth = None self.eventdns = True - self.read_history = None + self.read_history_line = None self.read_history_end = None self.read_history_interval = None self.read_history_values = [] - self.write_history = None + self.write_history_line = None self.write_history_end = None self.write_history_interval = None self.write_history_values = [] @@ -481,12 +481,12 @@ class ServerDescriptor(stem.descriptor.Descriptor): raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) if keyword == "read-history": - self.read_history = value + self.read_history_line = value self.read_history_end = timestamp self.read_history_interval = interval self.read_history_values = history_values else: - self.write_history = value + self.write_history_line = value self.write_history_end = timestamp self.write_history_interval = interval self.write_history_values = history_values diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index d62a164..51af571 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -47,13 +47,13 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw # the initial contents for the line and parsed values. read_start = "2012-05-05 17:02:45 (900 s) 3309568,9216,41984" - self.assertTrue(desc.read_history.startswith(read_start)) + self.assertTrue(desc.read_history_line.startswith(read_start)) read_values_start = [3309568, 9216, 41984, 27648, 123904] self.assertEquals(read_values_start, desc.read_history_values[:5]) write_start = "2012-05-05 17:02:45 (900 s) 1082368,19456,50176,272384" - self.assertTrue(desc.write_history.startswith(write_start)) + self.assertTrue(desc.write_history_line.startswith(write_start)) write_values_start = [1082368, 19456, 50176, 272384, 485376] self.assertEquals(write_values_start, desc.write_history_values[:5]) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 57f6879..945d147 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -134,13 +134,13 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= # the initial contents for the line and parsed values. read_start = "2005-12-16 18:00:48 (900 s) 20774,489973,510022" - self.assertTrue(desc.read_history.startswith(read_start)) + self.assertTrue(desc.read_history_line.startswith(read_start)) read_values_start = [20774, 489973, 510022, 511163, 20949] self.assertEquals(read_values_start, desc.read_history_values[:5]) write_start = "2005-12-16 18:00:48 (900 s) 81,8848,8927,8927" - self.assertTrue(desc.write_history.startswith(write_start)) + self.assertTrue(desc.write_history_line.startswith(write_start)) write_values_start = [81, 8848, 8927, 8927, 83, 8848, 8931, 8929, 81, 8846] self.assertEquals(write_values_start, desc.write_history_values[:10]) @@ -164,8 +164,8 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= with open(descriptor_path) as descriptor_file: for desc in stem.descriptor.server_descriptor.parse_file(descriptor_file): # the following attributes should be deprecated, and not appear in the wild - self.assertEquals(None, desc.read_history) - self.assertEquals(None, desc.write_history) + self.assertEquals(None, desc.read_history_line) + self.assertEquals(None, desc.write_history_line) self.assertEquals(True, desc.eventdns) self.assertEquals(0, desc.socks_port) diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index e3b2ec6..63fd89d 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -235,10 +235,10 @@ class TestServerDescriptor(unittest.TestCase): desc = RelayDescriptor(desc_text) if field == "read-history": - attr = (desc.read_history, desc.read_history_end, + attr = (desc.read_history_line, desc.read_history_end, desc.read_history_interval, desc.read_history_values) else: - attr = (desc.write_history, desc.write_history_end, + attr = (desc.write_history_line, desc.write_history_end, desc.write_history_interval, desc.write_history_values) expected_end = datetime.datetime(2005, 12, 16, 18, 0, 48) @@ -257,7 +257,7 @@ class TestServerDescriptor(unittest.TestCase): value = "2005-12-17 01:23:11 (900 s) " desc_text = _make_descriptor({"opt read-history": value}) desc = RelayDescriptor(desc_text) - self.assertEquals(value, desc.read_history) + self.assertEquals(value, desc.read_history_line) self.assertEquals(datetime.datetime(2005, 12, 17, 1, 23, 11), desc.read_history_end) self.assertEquals(900, desc.read_history_interval) self.assertEquals([], desc.read_history_values)

1 0

[stem/master] Parsing dirreq-stats-end extrainfo lines
by atagar＠torproject.org 14 May '12

14 May '12

commit 6f40a5e87228ea8f0af76d2aeba859025ce468b2 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 12:32:09 2012 -0700 Parsing dirreq-stats-end extrainfo lines Parsing and test. This is the same format as bridge-stats-end so combining those tests. --- stem/descriptor/extrainfo_descriptor.py | 32 ++++++++++++++---- test/unit/descriptor/extrainfo_descriptor.py | 47 +++++++++++++++----------- 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 380d022..a8f7961 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -154,6 +154,12 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): write_history_end (datetime.datetime) - end of the sampling interval write_history_interval (int) - seconds per interval write_history_values (list) - bytes written during each interval (*) + + Directory Mirror Attributes: + dirreq_stats_end (datetime.datetime) - end of the period where directory + mirroring stats were gathered + dirreq_stats_interval (int) - length in seconds of the interval where + directory stats were gathered dir_read_history_line (str) - bytes read for directory mirroring dir_read_history_end (datetime.datetime) - end of the sampling interval @@ -164,18 +170,18 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_write_history_end (datetime.datetime) - end of the sampling interval dir_write_history_interval (int) - seconds per interval dir_write_history_values (list) - bytes read during each interval (*) - - Bridge-only Attributes: + + Bridge Attributes: bridge_stats_end (datetime.datetime) - end of the period when geoip statistics were gathered - bridge_stats_end_interval (int) - length in seconds of th interval where + bridge_stats_interval (int) - length in seconds of the interval where stats were gathered bridge_ips (dict) - mapping of country codes to a rounded number of unique ips from that region geoip_start_time (datetime.datetime) - replaced by bridge_stats_end geoip_client_origins (dict) - replaced by bridge_ips - - (*) required fields, others are left as None if undefined + + (*) required fields, others are left as None if undefined """ def __init__(self, raw_contents, validate = True, annotations = None): @@ -215,6 +221,9 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.write_history_interval = None self.write_history_values = [] + self.dirreq_stats_end = None + self.dirreq_stats_interval = None + self.dir_read_history_line = None self.dir_read_history_end = None self.dir_read_history_interval = None @@ -226,7 +235,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_write_history_values = [] self.bridge_stats_end = None - self.bridge_stats_end_interval = None + self.bridge_stats_interval = None self.bridge_ips = None self.geoip_start_time = None self.geoip_client_origins = None @@ -320,7 +329,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): try: timestamp, interval, _ = _parse_timestamp_and_interval(keyword, value) self.bridge_stats_end = timestamp - self.bridge_stats_end_interval = interval + self.bridge_stats_interval = interval except ValueError, exc: if validate: raise exc elif keyword in ("geoip-client-origins", "bridge-ips"): @@ -345,6 +354,15 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.geoip_client_origins = locale_usage elif keyword == "bridge-ips": self.bridge_ips = locale_usage + elif keyword == "dirreq-stats-end": + # "dirreq-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) + + try: + timestamp, interval, _ = _parse_timestamp_and_interval(keyword, value) + self.dirreq_stats_end = timestamp + self.dirreq_stats_interval = interval + except ValueError, exc: + if validate: raise exc elif keyword in ("read-history", "write-history", "dirreq-read-history", "dirreq-write-history"): try: timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value) diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 32732e3..b4c5fbe 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -151,29 +151,36 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc_text = _make_descriptor({"geoip-start-time": entry}) desc = self._expect_invalid_attr(desc_text, "geoip_start_time") - def test_bridge_stats_end(self): + def test_stats_end(self): """ - Parses the bridge-stats-end line with valid and invalid data. + Parses the bridge-stats-end and dirreq-stats-end lines with valid and + invalid data. """ - desc_text = _make_descriptor({"bridge-stats-end": "2012-05-03 12:07:50 (500 s)"}) - desc = ExtraInfoDescriptor(desc_text) - self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), desc.bridge_stats_end) - self.assertEquals(500, desc.bridge_stats_end_interval) - - test_entry = ( - "", - "2012-05-03 12:07:60 (500 s)", - "2012-05-03 12:07:50 (500s)", - "2012-05-03 12:07:50 (500 s", - "2012-05-03 12:07:50 (500 )", - "2012-05-03 ", - "2012-05-03", - ) - - for entry in test_entry: - desc_text = _make_descriptor({"bridge-stats-end": entry}) - desc = self._expect_invalid_attr(desc_text, "bridge_stats_end") + for keyword in ('bridge-stats-end', 'dirreq-stats-end'): + end_attr = keyword.replace('-', '_') + interval_attr = end_attr[:-4] + "_interval" + + desc_text = _make_descriptor({keyword: "2012-05-03 12:07:50 (500 s)"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), getattr(desc, end_attr)) + self.assertEquals(500, getattr(desc, interval_attr)) + + test_entry = ( + "", + "2012-05-03 12:07:60 (500 s)", + "2012-05-03 12:07:50 (500s)", + "2012-05-03 12:07:50 (500 s", + "2012-05-03 12:07:50 (500 )", + "2012-05-03 ", + "2012-05-03", + ) + + for entry in test_entry: + desc_text = _make_descriptor({keyword: entry}) + desc = self._expect_invalid_attr(desc_text) + self.assertEquals(None, getattr(desc, end_attr)) + self.assertEquals(None, getattr(desc, interval_attr)) def test_bridge_ips(self): """

1 0