May 2012 - tor-commits - lists.torproject.org

[stem/master] Accepting additional args on extra-info line
by atagar＠torproject.org 14 May '12

14 May '12

commit d3f7c00fa6cf2573f30551589338e4d42d61465a Author: Damian Johnson <atagar(a)torproject.org> Date: Thu May 10 08:00:36 2012 -0700 Accepting additional args on extra-info line Forgot that the spec says that extra arguments should be accepted and ignored. --- stem/descriptor/extrainfo_descriptor.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index aa02bc9..b0d7864 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -263,7 +263,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): # "extra-info" Nickname Fingerprint extra_info_comp = value.split() - if len(extra_info_comp) != 2: + if len(extra_info_comp) < 2: if not validate: continue raise ValueError("Extra-info line must have two values: %s" % line)

1 0

[stem/master] Parsing geoip-db-digest lines in extrainfo descriptors
by atagar＠torproject.org 14 May '12

14 May '12

commit 15a25f0b05eacfff21309715a4838d4ccd6f11f9 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu May 10 08:31:26 2012 -0700 Parsing geoip-db-digest lines in extrainfo descriptors --- stem/descriptor/extrainfo_descriptor.py | 7 ++++++ test/unit/descriptor/extrainfo_descriptor.py | 27 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index b0d7864..d2ee780 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -283,6 +283,13 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): except ValueError: if validate: raise ValueError("Published line's time wasn't parseable: %s" % line) + elif keyword == "geoip-db-digest": + # "geoip-db-digest" Digest + + if validate and not re.match("^[0-9a-fA-F]{40}$", value): + raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) + + self.geoip_db_digest = value elif keyword in ("read-history", "write-history", "dirreq-read-history", "dirreq-write-history"): try: timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value) diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 15ffe68..5a4a450 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -108,6 +108,33 @@ class TestExtraInfoDescriptor(unittest.TestCase): self.assertEquals(None, desc.nickname) self.assertEquals(None, desc.fingerprint) + def test_geoip_db_digest(self): + """ + Parses a geoip-db-digest line with valid data. + """ + + geoip_db_digest = "916A3CA8B7DF61473D5AE5B21711F35F301CE9E8" + desc_text = _make_descriptor({"geoip-db-digest": geoip_db_digest}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(geoip_db_digest, desc.geoip_db_digest) + + def test_geoip_db_digest_invalid(self): + """ + Parses the geoip-db-digest line with a variety of bad input. + """ + + test_entry = ( + "", + "916A3CA8B7DF61473D5AE5B21711F35F301CE9E", + "916A3CA8B7DF61473D5AE5B21711F35F301CE9E88", + "916A3CA8B7DF61473D5AE5B21711F35F301CE9EG", + "916A3CA8B7DF61473D5AE5B21711F35F301CE9E-", + ) + + for entry in test_entry: + desc_text = _make_descriptor({"geoip-db-digest": entry}) + desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) + def _expect_invalid_attr(self, desc_text, attr = None, expected_value = None): """ Asserts that construction will fail due to desc_text having a malformed

1 0

[stem/master] Parsing extrainfo dirreq-read/write-history lines
by atagar＠torproject.org 14 May '12

14 May '12

commit 0737bada04f71434e9169bc964257dffce8f90c3 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed May 9 10:01:32 2012 -0700 Parsing extrainfo dirreq-read/write-history lines Handling the dirreq-read-history and dirreq-write-history. These are almost identical to the read-history and write-history lines so tieing it into that parser. This is the last field I need to parse the metrics descriptor I'm using for an example in the integ tests. --- stem/descriptor/extrainfo_descriptor.py | 38 +++++++++++++++++++++++- test/integ/descriptor/extrainfo_descriptor.py | 23 ++++++++++++-- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 1def09c..aa02bc9 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -155,6 +155,16 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): write_history_interval (int) - seconds per interval write_history_values (list) - bytes written during each interval (*) + dir_read_history_line (str) - bytes read for directory mirroring + dir_read_history_end (datetime.datetime) - end of the sampling interval + dir_read_history_interval (int) - seconds per interval + dir_read_history_values (list) - bytes read during each interval (*) + + dir_write_history_line (str) - bytes written for directory mirroring + dir_write_history_end (datetime.datetime) - end of the sampling interval + dir_write_history_interval (int) - seconds per interval + dir_write_history_values (list) - bytes read during each interval (*) + (*) required fields, others are left as None if undefined """ @@ -195,6 +205,16 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.write_history_interval = None self.write_history_values = [] + self.dir_read_history_line = None + self.dir_read_history_end = None + self.dir_read_history_interval = None + self.dir_read_history_values = [] + + self.dir_write_history_line = None + self.dir_write_history_end = None + self.dir_write_history_interval = None + self.dir_write_history_values = [] + self._unrecognized_lines = [] entries, first_keyword, last_keyword, _ = \ @@ -263,7 +283,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): except ValueError: if validate: raise ValueError("Published line's time wasn't parseable: %s" % line) - elif keyword in ("read-history", "write-history"): + elif keyword in ("read-history", "write-history", "dirreq-read-history", "dirreq-write-history"): try: timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value) @@ -280,11 +300,25 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.read_history_end = timestamp self.read_history_interval = interval self.read_history_values = history_values - else: + elif keyword == "write-history": self.write_history_line = value self.write_history_end = timestamp self.write_history_interval = interval self.write_history_values = history_values + elif keyword == "dirreq-read-history": + self.dir_read_history_line = value + self.dir_read_history_end = timestamp + self.dir_read_history_interval = interval + self.dir_read_history_values = history_values + elif keyword == "dirreq-write-history": + self.dir_write_history_line = value + self.dir_write_history_end = timestamp + self.dir_write_history_interval = interval + self.dir_write_history_values = history_values + else: + # not gonna happen unless we change the main loop's conditional + # without fixing this one + raise ValueError("BUG: unrecognized keyword '%s'" % keyword) except ValueError, exc: if not validate: continue else: raise exc diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 51af571..7a041d0 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -38,12 +38,15 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw self.assertEquals(900, desc.read_history_interval) self.assertEquals(datetime.datetime(2012, 5, 5, 17, 2, 45), desc.write_history_end) self.assertEquals(900, desc.write_history_interval) + self.assertEquals(datetime.datetime(2012, 5, 5, 17, 2, 45), desc.dir_read_history_end) + self.assertEquals(900, desc.dir_read_history_interval) + self.assertEquals(datetime.datetime(2012, 5, 5, 17, 2, 45), desc.dir_write_history_end) + self.assertEquals(900, desc.dir_write_history_interval) self.assertEquals(expected_signature, desc.signature) + self.assertEquals([], desc.get_unrecognized_lines()) - # TODO: still missing dirreq-read-history and dirreq-write-history - #self.assertEquals([], desc.get_unrecognized_lines()) - - # The read-history and write-history lines are pretty long so just checking + # The read-history, write-history, dirreq-read-history, and + # dirreq-write-history lines are pretty long so just checking # the initial contents for the line and parsed values. read_start = "2012-05-05 17:02:45 (900 s) 3309568,9216,41984" @@ -57,6 +60,18 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw write_values_start = [1082368, 19456, 50176, 272384, 485376] self.assertEquals(write_values_start, desc.write_history_values[:5]) + + dir_read_start = "2012-05-05 17:02:45 (900 s) 0,0,0,0,33792,27648,48128" + self.assertTrue(desc.dir_read_history_line.startswith(dir_read_start)) + + dir_read_values_start = [0, 0, 0, 0, 33792, 27648, 48128] + self.assertEquals(dir_read_values_start, desc.dir_read_history_values[:7]) + + dir_write_start = "2012-05-05 17:02:45 (900 s) 0,0,0,227328,349184" + self.assertTrue(desc.dir_write_history_line.startswith(dir_write_start)) + + dir_write_values_start = [0, 0, 0, 227328, 349184, 382976, 738304] + self.assertEquals(dir_write_values_start, desc.dir_write_history_values[:7]) def test_cached_descriptor(self): """

1 0

[stem/master] Parsing bridge specific extra-info params
by atagar＠torproject.org 14 May '12

14 May '12

commit c33c4a0f0e6b98898622d0929e531d12abdce72a Author: Damian Johnson <atagar(a)torproject.org> Date: Thu May 10 09:29:04 2012 -0700 Parsing bridge specific extra-info params Parsing, validation, and tests for bridge specific lines... - geoip-start-time - geoip-client-origins - bridge-stats-end - bridge-stats-ips I haven't yet seen an actual bridge descriptor, and I'd like to add one as an integ test but that can come later. --- stem/descriptor/extrainfo_descriptor.py | 58 ++++++++++++++++++- stem/descriptor/server_descriptor.py | 3 +- test/unit/descriptor/extrainfo_descriptor.py | 81 ++++++++++++++++++++++++-- 3 files changed, 132 insertions(+), 10 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index d2ee780..380d022 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -165,6 +165,16 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_write_history_interval (int) - seconds per interval dir_write_history_values (list) - bytes read during each interval (*) + Bridge-only Attributes: + bridge_stats_end (datetime.datetime) - end of the period when geoip + statistics were gathered + bridge_stats_end_interval (int) - length in seconds of th interval where + stats were gathered + bridge_ips (dict) - mapping of country codes to a rounded number of unique + ips from that region + geoip_start_time (datetime.datetime) - replaced by bridge_stats_end + geoip_client_origins (dict) - replaced by bridge_ips + (*) required fields, others are left as None if undefined """ @@ -215,6 +225,12 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_write_history_interval = None self.dir_write_history_values = [] + self.bridge_stats_end = None + self.bridge_stats_end_interval = None + self.bridge_ips = None + self.geoip_start_time = None + self.geoip_client_origins = None + self._unrecognized_lines = [] entries, first_keyword, last_keyword, _ = \ @@ -290,6 +306,45 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value + elif keyword == "geoip-start-time": + # "geoip-start-time" YYYY-MM-DD HH:MM:SS + + try: + self.geoip_start_time = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + except ValueError: + if validate: + raise ValueError("Geoip start time line's time wasn't parseable: %s" % line) + elif keyword == "bridge-stats-end": + # "bridge-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) + + try: + timestamp, interval, _ = _parse_timestamp_and_interval(keyword, value) + self.bridge_stats_end = timestamp + self.bridge_stats_end_interval = interval + except ValueError, exc: + if validate: raise exc + elif keyword in ("geoip-client-origins", "bridge-ips"): + # "geoip-client-origins" CC=N,CC=N,... + + locale_usage = {} + error_msg = "Entries in %s line should only be CC=N entries: %s" % (keyword, line) + + for entry in value.split(","): + if not "=" in entry: + if validate: raise ValueError(error_msg) + else: continue + + locale, count = entry.split("=", 1) + + if re.match("^[a-zA-Z]{2}$", locale) and count.isdigit(): + locale_usage[locale] = int(count) + elif validate: + raise ValueError(error_msg) + + if keyword == "geoip-client-origins": + self.geoip_client_origins = locale_usage + elif keyword == "bridge-ips": + self.bridge_ips = locale_usage elif keyword in ("read-history", "write-history", "dirreq-read-history", "dirreq-write-history"): try: timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value) @@ -327,8 +382,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): # without fixing this one raise ValueError("BUG: unrecognized keyword '%s'" % keyword) except ValueError, exc: - if not validate: continue - else: raise exc + if validate: raise exc elif keyword == "router-signature": if validate and not block_contents: raise ValueError("Router signature line must be followed by a signature block: %s" % line) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 80f1538..2963a53 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -491,8 +491,7 @@ class ServerDescriptor(stem.descriptor.Descriptor): self.write_history_interval = interval self.write_history_values = history_values except ValueError, exc: - if not validate: continue - else: raise exc + if validate: raise exc else: self._unrecognized_lines.append(line) diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 5a4a450..32732e3 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -2,6 +2,7 @@ Unit tests for stem.descriptor.extrainfo_descriptor. """ +import datetime import unittest from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor @@ -110,18 +111,13 @@ class TestExtraInfoDescriptor(unittest.TestCase): def test_geoip_db_digest(self): """ - Parses a geoip-db-digest line with valid data. + Parses the geoip-db-digest line with valid and invalid data. """ geoip_db_digest = "916A3CA8B7DF61473D5AE5B21711F35F301CE9E8" desc_text = _make_descriptor({"geoip-db-digest": geoip_db_digest}) desc = ExtraInfoDescriptor(desc_text) self.assertEquals(geoip_db_digest, desc.geoip_db_digest) - - def test_geoip_db_digest_invalid(self): - """ - Parses the geoip-db-digest line with a variety of bad input. - """ test_entry = ( "", @@ -135,6 +131,79 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc_text = _make_descriptor({"geoip-db-digest": entry}) desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) + def test_geoip_start_time(self): + """ + Parses the geoip-start-time line with valid and invalid data. + """ + + desc_text = _make_descriptor({"geoip-start-time": "2012-05-03 12:07:50"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), desc.geoip_start_time) + + test_entry = ( + "", + "2012-05-03 12:07:60", + "2012-05-03 ", + "2012-05-03", + ) + + for entry in test_entry: + desc_text = _make_descriptor({"geoip-start-time": entry}) + desc = self._expect_invalid_attr(desc_text, "geoip_start_time") + + def test_bridge_stats_end(self): + """ + Parses the bridge-stats-end line with valid and invalid data. + """ + + desc_text = _make_descriptor({"bridge-stats-end": "2012-05-03 12:07:50 (500 s)"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), desc.bridge_stats_end) + self.assertEquals(500, desc.bridge_stats_end_interval) + + test_entry = ( + "", + "2012-05-03 12:07:60 (500 s)", + "2012-05-03 12:07:50 (500s)", + "2012-05-03 12:07:50 (500 s", + "2012-05-03 12:07:50 (500 )", + "2012-05-03 ", + "2012-05-03", + ) + + for entry in test_entry: + desc_text = _make_descriptor({"bridge-stats-end": entry}) + desc = self._expect_invalid_attr(desc_text, "bridge_stats_end") + + def test_bridge_ips(self): + """ + Parses both the bridge-ips and geoip-client-origins lines with valid and + invalid data. + """ + + # Testing both attributes since they contain the exact same data, + # geoip-client-origins was simply replaced by bridge-ips while adding an + # interval value for the period. + + for keyword in ('bridge-ips', 'geoip-client-origins'): + attr = keyword.replace('-', '_') + + desc_text = _make_descriptor({keyword: "uk=5,de=3,jp=2"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals({'uk': 5, 'de': 3, 'jp': 2}, getattr(desc, attr)) + + test_entry = ( + "", + "uk=-4", + "uki=4", + "uk:4", + "uk=4.de=3", + ) + + for entry in test_entry: + desc_text = _make_descriptor({keyword: entry}) + desc = self._expect_invalid_attr(desc_text, attr, {}) + def _expect_invalid_attr(self, desc_text, attr = None, expected_value = None): """ Asserts that construction will fail due to desc_text having a malformed

1 0

[stem/master] Revising descriptor pydocs
by atagar＠torproject.org 14 May '12

14 May '12

commit bff56e18ecde250eb9ad2eba7db5e29d1817bd8f Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 13:57:53 2012 -0700 Revising descriptor pydocs Revisited the ordering, defaults, and descriptions of descriptor attributes to be shorter and more user friendly. --- stem/descriptor/extrainfo_descriptor.py | 74 ++++++++++----------- stem/descriptor/server_descriptor.py | 96 +++++++++++++++------------- test/integ/descriptor/server_descriptor.py | 2 +- 3 files changed, 87 insertions(+), 85 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index a9bbfec..93b438c 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -139,47 +139,43 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): Extra-info descriptor document. Attributes: - nickname (str) - relay's nickname (*) - fingerprint (str) - fourty hex digits that make up the relay's fingerprint (*) - published (datetime.datetime) - time in GMT when the descriptor was generated (*) - geoip_db_digest (str) - sha1 of geoIP database file - signature (str) - signature for this extrainfo descriptor (*) + nickname (str) - relay's nickname (*) + fingerprint (str) - identity key fingerprint (*) + published (datetime) - time in GMT when this descriptor was made (*) + geoip_db_digest (str) - sha1 of geoIP database file + signature (str) - signature for this extrainfo descriptor (*) Bytes read/written for relayed traffic - read_history_end (datetime.datetime) - end of the sampling interval + read_history_end (datetime) - end of the sampling interval read_history_interval (int) - seconds per interval - read_history_values (list) - bytes read during each interval (*) + read_history_values (list) - bytes read during each interval - write_history_end (datetime.datetime) - end of the sampling interval + write_history_end (datetime) - end of the sampling interval write_history_interval (int) - seconds per interval - write_history_values (list) - bytes written during each interval (*) - - Directory Mirror Attributes: - dirreq_stats_end (datetime.datetime) - end of the period where directory - mirroring stats were gathered - dirreq_stats_interval (int) - length in seconds of the interval where - directory stats were gathered + write_history_values (list) - bytes written during each interval - Bytes read/written for directory mirroring - dir_read_history_end (datetime.datetime) - end of the sampling interval - dir_read_history_interval (int) - seconds per interval - dir_read_history_values (list) - bytes read during each interval (*) + Directory Mirror Attributes: + dirreq_stats_end (datetime) - end of the period when stats were gathered + dirreq_stats_interval (int) - length in seconds of the interval - dir_write_history_end (datetime.datetime) - end of the sampling interval - dir_write_history_interval (int) - seconds per interval - dir_write_history_values (list) - bytes read during each interval (*) - - Bridge Attributes: - bridge_stats_end (datetime.datetime) - end of the period when geoip - statistics were gathered - bridge_stats_interval (int) - length in seconds of the interval where - stats were gathered - bridge_ips (dict) - mapping of country codes to a rounded number of unique - ips from that region - geoip_start_time (datetime.datetime) - replaced by bridge_stats_end - geoip_client_origins (dict) - replaced by bridge_ips + Bytes read/written for directory mirroring + dir_read_history_end (datetime) - end of the sampling interval + dir_read_history_interval (int) - seconds per interval + dir_read_history_values (list) - bytes read during each interval + + dir_write_history_end (datetime) - end of the sampling interval + dir_write_history_interval (int) - seconds per interval + dir_write_history_values (list) - bytes read during each interval + + Bridge Attributes: + bridge_stats_end (datetime) - end of the period when stats were gathered + bridge_stats_interval (int) - length in seconds of the interval + bridge_ips (dict) - mapping of locales to rounded count of unique user ips + geoip_start_time (datetime) - (deprecated) replaced by bridge_stats_end + geoip_client_origins (dict) - (deprecated) replaced by bridge_ips - (*) required fields, others are left as None if undefined + (*) attribute is either required when we're parsed with validation or has a + default value, others are left as None if undefined """ def __init__(self, raw_contents, validate = True, annotations = None): @@ -211,22 +207,22 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.read_history_end = None self.read_history_interval = None - self.read_history_values = [] + self.read_history_values = None self.write_history_end = None self.write_history_interval = None - self.write_history_values = [] + self.write_history_values = None self.dirreq_stats_end = None self.dirreq_stats_interval = None self.dir_read_history_end = None self.dir_read_history_interval = None - self.dir_read_history_values = [] + self.dir_read_history_values = None self.dir_write_history_end = None self.dir_write_history_interval = None - self.dir_write_history_values = [] + self.dir_write_history_values = None self.bridge_stats_end = None self.bridge_stats_interval = None diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 7e16c77..d347b15 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -141,42 +141,47 @@ class ServerDescriptor(stem.descriptor.Descriptor): Common parent for server descriptors. Attributes: - nickname (str) - relay's nickname (*) - fingerprint (str) - fourty hex digits that make up the relay's fingerprint - address (str) - IPv4 address of the relay (*) - or_port (int) - port used for relaying (*) - socks_port (int) - (deprecated) always zero (*) - dir_port (int) - port used for descriptor mirroring (*) - platform (str) - operating system and tor version + nickname (str) - relay's nickname (*) + fingerprint (str) - identity key fingerprint + published (datetime) - time in GMT when this descriptor was made (*) + + address (str) - IPv4 address of the relay (*) + or_port (int) - port used for relaying (*) + socks_port (int) - (deprecated, always zero) port used as client (*) + dir_port (int) - port used for descriptor mirroring (*) + + platform (str) - line with operating system and tor version tor_version (stem.version.Version) - version of tor - operating_system (str) - relay's operating system - uptime (int) - relay's uptime when published in seconds - published (datetime.datetime) - time in GMT when the descriptor was generated (*) - contact (str) - relay's contact information - link_protocols (list) - link protocols supported by the relay - circuit_protocols (list) - circuit protocols supported by the relay - hibernating (bool) - flag to indicate if the relay was hibernating when published (*) - allow_single_hop_exits (bool) - flag to indicate if single hop exiting is allowed from it (*) - extra_info_cache (bool) - flag to indicate if it's a mirror for extra-info documents (*) - extra_info_digest (str) - hex encoded digest of our extra-info document - hidden_service_dir (list) - hidden service descriptor versions that it stores - exit_policy (stem.exit_policy.ExitPolicy) - relay's stated exit policy - family (list) - nicknames or fingerprints of relays it has a declared family with (*) - average_bandwidth (int) - rate of traffic relay is willing to relay in bytes/s (*) - burst_bandwidth (int) - rate of traffic relay is willing to burst to in bytes/s (*) - observed_bandwidth (int) - estimated capacity of the relay based on usage in bytes/s (*) - eventdns (bool) - (deprecated) always unset (*) + operating_system (str) - operating system + uptime (int) - uptime when published in seconds + contact (str) - contact information + exit_policy (stem.exit_policy.ExitPolicy) - stated exit policy (*) + family (list) - nicknames or fingerprints of declared family (*) + + average_bandwidth (int) - averate rate it's willing to relay in bytes/s (*) + burst_bandwidth (int) - burst rate it's willing to relay in bytes/s (*) + observed_bandwidth (int) - estimated capacity based on usage in bytes/s (*) + + link_protocols (list) - link protocols supported by the relay + circuit_protocols (list) - circuit protocols supported by the relay + hibernating (bool) - hibernating when published (*) + allow_single_hop_exits (bool) - flag if single hop exiting is allowed (*) + extra_info_cache (bool) - flag if a mirror for extra-info documents (*) + extra_info_digest (str) - hex encoded digest of our extra-info document + hidden_service_dir (list) - hidden service descriptor versions it stores + eventdns (bool) - (deprecated, always unset) flag for evdns backend Deprecated, moved to extra-info descriptor... - read_history_end (datetime.datetime) - end of the sampling interval + read_history_end (datetime) - end of the sampling interval read_history_interval (int) - seconds per interval - read_history_values (list) - bytes read during each interval (*) + read_history_values (list) - bytes read during each interval - write_history_end (datetime.datetime) - end of the sampling interval + write_history_end (datetime) - end of the sampling interval write_history_interval (int) - seconds per interval - write_history_values (list) - bytes written during each interval (*) - - (*) required fields, others are left as None if undefined + write_history_values (list) - bytes written during each interval + + (*) attribute is either required when we're parsed with validation or has a + default value, others are left as None if undefined """ def __init__(self, raw_contents, validate = True, annotations = None): @@ -203,16 +208,25 @@ class ServerDescriptor(stem.descriptor.Descriptor): self.nickname = None self.fingerprint = None + self.published = None + self.address = None self.or_port = None self.socks_port = None self.dir_port = None + self.platform = None self.tor_version = None self.operating_system = None self.uptime = None - self.published = None self.contact = None + self.exit_policy = [] # should be an ExitPolicy instance when we have the class... + self.family = [] + + self.average_bandwidth = None + self.burst_bandwidth = None + self.observed_bandwidth = None + self.link_protocols = None self.circuit_protocols = None self.hibernating = False @@ -220,30 +234,21 @@ class ServerDescriptor(stem.descriptor.Descriptor): self.extra_info_cache = False self.extra_info_digest = None self.hidden_service_dir = None - self.family = [] - self.average_bandwidth = None - self.burst_bandwidth = None - self.observed_bandwidth = None - self.eventdns = True + self.eventdns = None self.read_history_end = None self.read_history_interval = None - self.read_history_values = [] + self.read_history_values = None self.write_history_end = None self.write_history_interval = None - self.write_history_values = [] + self.write_history_values = None self._unrecognized_lines = [] self._annotation_lines = annotations if annotations else [] self._annotation_dict = None # cached breakdown of key/value mappings - # TODO: Until we have a proper ExitPolicy class this is just a list of the - # exit policy strings... - - self.exit_policy = [] - # A descriptor contains a series of 'keyword lines' which are simply a # keyword followed by an optional value. Lines can also be followed by a # signature block. @@ -551,8 +556,9 @@ class RelayDescriptor(ServerDescriptor): onion_key (str) - key used to encrypt EXTEND cells (*) signing_key (str) - relay's long-term identity key (*) signature (str) - signature for this descriptor (*) - - (*) required fields, others are left as None if undefined + + (*) attribute is either required when we're parsed with validation or has a + default value, others are left as None if undefined """ def __init__(self, raw_contents, validate = True, annotations = None): diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index d7b5c14..3c07364 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -160,7 +160,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= # the following attributes should be deprecated, and not appear in the wild self.assertEquals(None, desc.read_history_end) self.assertEquals(None, desc.write_history_end) - self.assertEquals(True, desc.eventdns) + self.assertEquals(None, desc.eventdns) self.assertEquals(0, desc.socks_port) unrecognized_lines = desc.get_unrecognized_lines()

1 0

[stem/master] Parsing 'dirreq-v2-reqs' and 'dirreq-v3-reqs' lines
by atagar＠torproject.org 14 May '12

14 May '12

commit b82dca6e2e2543cfa19e9d15be1d8ba89227407d Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 15:15:55 2012 -0700 Parsing 'dirreq-v2-reqs' and 'dirreq-v3-reqs' lines Adding support for a couple more extrainfo fields. --- stem/descriptor/extrainfo_descriptor.py | 10 +++++++++- test/unit/descriptor/extrainfo_descriptor.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index ce39ee3..61e00dd 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -159,6 +159,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_stats_interval (int) - length in seconds of the interval dir_v2_ips (dict) - mapping of locales to rounded count of requester ips dir_v3_ips (dict) - mapping of locales to rounded count of requester ips + dir_v2_reqs (dict) - mapping of locales to rounded count of requests + dir_v3_reqs (dict) - mapping of locales to rounded count of requests Bytes read/written for directory mirroring dir_read_history_end (datetime) - end of the sampling interval @@ -219,6 +221,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_stats_interval = None self.dir_v2_ips = None self.dir_v3_ips = None + self.dir_v2_reqs = None + self.dir_v3_reqs = None self.dir_read_history_end = None self.dir_read_history_interval = None @@ -358,7 +362,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_write_history_values = history_values except ValueError, exc: if validate: raise exc - elif keyword in ("dirreq-v2-ips", "dirreq-v3-ips", "geoip-client-origins", "bridge-ips"): + elif keyword in ("dirreq-v2-ips", "dirreq-v3-ips", "dirreq-v2-reqs", "dirreq-v3-reqs", "geoip-client-origins", "bridge-ips"): # "<keyword>" CC=N,CC=N,... locale_usage = {} @@ -391,6 +395,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_v2_ips = locale_usage elif keyword == "dirreq-v3-ips": self.dir_v3_ips = locale_usage + elif keyword == "dirreq-v2-reqs": + self.dir_v2_reqs = locale_usage + elif keyword == "dirreq-v3-reqs": + self.dir_v3_reqs = locale_usage elif keyword == "geoip-client-origins": self.geoip_client_origins = locale_usage elif keyword == "bridge-ips": diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 21f49a4..9732d4d 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -235,7 +235,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): "<keyword>" CC=N,CC=N,... """ - for keyword in ('dirreq-v2-ips', 'dirreq-v3-ips', 'geoip-client-origins', 'bridge-ips'): + for keyword in ('dirreq-v2-ips', 'dirreq-v3-ips', 'dirreq-v2-reqs', 'dirreq-v3-reqs', 'geoip-client-origins', 'bridge-ips'): attr = keyword.replace('-', '_').replace('dirreq', 'dir') test_entries = (

1 0

[stem/master] Dropping measurement *_line attributes
by atagar＠torproject.org 14 May '12

14 May '12

commit 2ddffb3d961bcc01829ad58c96b8fa236a35cfff Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 12:42:25 2012 -0700 Dropping measurement *_line attributes Removing attributes like read_history_line which contained the raw value for the read-history line. These are left over from when I made no effort to parse the line, and are useless since we have their parsed values. Minor changes to our pydocs so these related items are better labeled. --- stem/descriptor/extrainfo_descriptor.py | 36 +++++++++---------------- stem/descriptor/server_descriptor.py | 21 +++++--------- test/integ/descriptor/extrainfo_descriptor.py | 12 -------- test/integ/descriptor/server_descriptor.py | 10 +----- test/unit/descriptor/server_descriptor.py | 14 +++------ 5 files changed, 28 insertions(+), 65 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index a8f7961..a9bbfec 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -145,15 +145,14 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): geoip_db_digest (str) - sha1 of geoIP database file signature (str) - signature for this extrainfo descriptor (*) - read_history_line (str) - bytes read for relayed traffic - read_history_end (datetime.datetime) - end of the sampling interval - read_history_interval (int) - seconds per interval - read_history_values (list) - bytes read during each interval (*) - - write_history_line (str) - bytes written for relayed traffic - write_history_end (datetime.datetime) - end of the sampling interval - write_history_interval (int) - seconds per interval - write_history_values (list) - bytes written during each interval (*) + Bytes read/written for relayed traffic + read_history_end (datetime.datetime) - end of the sampling interval + read_history_interval (int) - seconds per interval + read_history_values (list) - bytes read during each interval (*) + + write_history_end (datetime.datetime) - end of the sampling interval + write_history_interval (int) - seconds per interval + write_history_values (list) - bytes written during each interval (*) Directory Mirror Attributes: dirreq_stats_end (datetime.datetime) - end of the period where directory @@ -161,12 +160,11 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dirreq_stats_interval (int) - length in seconds of the interval where directory stats were gathered - dir_read_history_line (str) - bytes read for directory mirroring - dir_read_history_end (datetime.datetime) - end of the sampling interval - dir_read_history_interval (int) - seconds per interval - dir_read_history_values (list) - bytes read during each interval (*) - - dir_write_history_line (str) - bytes written for directory mirroring + Bytes read/written for directory mirroring + dir_read_history_end (datetime.datetime) - end of the sampling interval + dir_read_history_interval (int) - seconds per interval + dir_read_history_values (list) - bytes read during each interval (*) + dir_write_history_end (datetime.datetime) - end of the sampling interval dir_write_history_interval (int) - seconds per interval dir_write_history_values (list) - bytes read during each interval (*) @@ -211,12 +209,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.geoip_db_digest = None self.signature = None - self.read_history_line = None self.read_history_end = None self.read_history_interval = None self.read_history_values = [] - self.write_history_line = None self.write_history_end = None self.write_history_interval = None self.write_history_values = [] @@ -224,12 +220,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dirreq_stats_end = None self.dirreq_stats_interval = None - self.dir_read_history_line = None self.dir_read_history_end = None self.dir_read_history_interval = None self.dir_read_history_values = [] - self.dir_write_history_line = None self.dir_write_history_end = None self.dir_write_history_interval = None self.dir_write_history_values = [] @@ -376,22 +370,18 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) if keyword == "read-history": - self.read_history_line = value self.read_history_end = timestamp self.read_history_interval = interval self.read_history_values = history_values elif keyword == "write-history": - self.write_history_line = value self.write_history_end = timestamp self.write_history_interval = interval self.write_history_values = history_values elif keyword == "dirreq-read-history": - self.dir_read_history_line = value self.dir_read_history_end = timestamp self.dir_read_history_interval = interval self.dir_read_history_values = history_values elif keyword == "dirreq-write-history": - self.dir_write_history_line = value self.dir_write_history_end = timestamp self.dir_write_history_interval = interval self.dir_write_history_values = history_values diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 2963a53..7e16c77 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -167,15 +167,14 @@ class ServerDescriptor(stem.descriptor.Descriptor): observed_bandwidth (int) - estimated capacity of the relay based on usage in bytes/s (*) eventdns (bool) - (deprecated) always unset (*) - read_history_line (str) - (deprecated) bytes read for relayed traffic - read_history_end (datetime.datetime) - (deprecated) end of the sampling interval - read_history_interval (int) - (deprecated) seconds per interval - read_history_values (list) - (deprecated) bytes read during each interval (*) - - write_history_line (str) - (deprecated) bytes written for relayed traffic - write_history_end (datetime.datetime) - (deprecated) end of the sampling interval - write_history_interval (int) - (deprecated) seconds per interval - write_history_values (list) - (deprecated) bytes written during each interval (*) + Deprecated, moved to extra-info descriptor... + read_history_end (datetime.datetime) - end of the sampling interval + read_history_interval (int) - seconds per interval + read_history_values (list) - bytes read during each interval (*) + + write_history_end (datetime.datetime) - end of the sampling interval + write_history_interval (int) - seconds per interval + write_history_values (list) - bytes written during each interval (*) (*) required fields, others are left as None if undefined """ @@ -227,12 +226,10 @@ class ServerDescriptor(stem.descriptor.Descriptor): self.observed_bandwidth = None self.eventdns = True - self.read_history_line = None self.read_history_end = None self.read_history_interval = None self.read_history_values = [] - self.write_history_line = None self.write_history_end = None self.write_history_interval = None self.write_history_values = [] @@ -481,12 +478,10 @@ class ServerDescriptor(stem.descriptor.Descriptor): raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) if keyword == "read-history": - self.read_history_line = value self.read_history_end = timestamp self.read_history_interval = interval self.read_history_values = history_values else: - self.write_history_line = value self.write_history_end = timestamp self.write_history_interval = interval self.write_history_values = history_values diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 7a041d0..75b26e0 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -49,27 +49,15 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw # dirreq-write-history lines are pretty long so just checking # the initial contents for the line and parsed values. - read_start = "2012-05-05 17:02:45 (900 s) 3309568,9216,41984" - self.assertTrue(desc.read_history_line.startswith(read_start)) - read_values_start = [3309568, 9216, 41984, 27648, 123904] self.assertEquals(read_values_start, desc.read_history_values[:5]) - write_start = "2012-05-05 17:02:45 (900 s) 1082368,19456,50176,272384" - self.assertTrue(desc.write_history_line.startswith(write_start)) - write_values_start = [1082368, 19456, 50176, 272384, 485376] self.assertEquals(write_values_start, desc.write_history_values[:5]) - dir_read_start = "2012-05-05 17:02:45 (900 s) 0,0,0,0,33792,27648,48128" - self.assertTrue(desc.dir_read_history_line.startswith(dir_read_start)) - dir_read_values_start = [0, 0, 0, 0, 33792, 27648, 48128] self.assertEquals(dir_read_values_start, desc.dir_read_history_values[:7]) - dir_write_start = "2012-05-05 17:02:45 (900 s) 0,0,0,227328,349184" - self.assertTrue(desc.dir_write_history_line.startswith(dir_write_start)) - dir_write_values_start = [0, 0, 0, 227328, 349184, 382976, 738304] self.assertEquals(dir_write_values_start, desc.dir_write_history_values[:7]) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 945d147..d7b5c14 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -133,15 +133,9 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= # The read-history and write-history lines are pretty long so just checking # the initial contents for the line and parsed values. - read_start = "2005-12-16 18:00:48 (900 s) 20774,489973,510022" - self.assertTrue(desc.read_history_line.startswith(read_start)) - read_values_start = [20774, 489973, 510022, 511163, 20949] self.assertEquals(read_values_start, desc.read_history_values[:5]) - write_start = "2005-12-16 18:00:48 (900 s) 81,8848,8927,8927" - self.assertTrue(desc.write_history_line.startswith(write_start)) - write_values_start = [81, 8848, 8927, 8927, 83, 8848, 8931, 8929, 81, 8846] self.assertEquals(write_values_start, desc.write_history_values[:10]) @@ -164,8 +158,8 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= with open(descriptor_path) as descriptor_file: for desc in stem.descriptor.server_descriptor.parse_file(descriptor_file): # the following attributes should be deprecated, and not appear in the wild - self.assertEquals(None, desc.read_history_line) - self.assertEquals(None, desc.write_history_line) + self.assertEquals(None, desc.read_history_end) + self.assertEquals(None, desc.write_history_end) self.assertEquals(True, desc.eventdns) self.assertEquals(0, desc.socks_port) diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 63fd89d..e381318 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -235,19 +235,16 @@ class TestServerDescriptor(unittest.TestCase): desc = RelayDescriptor(desc_text) if field == "read-history": - attr = (desc.read_history_line, desc.read_history_end, - desc.read_history_interval, desc.read_history_values) + attr = (desc.read_history_end, desc.read_history_interval, desc.read_history_values) else: - attr = (desc.write_history_line, desc.write_history_end, - desc.write_history_interval, desc.write_history_values) + attr = (desc.write_history_end, desc.write_history_interval, desc.write_history_values) expected_end = datetime.datetime(2005, 12, 16, 18, 0, 48) expected_values = [81, 8848, 8927, 8927, 83, 8848] - self.assertEquals(value, attr[0]) - self.assertEquals(expected_end, attr[1]) - self.assertEquals(900, attr[2]) - self.assertEquals(expected_values, attr[3]) + self.assertEquals(expected_end, attr[0]) + self.assertEquals(900, attr[1]) + self.assertEquals(expected_values, attr[2]) def test_read_history_empty(self): """ @@ -257,7 +254,6 @@ class TestServerDescriptor(unittest.TestCase): value = "2005-12-17 01:23:11 (900 s) " desc_text = _make_descriptor({"opt read-history": value}) desc = RelayDescriptor(desc_text) - self.assertEquals(value, desc.read_history_line) self.assertEquals(datetime.datetime(2005, 12, 17, 1, 23, 11), desc.read_history_end) self.assertEquals(900, desc.read_history_interval) self.assertEquals([], desc.read_history_values)

1 0

[stem/master] Reorganizing extrainfo fields by value type
by atagar＠torproject.org 14 May '12

14 May '12

commit b9cd295c09afe662247cc6a89442b40860b9157b Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 15:07:02 2012 -0700 Reorganizing extrainfo fields by value type Grouping both parsing and unit tests by the value format to make it more maintainable and easier to add the dozens of other attributes. Also using a 'dir_' prefix for all directory mirroring attributes rather than 'dirreq_' (I was previously being inconsistent on this). This expands the testing to cover more attributes, along with adding support for the "dirreq-v2-ips" and "dirreq-v3-ips" lines. Those lines revealed some misses in how I was parsing the maxmind locales. In particular I was missing '??' and numeric locales like 'a1' which it uses for an 'Anonymous Proxy'. --- stem/descriptor/extrainfo_descriptor.py | 128 +++++++++++++------------ test/unit/descriptor/extrainfo_descriptor.py | 130 ++++++++++++++++++-------- 2 files changed, 157 insertions(+), 101 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 93b438c..ce39ee3 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -155,8 +155,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): write_history_values (list) - bytes written during each interval Directory Mirror Attributes: - dirreq_stats_end (datetime) - end of the period when stats were gathered - dirreq_stats_interval (int) - length in seconds of the interval + dir_stats_end (datetime) - end of the period when stats were gathered + dir_stats_interval (int) - length in seconds of the interval + dir_v2_ips (dict) - mapping of locales to rounded count of requester ips + dir_v3_ips (dict) - mapping of locales to rounded count of requester ips Bytes read/written for directory mirroring dir_read_history_end (datetime) - end of the sampling interval @@ -213,8 +215,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.write_history_interval = None self.write_history_values = None - self.dirreq_stats_end = None - self.dirreq_stats_interval = None + self.dir_stats_end = None + self.dir_stats_interval = None + self.dir_v2_ips = None + self.dir_v3_ips = None self.dir_read_history_end = None self.dir_read_history_interval = None @@ -290,14 +294,6 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.nickname = extra_info_comp[0] self.fingerprint = extra_info_comp[1] - elif keyword == "published": - # "published" YYYY-MM-DD HH:MM:SS - - try: - self.published = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") - except ValueError: - if validate: - raise ValueError("Published line's time wasn't parseable: %s" % line) elif keyword == "geoip-db-digest": # "geoip-db-digest" Digest @@ -305,65 +301,44 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value - elif keyword == "geoip-start-time": - # "geoip-start-time" YYYY-MM-DD HH:MM:SS + elif keyword in ("published", "geoip-start-time"): + # "<keyword>" YYYY-MM-DD HH:MM:SS try: - self.geoip_start_time = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + timestamp = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + + if keyword == "published": + self.published = timestamp + elif keyword == "geoip-start-time": + self.geoip_start_time = timestamp except ValueError: if validate: - raise ValueError("Geoip start time line's time wasn't parseable: %s" % line) - elif keyword == "bridge-stats-end": - # "bridge-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) + raise ValueError("Timestamp on %s line wasn't parseable: %s" % (keyword, line)) + elif keyword in ("bridge-stats-end", "dirreq-stats-end"): + # "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) try: timestamp, interval, _ = _parse_timestamp_and_interval(keyword, value) - self.bridge_stats_end = timestamp - self.bridge_stats_interval = interval - except ValueError, exc: - if validate: raise exc - elif keyword in ("geoip-client-origins", "bridge-ips"): - # "geoip-client-origins" CC=N,CC=N,... - - locale_usage = {} - error_msg = "Entries in %s line should only be CC=N entries: %s" % (keyword, line) - - for entry in value.split(","): - if not "=" in entry: - if validate: raise ValueError(error_msg) - else: continue - locale, count = entry.split("=", 1) - - if re.match("^[a-zA-Z]{2}$", locale) and count.isdigit(): - locale_usage[locale] = int(count) - elif validate: - raise ValueError(error_msg) - - if keyword == "geoip-client-origins": - self.geoip_client_origins = locale_usage - elif keyword == "bridge-ips": - self.bridge_ips = locale_usage - elif keyword == "dirreq-stats-end": - # "dirreq-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) - - try: - timestamp, interval, _ = _parse_timestamp_and_interval(keyword, value) - self.dirreq_stats_end = timestamp - self.dirreq_stats_interval = interval + if keyword == "bridge-stats-end": + self.bridge_stats_end = timestamp + self.bridge_stats_interval = interval + elif keyword == "dirreq-stats-end": + self.dir_stats_end = timestamp + self.dir_stats_interval = interval except ValueError, exc: if validate: raise exc elif keyword in ("read-history", "write-history", "dirreq-read-history", "dirreq-write-history"): + # "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM,NUM,NUM... try: timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value) + history_values = [] - try: - if remainder: + if remainder: + try: history_values = [int(entry) for entry in remainder.split(",")] - else: - history_values = [] - except ValueError: - raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) + except ValueError: + raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) if keyword == "read-history": self.read_history_end = timestamp @@ -381,12 +356,45 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_write_history_end = timestamp self.dir_write_history_interval = interval self.dir_write_history_values = history_values - else: - # not gonna happen unless we change the main loop's conditional - # without fixing this one - raise ValueError("BUG: unrecognized keyword '%s'" % keyword) except ValueError, exc: if validate: raise exc + elif keyword in ("dirreq-v2-ips", "dirreq-v3-ips", "geoip-client-origins", "bridge-ips"): + # "<keyword>" CC=N,CC=N,... + + locale_usage = {} + error_msg = "Entries in %s line should only be CC=N entries: %s" % (keyword, line) + + if value: + for entry in value.split(","): + if not "=" in entry: + if validate: raise ValueError(error_msg) + else: continue + + # The maxmind geoip has numeric locale codes for some special + # values, for instance... + # + # A1,"Anonymous Proxy" + # A2,"Satellite Provider" + # ??,"Unknown" + # + # https://www.maxmind.com/app/iso3166 + + + locale, count = entry.split("=", 1) + + if re.match("^[a-zA-Z0-9\?]{2}$", locale) and count.isdigit(): + locale_usage[locale] = int(count) + elif validate: + raise ValueError(error_msg) + + if keyword == "dirreq-v2-ips": + self.dir_v2_ips = locale_usage + elif keyword == "dirreq-v3-ips": + self.dir_v3_ips = locale_usage + elif keyword == "geoip-client-origins": + self.geoip_client_origins = locale_usage + elif keyword == "bridge-ips": + self.bridge_ips = locale_usage elif keyword == "router-signature": if validate and not block_contents: raise ValueError("Router signature line must be followed by a signature block: %s" % line) diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index b4c5fbe..21f49a4 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -96,14 +96,14 @@ class TestExtraInfoDescriptor(unittest.TestCase): None. """ - test_entry = ( + test_entries = ( "ninja", "ninja ", "B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48", " B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48", ) - for entry in test_entry: + for entry in test_entries: desc_text = _make_descriptor({"extra-info": entry}) desc = self._expect_invalid_attr(desc_text, "nickname") self.assertEquals(None, desc.nickname) @@ -119,7 +119,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc = ExtraInfoDescriptor(desc_text) self.assertEquals(geoip_db_digest, desc.geoip_db_digest) - test_entry = ( + test_entries = ( "", "916A3CA8B7DF61473D5AE5B21711F35F301CE9E", "916A3CA8B7DF61473D5AE5B21711F35F301CE9E88", @@ -127,38 +127,42 @@ class TestExtraInfoDescriptor(unittest.TestCase): "916A3CA8B7DF61473D5AE5B21711F35F301CE9E-", ) - for entry in test_entry: + for entry in test_entries: desc_text = _make_descriptor({"geoip-db-digest": entry}) desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) - def test_geoip_start_time(self): + def test_timestamp_lines(self): """ - Parses the geoip-start-time line with valid and invalid data. + Uses valid and invalid data to tests lines of the form... + "<keyword>" YYYY-MM-DD HH:MM:SS """ - desc_text = _make_descriptor({"geoip-start-time": "2012-05-03 12:07:50"}) - desc = ExtraInfoDescriptor(desc_text) - self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), desc.geoip_start_time) - - test_entry = ( - "", - "2012-05-03 12:07:60", - "2012-05-03 ", - "2012-05-03", - ) - - for entry in test_entry: - desc_text = _make_descriptor({"geoip-start-time": entry}) - desc = self._expect_invalid_attr(desc_text, "geoip_start_time") + for keyword in ('published', 'geoip-start-time'): + attr = keyword.replace('-', '_') + + desc_text = _make_descriptor({keyword: "2012-05-03 12:07:50"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), getattr(desc, attr)) + + test_entries = ( + "", + "2012-05-03 12:07:60", + "2012-05-03 ", + "2012-05-03", + ) + + for entry in test_entries: + desc_text = _make_descriptor({keyword: entry}) + self._expect_invalid_attr(desc_text, attr) - def test_stats_end(self): + def test_timestamp_and_interval_lines(self): """ - Parses the bridge-stats-end and dirreq-stats-end lines with valid and - invalid data. + Uses valid and invalid data to tests lines of the form... + "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) """ for keyword in ('bridge-stats-end', 'dirreq-stats-end'): - end_attr = keyword.replace('-', '_') + end_attr = keyword.replace('-', '_').replace('dirreq', 'dir') interval_attr = end_attr[:-4] + "_interval" desc_text = _make_descriptor({keyword: "2012-05-03 12:07:50 (500 s)"}) @@ -166,48 +170,92 @@ class TestExtraInfoDescriptor(unittest.TestCase): self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), getattr(desc, end_attr)) self.assertEquals(500, getattr(desc, interval_attr)) - test_entry = ( + test_entries = ( "", + "2012-05-03 ", + "2012-05-03", "2012-05-03 12:07:60 (500 s)", "2012-05-03 12:07:50 (500s)", "2012-05-03 12:07:50 (500 s", "2012-05-03 12:07:50 (500 )", + ) + + for entry in test_entries: + desc_text = _make_descriptor({keyword: entry}) + desc = self._expect_invalid_attr(desc_text) + self.assertEquals(None, getattr(desc, end_attr)) + self.assertEquals(None, getattr(desc, interval_attr)) + + def test_timestamp_interval_and_value_lines(self): + """ + Uses valid and invalid data to tests lines of the form... + "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM,NUM,NUM... + """ + + for keyword in ('read-history', 'write-history', 'dirreq-read-history', 'dirreq-write-history'): + base_attr = keyword.replace('-', '_').replace('dirreq', 'dir') + end_attr = base_attr + "_end" + interval_attr = base_attr + "_interval" + values_attr = base_attr + "_values" + + test_entries = ( + ("", []), + (" ", []), + (" 50,11,5", [50, 11, 5]), + ) + + for test_values, expected_values in test_entries: + desc_text = _make_descriptor({keyword: "2012-05-03 12:07:50 (500 s)%s" % test_values}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), getattr(desc, end_attr)) + self.assertEquals(500, getattr(desc, interval_attr)) + self.assertEquals(expected_values, getattr(desc, values_attr)) + + test_entries = ( + "", "2012-05-03 ", "2012-05-03", + "2012-05-03 12:07:60 (500 s)", + "2012-05-03 12:07:50 (500s)", + "2012-05-03 12:07:50 (500 s", + "2012-05-03 12:07:50 (500 )", + "2012-05-03 12:07:50 (500 s)11", ) - for entry in test_entry: + for entry in test_entries: desc_text = _make_descriptor({keyword: entry}) desc = self._expect_invalid_attr(desc_text) self.assertEquals(None, getattr(desc, end_attr)) self.assertEquals(None, getattr(desc, interval_attr)) + self.assertEquals(None, getattr(desc, values_attr)) - def test_bridge_ips(self): + def test_locale_mapping_lines(self): """ - Parses both the bridge-ips and geoip-client-origins lines with valid and - invalid data. + Uses valid and invalid data to tests lines of the form... + "<keyword>" CC=N,CC=N,... """ - # Testing both attributes since they contain the exact same data, - # geoip-client-origins was simply replaced by bridge-ips while adding an - # interval value for the period. - - for keyword in ('bridge-ips', 'geoip-client-origins'): - attr = keyword.replace('-', '_') + for keyword in ('dirreq-v2-ips', 'dirreq-v3-ips', 'geoip-client-origins', 'bridge-ips'): + attr = keyword.replace('-', '_').replace('dirreq', 'dir') - desc_text = _make_descriptor({keyword: "uk=5,de=3,jp=2"}) - desc = ExtraInfoDescriptor(desc_text) - self.assertEquals({'uk': 5, 'de': 3, 'jp': 2}, getattr(desc, attr)) + test_entries = ( + ("", {}), + ("uk=5,de=3,jp=2", {'uk': 5, 'de': 3, 'jp': 2}), + ) - test_entry = ( - "", + for test_value, expected_value in test_entries: + desc_text = _make_descriptor({keyword: test_value}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(expected_value, getattr(desc, attr)) + + test_entries = ( "uk=-4", "uki=4", "uk:4", "uk=4.de=3", ) - for entry in test_entry: + for entry in test_entries: desc_text = _make_descriptor({keyword: entry}) desc = self._expect_invalid_attr(desc_text, attr, {})

1 0

[stem/master] Renaming dir_v*_reqs to dir_v*_requests
by atagar＠torproject.org 14 May '12

14 May '12

commit 63acd8169345270df0f1f6b545f67b5453b555fb Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 18:11:29 2012 -0700 Renaming dir_v*_reqs to dir_v*_requests It's a little understandable that we'd want to squeeze out a few characters in the descriptors (er, or it would if we didn't compress them), but there's no point for us to do the same. --- stem/descriptor/extrainfo_descriptor.py | 12 ++++++------ test/unit/descriptor/extrainfo_descriptor.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 58e2220..9b01ffc 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -159,8 +159,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_stats_interval (int) - length in seconds of the interval dir_v2_ips (dict) - mapping of locales to rounded count of requester ips dir_v3_ips (dict) - mapping of locales to rounded count of requester ips - dir_v2_reqs (dict) - mapping of locales to rounded count of requests - dir_v3_reqs (dict) - mapping of locales to rounded count of requests + dir_v2_requests (dict) - mapping of locales to rounded count of requests + dir_v3_requests (dict) - mapping of locales to rounded count of requests dir_v2_share (float) - percent of total directory traffic it expects to serve dir_v3_share (float) - percent of total directory traffic it expects to serve @@ -223,8 +223,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_stats_interval = None self.dir_v2_ips = None self.dir_v3_ips = None - self.dir_v2_reqs = None - self.dir_v3_reqs = None + self.dir_v2_requests = None + self.dir_v3_requests = None self.dir_v2_share = None self.dir_v3_share = None @@ -417,9 +417,9 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): elif keyword == "dirreq-v3-ips": self.dir_v3_ips = locale_usage elif keyword == "dirreq-v2-reqs": - self.dir_v2_reqs = locale_usage + self.dir_v2_requests = locale_usage elif keyword == "dirreq-v3-reqs": - self.dir_v3_reqs = locale_usage + self.dir_v3_requests = locale_usage elif keyword == "geoip-client-origins": self.geoip_client_origins = locale_usage elif keyword == "bridge-ips": diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 24034be..d8849c0 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -269,7 +269,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): """ for keyword in ('dirreq-v2-ips', 'dirreq-v3-ips', 'dirreq-v2-reqs', 'dirreq-v3-reqs', 'geoip-client-origins', 'bridge-ips'): - attr = keyword.replace('-', '_').replace('dirreq', 'dir') + attr = keyword.replace('-', '_').replace('dirreq', 'dir').replace('reqs', 'requests') test_entries = ( ("", {}),

1 0

[stem/master] Parsing dirreq-v*-direct-dl and dirreq-v*-tunneled-dl lines
by atagar＠torproject.org 14 May '12

14 May '12

commit bbf0e80cf889cc4b0738acf93c698fc54720c767 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 13:35:33 2012 -0700 Parsing dirreq-v*-direct-dl and dirreq-v*-tunneled-dl lines Parsing four extrainfo descriptor fields related to directory mirroring stat collection. These are similar to dirreq-v*-resp in that they're key=value mappings, with largely known key sets. --- stem/descriptor/extrainfo_descriptor.py | 69 ++++++++++++++++++++++--- test/integ/descriptor/extrainfo_descriptor.py | 6 ++ test/unit/descriptor/extrainfo_descriptor.py | 45 ++++++++++++++++- 3 files changed, 111 insertions(+), 9 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index d1967a9..9493f2b 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -24,6 +24,16 @@ DirResponses - known statuses for ExtraInfoDescriptor's dir_*_responses |- NOT_MODIFIED - network status unmodified since If-Modified-Since time +- BUSY - directory was busy +DirStats - known stats for ExtraInfoDescriptor's dir_*_direct_dl and dir_*_tunneled_dl + |- COMPLETE - requests that completed successfully + |- TIMEOUT - requests that didn't complete within a ten minute timeout + |- RUNNING - requests still in procress when measurement's taken + |- MIN - smallest rate at which a descriptor was downloaded in B/s + |- MAX - largest rate at which a descriptor was downloaded in B/s + |- D1-4 and D6-9 - rate of the slowest x/10 download rates in B/s + |- Q1 and Q3 - rate of the slowest and fastest querter download rates in B/s + +- MD - median download rate in B/s + parse_file - Iterates over the extra-info descriptors in a file. ExtraInfoDescriptor - Tor extra-info descriptor. +- get_unrecognized_lines - lines with unrecognized content @@ -45,6 +55,12 @@ DirResponses = stem.util.enum.Enum( ("BUSY", "busy"), ) +# known stats for dirreq-v2/3-direct-dl and dirreq-v2/3-tunneled-dl... +dir_stats = ['complete', 'timeout', 'running', 'min', 'max', 'q1', 'q3', 'md'] +dir_stats += ['d%i' % i for i in range(1, 5)] +dir_stats += ['d%i' % i for i in range(6, 10)] +DirStats = stem.util.enum.Enum(*[(stat.upper(), stat) for stat in dir_stats]) + # relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( "extra-info", @@ -178,14 +194,25 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_stats_interval (int) - length in seconds of the interval dir_v2_ips (dict) - mapping of locales to rounded count of requester ips dir_v3_ips (dict) - mapping of locales to rounded count of requester ips + dir_v2_share (float) - percent of total directory traffic it expects to serve + dir_v3_share (float) - percent of total directory traffic it expects to serve dir_v2_requests (dict) - mapping of locales to rounded count of requests dir_v3_requests (dict) - mapping of locales to rounded count of requests + dir_v2_responses (dict) - mapping of DirResponses to their rounded count dir_v3_responses (dict) - mapping of DirResponses to their rounded count dir_v2_responses_unknown (dict) - mapping of unrecognized statuses to their count dir_v3_responses_unknown (dict) - mapping of unrecognized statuses to their count - dir_v2_share (float) - percent of total directory traffic it expects to serve - dir_v3_share (float) - percent of total directory traffic it expects to serve + + dir_v2_direct_dl (dict) - mapping of DirStats to measurement over DirPort + dir_v3_direct_dl (dict) - mapping of DirStats to measurement over DirPort + dir_v2_direct_dl_unknown (dict) - mapping of unrecognized stats to their measurement + dir_v3_direct_dl_unknown (dict) - mapping of unrecognized stats to their measurement + + dir_v2_tunneled_dl (dict) - mapping of DirStats to measurement over ORPort + dir_v3_tunneled_dl (dict) - mapping of DirStats to measurement over ORPort + dir_v2_tunneled_dl_unknown (dict) - mapping of unrecognized stats to their measurement + dir_v3_tunneled_dl_unknown (dict) - mapping of unrecognized stats to their measurement Bytes read/written for directory mirroring dir_read_history_end (datetime) - end of the sampling interval @@ -246,14 +273,22 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_stats_interval = None self.dir_v2_ips = None self.dir_v3_ips = None + self.dir_v2_share = None + self.dir_v3_share = None self.dir_v2_requests = None self.dir_v3_requests = None self.dir_v2_responses = None self.dir_v3_responses = None self.dir_v2_responses_unknown = None self.dir_v3_responses_unknown = None - self.dir_v2_share = None - self.dir_v3_share = None + self.dir_v2_direct_dl = None + self.dir_v3_direct_dl = None + self.dir_v2_direct_dl_unknown = None + self.dir_v3_direct_dl_unknown = None + self.dir_v2_tunneled_dl = None + self.dir_v3_tunneled_dl = None + self.dir_v2_tunneled_dl_unknown = None + self.dir_v3_tunneled_dl_unknown = None self.dir_read_history_end = None self.dir_read_history_interval = None @@ -336,10 +371,16 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value - elif keyword in ("dirreq-v2-resp", "dirreq-v3-resp"): + elif keyword in ("dirreq-v2-resp", "dirreq-v3-resp", "dirreq-v2-direct-dl", "dirreq-v3-direct-dl", "dirreq-v2-tunneled-dl", "dirreq-v3-tunneled-dl"): recognized_counts = {} unrecognized_counts = {} - error_msg = "%s lines should contain STATUS=COUNT mappings: %s" % (keyword, line) + + is_response_stats = keyword in ("dirreq-v2-resp", "dirreq-v3-resp") + key_set = DirResponses if is_response_stats else DirStats + + key_type = "STATUS" if is_response_stats else "STAT" + error_msg = "%s lines should contain %s=COUNT mappings: %s" % (keyword, key_type, line) + if value: for entry in value.split(","): @@ -350,7 +391,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): status, count = entry.split("=", 1) if count.isdigit(): - if status in DirResponses: + if status in key_set: recognized_counts[status] = int(count) else: unrecognized_counts[status] = int(count) @@ -360,9 +401,21 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): if keyword == "dirreq-v2-resp": self.dir_v2_responses = recognized_counts self.dir_v2_responses_unknown = unrecognized_counts - else: + elif keyword == "dirreq-v3-resp": self.dir_v3_responses = recognized_counts self.dir_v3_responses_unknown = unrecognized_counts + elif keyword == "dirreq-v2-direct-dl": + self.dir_v2_direct_dl = recognized_counts + self.dir_v2_direct_dl_unknown = unrecognized_counts + elif keyword == "dirreq-v3-direct-dl": + self.dir_v3_direct_dl = recognized_counts + self.dir_v3_direct_dl_unknown = unrecognized_counts + elif keyword == "dirreq-v2-tunneled-dl": + self.dir_v2_tunneled_dl = recognized_counts + self.dir_v2_tunneled_dl_unknown = unrecognized_counts + elif keyword == "dirreq-v3-tunneled-dl": + self.dir_v3_tunneled_dl = recognized_counts + self.dir_v3_tunneled_dl_unknown = unrecognized_counts elif keyword in ("dirreq-v2-share", "dirreq-v3-share"): # "<keyword>" num% diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 841aa54..99edfea 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -87,6 +87,12 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw self.fail("Unrecognized statuses on dirreq-v2-resp lines: %s" % desc.dir_v2_responses_unknown) elif desc.dir_v3_responses_unknown: self.fail("Unrecognized statuses on dirreq-v3-resp lines: %s" % desc.dir_v3_responses_unknown) + elif desc.dir_v2_direct_dl_unknown: + self.fail("Unrecognized stats on dirreq-v2-direct-dl lines: %s" % desc.dir_v2_direct_dl_unknown) + elif desc.dir_v3_direct_dl_unknown: + self.fail("Unrecognized stats on dirreq-v3-direct-dl lines: %s" % desc.dir_v2_direct_dl_unknown) + elif desc.dir_v2_tunneled_dl_unknown: + self.fail("Unrecognized stats on dirreq-v2-tunneled-dl lines: %s" % desc.dir_v2_tunneled_dl_unknown) elif unrecognized_lines: # TODO: This isn't actually a problem, and rather than failing we # should alert the user about these entries at the end of the tests diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index b167ad4..e855a1d 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -4,7 +4,7 @@ Unit tests for stem.descriptor.extrainfo_descriptor. import datetime import unittest -from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor, DirResponses +from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor, DirResponses, DirStats CRYPTO_BLOB = """ K5FSywk7qvw/boA4DQcqkls6Ize5vcBYfhQ8JnOeRQC9+uDxbnpm3qaYN9jZ8myj @@ -162,6 +162,49 @@ class TestExtraInfoDescriptor(unittest.TestCase): self.assertEqual({}, getattr(desc, attr)) self.assertEqual({}, getattr(desc, unknown_attr)) + def test_dir_stat_lines(self): + """ + Parses the dirreq-v2-direct-dl, dirreq-v3-direct-dl, dirreq-v2-tunneled-dl, + and dirreq-v3-tunneled-dl lines with valid and invalid data. + """ + + for keyword in ("dirreq-v2-direct-dl", "dirreq-v2-direct-dl", "dirreq-v2-tunneled-dl", "dirreq-v2-tunneled-dl"): + attr = keyword.replace('-', '_').replace('dirreq', 'dir') + unknown_attr = attr + "_unknown" + + test_value = "complete=2712,timeout=32,running=4,min=741,d1=14507,d2=22702,q1=28881,d3=38277,d4=73729,md=111455,d6=168231,d7=257218,q3=319833,d8=390507,d9=616301,something-new=11,max=29917857" + desc_text = _make_descriptor({keyword: test_value}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(2712, getattr(desc, attr)[DirStats.COMPLETE]) + self.assertEquals(32, getattr(desc, attr)[DirStats.TIMEOUT]) + self.assertEquals(4, getattr(desc, attr)[DirStats.RUNNING]) + self.assertEquals(741, getattr(desc, attr)[DirStats.MIN]) + self.assertEquals(14507, getattr(desc, attr)[DirStats.D1]) + self.assertEquals(22702, getattr(desc, attr)[DirStats.D2]) + self.assertEquals(28881, getattr(desc, attr)[DirStats.Q1]) + self.assertEquals(38277, getattr(desc, attr)[DirStats.D3]) + self.assertEquals(73729, getattr(desc, attr)[DirStats.D4]) + self.assertEquals(111455, getattr(desc, attr)[DirStats.MD]) + self.assertEquals(168231, getattr(desc, attr)[DirStats.D6]) + self.assertEquals(257218, getattr(desc, attr)[DirStats.D7]) + self.assertEquals(319833, getattr(desc, attr)[DirStats.Q3]) + self.assertEquals(390507, getattr(desc, attr)[DirStats.D8]) + self.assertEquals(616301, getattr(desc, attr)[DirStats.D9]) + self.assertEquals(29917857, getattr(desc, attr)[DirStats.MAX]) + self.assertEquals(11, getattr(desc, unknown_attr)["something-new"]) + + test_entries = ( + "complete=-4", + "complete:4", + "complete=4.timeout=3", + ) + + for entry in test_entries: + desc_text = _make_descriptor({keyword: entry}) + desc = self._expect_invalid_attr(desc_text) + self.assertEqual({}, getattr(desc, attr)) + self.assertEqual({}, getattr(desc, unknown_attr)) + def test_percentage_lines(self): """ Uses valid and invalid data to tests lines of the form...

1 0