- tor-commits - lists.torproject.org

[stem/master] Parsing 'dirreq-v2-reqs' and 'dirreq-v3-reqs' lines
by atagar＠torproject.org 13 May '12

13 May '12

commit b82dca6e2e2543cfa19e9d15be1d8ba89227407d Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 15:15:55 2012 -0700 Parsing 'dirreq-v2-reqs' and 'dirreq-v3-reqs' lines Adding support for a couple more extrainfo fields. --- stem/descriptor/extrainfo_descriptor.py | 10 +++++++++- test/unit/descriptor/extrainfo_descriptor.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index ce39ee3..61e00dd 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -159,6 +159,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_stats_interval (int) - length in seconds of the interval dir_v2_ips (dict) - mapping of locales to rounded count of requester ips dir_v3_ips (dict) - mapping of locales to rounded count of requester ips + dir_v2_reqs (dict) - mapping of locales to rounded count of requests + dir_v3_reqs (dict) - mapping of locales to rounded count of requests Bytes read/written for directory mirroring dir_read_history_end (datetime) - end of the sampling interval @@ -219,6 +221,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_stats_interval = None self.dir_v2_ips = None self.dir_v3_ips = None + self.dir_v2_reqs = None + self.dir_v3_reqs = None self.dir_read_history_end = None self.dir_read_history_interval = None @@ -358,7 +362,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_write_history_values = history_values except ValueError, exc: if validate: raise exc - elif keyword in ("dirreq-v2-ips", "dirreq-v3-ips", "geoip-client-origins", "bridge-ips"): + elif keyword in ("dirreq-v2-ips", "dirreq-v3-ips", "dirreq-v2-reqs", "dirreq-v3-reqs", "geoip-client-origins", "bridge-ips"): # "<keyword>" CC=N,CC=N,... locale_usage = {} @@ -391,6 +395,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_v2_ips = locale_usage elif keyword == "dirreq-v3-ips": self.dir_v3_ips = locale_usage + elif keyword == "dirreq-v2-reqs": + self.dir_v2_reqs = locale_usage + elif keyword == "dirreq-v3-reqs": + self.dir_v3_reqs = locale_usage elif keyword == "geoip-client-origins": self.geoip_client_origins = locale_usage elif keyword == "bridge-ips": diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 21f49a4..9732d4d 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -235,7 +235,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): "<keyword>" CC=N,CC=N,... """ - for keyword in ('dirreq-v2-ips', 'dirreq-v3-ips', 'geoip-client-origins', 'bridge-ips'): + for keyword in ('dirreq-v2-ips', 'dirreq-v3-ips', 'dirreq-v2-reqs', 'dirreq-v3-reqs', 'geoip-client-origins', 'bridge-ips'): attr = keyword.replace('-', '_').replace('dirreq', 'dir') test_entries = (

1 0

[stem/master] Dropping measurement *_line attributes
by atagar＠torproject.org 13 May '12

13 May '12

commit 2ddffb3d961bcc01829ad58c96b8fa236a35cfff Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 12:42:25 2012 -0700 Dropping measurement *_line attributes Removing attributes like read_history_line which contained the raw value for the read-history line. These are left over from when I made no effort to parse the line, and are useless since we have their parsed values. Minor changes to our pydocs so these related items are better labeled. --- stem/descriptor/extrainfo_descriptor.py | 36 +++++++++---------------- stem/descriptor/server_descriptor.py | 21 +++++--------- test/integ/descriptor/extrainfo_descriptor.py | 12 -------- test/integ/descriptor/server_descriptor.py | 10 +----- test/unit/descriptor/server_descriptor.py | 14 +++------ 5 files changed, 28 insertions(+), 65 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index a8f7961..a9bbfec 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -145,15 +145,14 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): geoip_db_digest (str) - sha1 of geoIP database file signature (str) - signature for this extrainfo descriptor (*) - read_history_line (str) - bytes read for relayed traffic - read_history_end (datetime.datetime) - end of the sampling interval - read_history_interval (int) - seconds per interval - read_history_values (list) - bytes read during each interval (*) - - write_history_line (str) - bytes written for relayed traffic - write_history_end (datetime.datetime) - end of the sampling interval - write_history_interval (int) - seconds per interval - write_history_values (list) - bytes written during each interval (*) + Bytes read/written for relayed traffic + read_history_end (datetime.datetime) - end of the sampling interval + read_history_interval (int) - seconds per interval + read_history_values (list) - bytes read during each interval (*) + + write_history_end (datetime.datetime) - end of the sampling interval + write_history_interval (int) - seconds per interval + write_history_values (list) - bytes written during each interval (*) Directory Mirror Attributes: dirreq_stats_end (datetime.datetime) - end of the period where directory @@ -161,12 +160,11 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dirreq_stats_interval (int) - length in seconds of the interval where directory stats were gathered - dir_read_history_line (str) - bytes read for directory mirroring - dir_read_history_end (datetime.datetime) - end of the sampling interval - dir_read_history_interval (int) - seconds per interval - dir_read_history_values (list) - bytes read during each interval (*) - - dir_write_history_line (str) - bytes written for directory mirroring + Bytes read/written for directory mirroring + dir_read_history_end (datetime.datetime) - end of the sampling interval + dir_read_history_interval (int) - seconds per interval + dir_read_history_values (list) - bytes read during each interval (*) + dir_write_history_end (datetime.datetime) - end of the sampling interval dir_write_history_interval (int) - seconds per interval dir_write_history_values (list) - bytes read during each interval (*) @@ -211,12 +209,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.geoip_db_digest = None self.signature = None - self.read_history_line = None self.read_history_end = None self.read_history_interval = None self.read_history_values = [] - self.write_history_line = None self.write_history_end = None self.write_history_interval = None self.write_history_values = [] @@ -224,12 +220,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dirreq_stats_end = None self.dirreq_stats_interval = None - self.dir_read_history_line = None self.dir_read_history_end = None self.dir_read_history_interval = None self.dir_read_history_values = [] - self.dir_write_history_line = None self.dir_write_history_end = None self.dir_write_history_interval = None self.dir_write_history_values = [] @@ -376,22 +370,18 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) if keyword == "read-history": - self.read_history_line = value self.read_history_end = timestamp self.read_history_interval = interval self.read_history_values = history_values elif keyword == "write-history": - self.write_history_line = value self.write_history_end = timestamp self.write_history_interval = interval self.write_history_values = history_values elif keyword == "dirreq-read-history": - self.dir_read_history_line = value self.dir_read_history_end = timestamp self.dir_read_history_interval = interval self.dir_read_history_values = history_values elif keyword == "dirreq-write-history": - self.dir_write_history_line = value self.dir_write_history_end = timestamp self.dir_write_history_interval = interval self.dir_write_history_values = history_values diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 2963a53..7e16c77 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -167,15 +167,14 @@ class ServerDescriptor(stem.descriptor.Descriptor): observed_bandwidth (int) - estimated capacity of the relay based on usage in bytes/s (*) eventdns (bool) - (deprecated) always unset (*) - read_history_line (str) - (deprecated) bytes read for relayed traffic - read_history_end (datetime.datetime) - (deprecated) end of the sampling interval - read_history_interval (int) - (deprecated) seconds per interval - read_history_values (list) - (deprecated) bytes read during each interval (*) - - write_history_line (str) - (deprecated) bytes written for relayed traffic - write_history_end (datetime.datetime) - (deprecated) end of the sampling interval - write_history_interval (int) - (deprecated) seconds per interval - write_history_values (list) - (deprecated) bytes written during each interval (*) + Deprecated, moved to extra-info descriptor... + read_history_end (datetime.datetime) - end of the sampling interval + read_history_interval (int) - seconds per interval + read_history_values (list) - bytes read during each interval (*) + + write_history_end (datetime.datetime) - end of the sampling interval + write_history_interval (int) - seconds per interval + write_history_values (list) - bytes written during each interval (*) (*) required fields, others are left as None if undefined """ @@ -227,12 +226,10 @@ class ServerDescriptor(stem.descriptor.Descriptor): self.observed_bandwidth = None self.eventdns = True - self.read_history_line = None self.read_history_end = None self.read_history_interval = None self.read_history_values = [] - self.write_history_line = None self.write_history_end = None self.write_history_interval = None self.write_history_values = [] @@ -481,12 +478,10 @@ class ServerDescriptor(stem.descriptor.Descriptor): raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) if keyword == "read-history": - self.read_history_line = value self.read_history_end = timestamp self.read_history_interval = interval self.read_history_values = history_values else: - self.write_history_line = value self.write_history_end = timestamp self.write_history_interval = interval self.write_history_values = history_values diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 7a041d0..75b26e0 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -49,27 +49,15 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw # dirreq-write-history lines are pretty long so just checking # the initial contents for the line and parsed values. - read_start = "2012-05-05 17:02:45 (900 s) 3309568,9216,41984" - self.assertTrue(desc.read_history_line.startswith(read_start)) - read_values_start = [3309568, 9216, 41984, 27648, 123904] self.assertEquals(read_values_start, desc.read_history_values[:5]) - write_start = "2012-05-05 17:02:45 (900 s) 1082368,19456,50176,272384" - self.assertTrue(desc.write_history_line.startswith(write_start)) - write_values_start = [1082368, 19456, 50176, 272384, 485376] self.assertEquals(write_values_start, desc.write_history_values[:5]) - dir_read_start = "2012-05-05 17:02:45 (900 s) 0,0,0,0,33792,27648,48128" - self.assertTrue(desc.dir_read_history_line.startswith(dir_read_start)) - dir_read_values_start = [0, 0, 0, 0, 33792, 27648, 48128] self.assertEquals(dir_read_values_start, desc.dir_read_history_values[:7]) - dir_write_start = "2012-05-05 17:02:45 (900 s) 0,0,0,227328,349184" - self.assertTrue(desc.dir_write_history_line.startswith(dir_write_start)) - dir_write_values_start = [0, 0, 0, 227328, 349184, 382976, 738304] self.assertEquals(dir_write_values_start, desc.dir_write_history_values[:7]) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 945d147..d7b5c14 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -133,15 +133,9 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= # The read-history and write-history lines are pretty long so just checking # the initial contents for the line and parsed values. - read_start = "2005-12-16 18:00:48 (900 s) 20774,489973,510022" - self.assertTrue(desc.read_history_line.startswith(read_start)) - read_values_start = [20774, 489973, 510022, 511163, 20949] self.assertEquals(read_values_start, desc.read_history_values[:5]) - write_start = "2005-12-16 18:00:48 (900 s) 81,8848,8927,8927" - self.assertTrue(desc.write_history_line.startswith(write_start)) - write_values_start = [81, 8848, 8927, 8927, 83, 8848, 8931, 8929, 81, 8846] self.assertEquals(write_values_start, desc.write_history_values[:10]) @@ -164,8 +158,8 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= with open(descriptor_path) as descriptor_file: for desc in stem.descriptor.server_descriptor.parse_file(descriptor_file): # the following attributes should be deprecated, and not appear in the wild - self.assertEquals(None, desc.read_history_line) - self.assertEquals(None, desc.write_history_line) + self.assertEquals(None, desc.read_history_end) + self.assertEquals(None, desc.write_history_end) self.assertEquals(True, desc.eventdns) self.assertEquals(0, desc.socks_port) diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 63fd89d..e381318 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -235,19 +235,16 @@ class TestServerDescriptor(unittest.TestCase): desc = RelayDescriptor(desc_text) if field == "read-history": - attr = (desc.read_history_line, desc.read_history_end, - desc.read_history_interval, desc.read_history_values) + attr = (desc.read_history_end, desc.read_history_interval, desc.read_history_values) else: - attr = (desc.write_history_line, desc.write_history_end, - desc.write_history_interval, desc.write_history_values) + attr = (desc.write_history_end, desc.write_history_interval, desc.write_history_values) expected_end = datetime.datetime(2005, 12, 16, 18, 0, 48) expected_values = [81, 8848, 8927, 8927, 83, 8848] - self.assertEquals(value, attr[0]) - self.assertEquals(expected_end, attr[1]) - self.assertEquals(900, attr[2]) - self.assertEquals(expected_values, attr[3]) + self.assertEquals(expected_end, attr[0]) + self.assertEquals(900, attr[1]) + self.assertEquals(expected_values, attr[2]) def test_read_history_empty(self): """ @@ -257,7 +254,6 @@ class TestServerDescriptor(unittest.TestCase): value = "2005-12-17 01:23:11 (900 s) " desc_text = _make_descriptor({"opt read-history": value}) desc = RelayDescriptor(desc_text) - self.assertEquals(value, desc.read_history_line) self.assertEquals(datetime.datetime(2005, 12, 17, 1, 23, 11), desc.read_history_end) self.assertEquals(900, desc.read_history_interval) self.assertEquals([], desc.read_history_values)

1 0

[stem/master] Reorganizing extrainfo fields by value type
by atagar＠torproject.org 13 May '12

13 May '12

commit b9cd295c09afe662247cc6a89442b40860b9157b Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 15:07:02 2012 -0700 Reorganizing extrainfo fields by value type Grouping both parsing and unit tests by the value format to make it more maintainable and easier to add the dozens of other attributes. Also using a 'dir_' prefix for all directory mirroring attributes rather than 'dirreq_' (I was previously being inconsistent on this). This expands the testing to cover more attributes, along with adding support for the "dirreq-v2-ips" and "dirreq-v3-ips" lines. Those lines revealed some misses in how I was parsing the maxmind locales. In particular I was missing '??' and numeric locales like 'a1' which it uses for an 'Anonymous Proxy'. --- stem/descriptor/extrainfo_descriptor.py | 128 +++++++++++++------------ test/unit/descriptor/extrainfo_descriptor.py | 130 ++++++++++++++++++-------- 2 files changed, 157 insertions(+), 101 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 93b438c..ce39ee3 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -155,8 +155,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): write_history_values (list) - bytes written during each interval Directory Mirror Attributes: - dirreq_stats_end (datetime) - end of the period when stats were gathered - dirreq_stats_interval (int) - length in seconds of the interval + dir_stats_end (datetime) - end of the period when stats were gathered + dir_stats_interval (int) - length in seconds of the interval + dir_v2_ips (dict) - mapping of locales to rounded count of requester ips + dir_v3_ips (dict) - mapping of locales to rounded count of requester ips Bytes read/written for directory mirroring dir_read_history_end (datetime) - end of the sampling interval @@ -213,8 +215,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.write_history_interval = None self.write_history_values = None - self.dirreq_stats_end = None - self.dirreq_stats_interval = None + self.dir_stats_end = None + self.dir_stats_interval = None + self.dir_v2_ips = None + self.dir_v3_ips = None self.dir_read_history_end = None self.dir_read_history_interval = None @@ -290,14 +294,6 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.nickname = extra_info_comp[0] self.fingerprint = extra_info_comp[1] - elif keyword == "published": - # "published" YYYY-MM-DD HH:MM:SS - - try: - self.published = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") - except ValueError: - if validate: - raise ValueError("Published line's time wasn't parseable: %s" % line) elif keyword == "geoip-db-digest": # "geoip-db-digest" Digest @@ -305,65 +301,44 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value - elif keyword == "geoip-start-time": - # "geoip-start-time" YYYY-MM-DD HH:MM:SS + elif keyword in ("published", "geoip-start-time"): + # "<keyword>" YYYY-MM-DD HH:MM:SS try: - self.geoip_start_time = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + timestamp = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + + if keyword == "published": + self.published = timestamp + elif keyword == "geoip-start-time": + self.geoip_start_time = timestamp except ValueError: if validate: - raise ValueError("Geoip start time line's time wasn't parseable: %s" % line) - elif keyword == "bridge-stats-end": - # "bridge-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) + raise ValueError("Timestamp on %s line wasn't parseable: %s" % (keyword, line)) + elif keyword in ("bridge-stats-end", "dirreq-stats-end"): + # "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) try: timestamp, interval, _ = _parse_timestamp_and_interval(keyword, value) - self.bridge_stats_end = timestamp - self.bridge_stats_interval = interval - except ValueError, exc: - if validate: raise exc - elif keyword in ("geoip-client-origins", "bridge-ips"): - # "geoip-client-origins" CC=N,CC=N,... - - locale_usage = {} - error_msg = "Entries in %s line should only be CC=N entries: %s" % (keyword, line) - - for entry in value.split(","): - if not "=" in entry: - if validate: raise ValueError(error_msg) - else: continue - locale, count = entry.split("=", 1) - - if re.match("^[a-zA-Z]{2}$", locale) and count.isdigit(): - locale_usage[locale] = int(count) - elif validate: - raise ValueError(error_msg) - - if keyword == "geoip-client-origins": - self.geoip_client_origins = locale_usage - elif keyword == "bridge-ips": - self.bridge_ips = locale_usage - elif keyword == "dirreq-stats-end": - # "dirreq-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) - - try: - timestamp, interval, _ = _parse_timestamp_and_interval(keyword, value) - self.dirreq_stats_end = timestamp - self.dirreq_stats_interval = interval + if keyword == "bridge-stats-end": + self.bridge_stats_end = timestamp + self.bridge_stats_interval = interval + elif keyword == "dirreq-stats-end": + self.dir_stats_end = timestamp + self.dir_stats_interval = interval except ValueError, exc: if validate: raise exc elif keyword in ("read-history", "write-history", "dirreq-read-history", "dirreq-write-history"): + # "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM,NUM,NUM... try: timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value) + history_values = [] - try: - if remainder: + if remainder: + try: history_values = [int(entry) for entry in remainder.split(",")] - else: - history_values = [] - except ValueError: - raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) + except ValueError: + raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) if keyword == "read-history": self.read_history_end = timestamp @@ -381,12 +356,45 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_write_history_end = timestamp self.dir_write_history_interval = interval self.dir_write_history_values = history_values - else: - # not gonna happen unless we change the main loop's conditional - # without fixing this one - raise ValueError("BUG: unrecognized keyword '%s'" % keyword) except ValueError, exc: if validate: raise exc + elif keyword in ("dirreq-v2-ips", "dirreq-v3-ips", "geoip-client-origins", "bridge-ips"): + # "<keyword>" CC=N,CC=N,... + + locale_usage = {} + error_msg = "Entries in %s line should only be CC=N entries: %s" % (keyword, line) + + if value: + for entry in value.split(","): + if not "=" in entry: + if validate: raise ValueError(error_msg) + else: continue + + # The maxmind geoip has numeric locale codes for some special + # values, for instance... + # + # A1,"Anonymous Proxy" + # A2,"Satellite Provider" + # ??,"Unknown" + # + # https://www.maxmind.com/app/iso3166 + + + locale, count = entry.split("=", 1) + + if re.match("^[a-zA-Z0-9\?]{2}$", locale) and count.isdigit(): + locale_usage[locale] = int(count) + elif validate: + raise ValueError(error_msg) + + if keyword == "dirreq-v2-ips": + self.dir_v2_ips = locale_usage + elif keyword == "dirreq-v3-ips": + self.dir_v3_ips = locale_usage + elif keyword == "geoip-client-origins": + self.geoip_client_origins = locale_usage + elif keyword == "bridge-ips": + self.bridge_ips = locale_usage elif keyword == "router-signature": if validate and not block_contents: raise ValueError("Router signature line must be followed by a signature block: %s" % line) diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index b4c5fbe..21f49a4 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -96,14 +96,14 @@ class TestExtraInfoDescriptor(unittest.TestCase): None. """ - test_entry = ( + test_entries = ( "ninja", "ninja ", "B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48", " B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48", ) - for entry in test_entry: + for entry in test_entries: desc_text = _make_descriptor({"extra-info": entry}) desc = self._expect_invalid_attr(desc_text, "nickname") self.assertEquals(None, desc.nickname) @@ -119,7 +119,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc = ExtraInfoDescriptor(desc_text) self.assertEquals(geoip_db_digest, desc.geoip_db_digest) - test_entry = ( + test_entries = ( "", "916A3CA8B7DF61473D5AE5B21711F35F301CE9E", "916A3CA8B7DF61473D5AE5B21711F35F301CE9E88", @@ -127,38 +127,42 @@ class TestExtraInfoDescriptor(unittest.TestCase): "916A3CA8B7DF61473D5AE5B21711F35F301CE9E-", ) - for entry in test_entry: + for entry in test_entries: desc_text = _make_descriptor({"geoip-db-digest": entry}) desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) - def test_geoip_start_time(self): + def test_timestamp_lines(self): """ - Parses the geoip-start-time line with valid and invalid data. + Uses valid and invalid data to tests lines of the form... + "<keyword>" YYYY-MM-DD HH:MM:SS """ - desc_text = _make_descriptor({"geoip-start-time": "2012-05-03 12:07:50"}) - desc = ExtraInfoDescriptor(desc_text) - self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), desc.geoip_start_time) - - test_entry = ( - "", - "2012-05-03 12:07:60", - "2012-05-03 ", - "2012-05-03", - ) - - for entry in test_entry: - desc_text = _make_descriptor({"geoip-start-time": entry}) - desc = self._expect_invalid_attr(desc_text, "geoip_start_time") + for keyword in ('published', 'geoip-start-time'): + attr = keyword.replace('-', '_') + + desc_text = _make_descriptor({keyword: "2012-05-03 12:07:50"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), getattr(desc, attr)) + + test_entries = ( + "", + "2012-05-03 12:07:60", + "2012-05-03 ", + "2012-05-03", + ) + + for entry in test_entries: + desc_text = _make_descriptor({keyword: entry}) + self._expect_invalid_attr(desc_text, attr) - def test_stats_end(self): + def test_timestamp_and_interval_lines(self): """ - Parses the bridge-stats-end and dirreq-stats-end lines with valid and - invalid data. + Uses valid and invalid data to tests lines of the form... + "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) """ for keyword in ('bridge-stats-end', 'dirreq-stats-end'): - end_attr = keyword.replace('-', '_') + end_attr = keyword.replace('-', '_').replace('dirreq', 'dir') interval_attr = end_attr[:-4] + "_interval" desc_text = _make_descriptor({keyword: "2012-05-03 12:07:50 (500 s)"}) @@ -166,48 +170,92 @@ class TestExtraInfoDescriptor(unittest.TestCase): self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), getattr(desc, end_attr)) self.assertEquals(500, getattr(desc, interval_attr)) - test_entry = ( + test_entries = ( "", + "2012-05-03 ", + "2012-05-03", "2012-05-03 12:07:60 (500 s)", "2012-05-03 12:07:50 (500s)", "2012-05-03 12:07:50 (500 s", "2012-05-03 12:07:50 (500 )", + ) + + for entry in test_entries: + desc_text = _make_descriptor({keyword: entry}) + desc = self._expect_invalid_attr(desc_text) + self.assertEquals(None, getattr(desc, end_attr)) + self.assertEquals(None, getattr(desc, interval_attr)) + + def test_timestamp_interval_and_value_lines(self): + """ + Uses valid and invalid data to tests lines of the form... + "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM,NUM,NUM... + """ + + for keyword in ('read-history', 'write-history', 'dirreq-read-history', 'dirreq-write-history'): + base_attr = keyword.replace('-', '_').replace('dirreq', 'dir') + end_attr = base_attr + "_end" + interval_attr = base_attr + "_interval" + values_attr = base_attr + "_values" + + test_entries = ( + ("", []), + (" ", []), + (" 50,11,5", [50, 11, 5]), + ) + + for test_values, expected_values in test_entries: + desc_text = _make_descriptor({keyword: "2012-05-03 12:07:50 (500 s)%s" % test_values}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), getattr(desc, end_attr)) + self.assertEquals(500, getattr(desc, interval_attr)) + self.assertEquals(expected_values, getattr(desc, values_attr)) + + test_entries = ( + "", "2012-05-03 ", "2012-05-03", + "2012-05-03 12:07:60 (500 s)", + "2012-05-03 12:07:50 (500s)", + "2012-05-03 12:07:50 (500 s", + "2012-05-03 12:07:50 (500 )", + "2012-05-03 12:07:50 (500 s)11", ) - for entry in test_entry: + for entry in test_entries: desc_text = _make_descriptor({keyword: entry}) desc = self._expect_invalid_attr(desc_text) self.assertEquals(None, getattr(desc, end_attr)) self.assertEquals(None, getattr(desc, interval_attr)) + self.assertEquals(None, getattr(desc, values_attr)) - def test_bridge_ips(self): + def test_locale_mapping_lines(self): """ - Parses both the bridge-ips and geoip-client-origins lines with valid and - invalid data. + Uses valid and invalid data to tests lines of the form... + "<keyword>" CC=N,CC=N,... """ - # Testing both attributes since they contain the exact same data, - # geoip-client-origins was simply replaced by bridge-ips while adding an - # interval value for the period. - - for keyword in ('bridge-ips', 'geoip-client-origins'): - attr = keyword.replace('-', '_') + for keyword in ('dirreq-v2-ips', 'dirreq-v3-ips', 'geoip-client-origins', 'bridge-ips'): + attr = keyword.replace('-', '_').replace('dirreq', 'dir') - desc_text = _make_descriptor({keyword: "uk=5,de=3,jp=2"}) - desc = ExtraInfoDescriptor(desc_text) - self.assertEquals({'uk': 5, 'de': 3, 'jp': 2}, getattr(desc, attr)) + test_entries = ( + ("", {}), + ("uk=5,de=3,jp=2", {'uk': 5, 'de': 3, 'jp': 2}), + ) - test_entry = ( - "", + for test_value, expected_value in test_entries: + desc_text = _make_descriptor({keyword: test_value}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(expected_value, getattr(desc, attr)) + + test_entries = ( "uk=-4", "uki=4", "uk:4", "uk=4.de=3", ) - for entry in test_entry: + for entry in test_entries: desc_text = _make_descriptor({keyword: entry}) desc = self._expect_invalid_attr(desc_text, attr, {})

1 0

[stem/master] Renaming dir_v*_reqs to dir_v*_requests
by atagar＠torproject.org 13 May '12

13 May '12

commit 63acd8169345270df0f1f6b545f67b5453b555fb Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 18:11:29 2012 -0700 Renaming dir_v*_reqs to dir_v*_requests It's a little understandable that we'd want to squeeze out a few characters in the descriptors (er, or it would if we didn't compress them), but there's no point for us to do the same. --- stem/descriptor/extrainfo_descriptor.py | 12 ++++++------ test/unit/descriptor/extrainfo_descriptor.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 58e2220..9b01ffc 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -159,8 +159,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_stats_interval (int) - length in seconds of the interval dir_v2_ips (dict) - mapping of locales to rounded count of requester ips dir_v3_ips (dict) - mapping of locales to rounded count of requester ips - dir_v2_reqs (dict) - mapping of locales to rounded count of requests - dir_v3_reqs (dict) - mapping of locales to rounded count of requests + dir_v2_requests (dict) - mapping of locales to rounded count of requests + dir_v3_requests (dict) - mapping of locales to rounded count of requests dir_v2_share (float) - percent of total directory traffic it expects to serve dir_v3_share (float) - percent of total directory traffic it expects to serve @@ -223,8 +223,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_stats_interval = None self.dir_v2_ips = None self.dir_v3_ips = None - self.dir_v2_reqs = None - self.dir_v3_reqs = None + self.dir_v2_requests = None + self.dir_v3_requests = None self.dir_v2_share = None self.dir_v3_share = None @@ -417,9 +417,9 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): elif keyword == "dirreq-v3-ips": self.dir_v3_ips = locale_usage elif keyword == "dirreq-v2-reqs": - self.dir_v2_reqs = locale_usage + self.dir_v2_requests = locale_usage elif keyword == "dirreq-v3-reqs": - self.dir_v3_reqs = locale_usage + self.dir_v3_requests = locale_usage elif keyword == "geoip-client-origins": self.geoip_client_origins = locale_usage elif keyword == "bridge-ips": diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 24034be..d8849c0 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -269,7 +269,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): """ for keyword in ('dirreq-v2-ips', 'dirreq-v3-ips', 'dirreq-v2-reqs', 'dirreq-v3-reqs', 'geoip-client-origins', 'bridge-ips'): - attr = keyword.replace('-', '_').replace('dirreq', 'dir') + attr = keyword.replace('-', '_').replace('dirreq', 'dir').replace('reqs', 'requests') test_entries = ( ("", {}),

1 0

[stem/master] Parsing dirreq-v*-direct-dl and dirreq-v*-tunneled-dl lines
by atagar＠torproject.org 13 May '12

13 May '12

commit bbf0e80cf889cc4b0738acf93c698fc54720c767 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 13:35:33 2012 -0700 Parsing dirreq-v*-direct-dl and dirreq-v*-tunneled-dl lines Parsing four extrainfo descriptor fields related to directory mirroring stat collection. These are similar to dirreq-v*-resp in that they're key=value mappings, with largely known key sets. --- stem/descriptor/extrainfo_descriptor.py | 69 ++++++++++++++++++++++--- test/integ/descriptor/extrainfo_descriptor.py | 6 ++ test/unit/descriptor/extrainfo_descriptor.py | 45 ++++++++++++++++- 3 files changed, 111 insertions(+), 9 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index d1967a9..9493f2b 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -24,6 +24,16 @@ DirResponses - known statuses for ExtraInfoDescriptor's dir_*_responses |- NOT_MODIFIED - network status unmodified since If-Modified-Since time +- BUSY - directory was busy +DirStats - known stats for ExtraInfoDescriptor's dir_*_direct_dl and dir_*_tunneled_dl + |- COMPLETE - requests that completed successfully + |- TIMEOUT - requests that didn't complete within a ten minute timeout + |- RUNNING - requests still in procress when measurement's taken + |- MIN - smallest rate at which a descriptor was downloaded in B/s + |- MAX - largest rate at which a descriptor was downloaded in B/s + |- D1-4 and D6-9 - rate of the slowest x/10 download rates in B/s + |- Q1 and Q3 - rate of the slowest and fastest querter download rates in B/s + +- MD - median download rate in B/s + parse_file - Iterates over the extra-info descriptors in a file. ExtraInfoDescriptor - Tor extra-info descriptor. +- get_unrecognized_lines - lines with unrecognized content @@ -45,6 +55,12 @@ DirResponses = stem.util.enum.Enum( ("BUSY", "busy"), ) +# known stats for dirreq-v2/3-direct-dl and dirreq-v2/3-tunneled-dl... +dir_stats = ['complete', 'timeout', 'running', 'min', 'max', 'q1', 'q3', 'md'] +dir_stats += ['d%i' % i for i in range(1, 5)] +dir_stats += ['d%i' % i for i in range(6, 10)] +DirStats = stem.util.enum.Enum(*[(stat.upper(), stat) for stat in dir_stats]) + # relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( "extra-info", @@ -178,14 +194,25 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_stats_interval (int) - length in seconds of the interval dir_v2_ips (dict) - mapping of locales to rounded count of requester ips dir_v3_ips (dict) - mapping of locales to rounded count of requester ips + dir_v2_share (float) - percent of total directory traffic it expects to serve + dir_v3_share (float) - percent of total directory traffic it expects to serve dir_v2_requests (dict) - mapping of locales to rounded count of requests dir_v3_requests (dict) - mapping of locales to rounded count of requests + dir_v2_responses (dict) - mapping of DirResponses to their rounded count dir_v3_responses (dict) - mapping of DirResponses to their rounded count dir_v2_responses_unknown (dict) - mapping of unrecognized statuses to their count dir_v3_responses_unknown (dict) - mapping of unrecognized statuses to their count - dir_v2_share (float) - percent of total directory traffic it expects to serve - dir_v3_share (float) - percent of total directory traffic it expects to serve + + dir_v2_direct_dl (dict) - mapping of DirStats to measurement over DirPort + dir_v3_direct_dl (dict) - mapping of DirStats to measurement over DirPort + dir_v2_direct_dl_unknown (dict) - mapping of unrecognized stats to their measurement + dir_v3_direct_dl_unknown (dict) - mapping of unrecognized stats to their measurement + + dir_v2_tunneled_dl (dict) - mapping of DirStats to measurement over ORPort + dir_v3_tunneled_dl (dict) - mapping of DirStats to measurement over ORPort + dir_v2_tunneled_dl_unknown (dict) - mapping of unrecognized stats to their measurement + dir_v3_tunneled_dl_unknown (dict) - mapping of unrecognized stats to their measurement Bytes read/written for directory mirroring dir_read_history_end (datetime) - end of the sampling interval @@ -246,14 +273,22 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_stats_interval = None self.dir_v2_ips = None self.dir_v3_ips = None + self.dir_v2_share = None + self.dir_v3_share = None self.dir_v2_requests = None self.dir_v3_requests = None self.dir_v2_responses = None self.dir_v3_responses = None self.dir_v2_responses_unknown = None self.dir_v3_responses_unknown = None - self.dir_v2_share = None - self.dir_v3_share = None + self.dir_v2_direct_dl = None + self.dir_v3_direct_dl = None + self.dir_v2_direct_dl_unknown = None + self.dir_v3_direct_dl_unknown = None + self.dir_v2_tunneled_dl = None + self.dir_v3_tunneled_dl = None + self.dir_v2_tunneled_dl_unknown = None + self.dir_v3_tunneled_dl_unknown = None self.dir_read_history_end = None self.dir_read_history_interval = None @@ -336,10 +371,16 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value - elif keyword in ("dirreq-v2-resp", "dirreq-v3-resp"): + elif keyword in ("dirreq-v2-resp", "dirreq-v3-resp", "dirreq-v2-direct-dl", "dirreq-v3-direct-dl", "dirreq-v2-tunneled-dl", "dirreq-v3-tunneled-dl"): recognized_counts = {} unrecognized_counts = {} - error_msg = "%s lines should contain STATUS=COUNT mappings: %s" % (keyword, line) + + is_response_stats = keyword in ("dirreq-v2-resp", "dirreq-v3-resp") + key_set = DirResponses if is_response_stats else DirStats + + key_type = "STATUS" if is_response_stats else "STAT" + error_msg = "%s lines should contain %s=COUNT mappings: %s" % (keyword, key_type, line) + if value: for entry in value.split(","): @@ -350,7 +391,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): status, count = entry.split("=", 1) if count.isdigit(): - if status in DirResponses: + if status in key_set: recognized_counts[status] = int(count) else: unrecognized_counts[status] = int(count) @@ -360,9 +401,21 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): if keyword == "dirreq-v2-resp": self.dir_v2_responses = recognized_counts self.dir_v2_responses_unknown = unrecognized_counts - else: + elif keyword == "dirreq-v3-resp": self.dir_v3_responses = recognized_counts self.dir_v3_responses_unknown = unrecognized_counts + elif keyword == "dirreq-v2-direct-dl": + self.dir_v2_direct_dl = recognized_counts + self.dir_v2_direct_dl_unknown = unrecognized_counts + elif keyword == "dirreq-v3-direct-dl": + self.dir_v3_direct_dl = recognized_counts + self.dir_v3_direct_dl_unknown = unrecognized_counts + elif keyword == "dirreq-v2-tunneled-dl": + self.dir_v2_tunneled_dl = recognized_counts + self.dir_v2_tunneled_dl_unknown = unrecognized_counts + elif keyword == "dirreq-v3-tunneled-dl": + self.dir_v3_tunneled_dl = recognized_counts + self.dir_v3_tunneled_dl_unknown = unrecognized_counts elif keyword in ("dirreq-v2-share", "dirreq-v3-share"): # "<keyword>" num% diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 841aa54..99edfea 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -87,6 +87,12 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw self.fail("Unrecognized statuses on dirreq-v2-resp lines: %s" % desc.dir_v2_responses_unknown) elif desc.dir_v3_responses_unknown: self.fail("Unrecognized statuses on dirreq-v3-resp lines: %s" % desc.dir_v3_responses_unknown) + elif desc.dir_v2_direct_dl_unknown: + self.fail("Unrecognized stats on dirreq-v2-direct-dl lines: %s" % desc.dir_v2_direct_dl_unknown) + elif desc.dir_v3_direct_dl_unknown: + self.fail("Unrecognized stats on dirreq-v3-direct-dl lines: %s" % desc.dir_v2_direct_dl_unknown) + elif desc.dir_v2_tunneled_dl_unknown: + self.fail("Unrecognized stats on dirreq-v2-tunneled-dl lines: %s" % desc.dir_v2_tunneled_dl_unknown) elif unrecognized_lines: # TODO: This isn't actually a problem, and rather than failing we # should alert the user about these entries at the end of the tests diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index b167ad4..e855a1d 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -4,7 +4,7 @@ Unit tests for stem.descriptor.extrainfo_descriptor. import datetime import unittest -from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor, DirResponses +from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor, DirResponses, DirStats CRYPTO_BLOB = """ K5FSywk7qvw/boA4DQcqkls6Ize5vcBYfhQ8JnOeRQC9+uDxbnpm3qaYN9jZ8myj @@ -162,6 +162,49 @@ class TestExtraInfoDescriptor(unittest.TestCase): self.assertEqual({}, getattr(desc, attr)) self.assertEqual({}, getattr(desc, unknown_attr)) + def test_dir_stat_lines(self): + """ + Parses the dirreq-v2-direct-dl, dirreq-v3-direct-dl, dirreq-v2-tunneled-dl, + and dirreq-v3-tunneled-dl lines with valid and invalid data. + """ + + for keyword in ("dirreq-v2-direct-dl", "dirreq-v2-direct-dl", "dirreq-v2-tunneled-dl", "dirreq-v2-tunneled-dl"): + attr = keyword.replace('-', '_').replace('dirreq', 'dir') + unknown_attr = attr + "_unknown" + + test_value = "complete=2712,timeout=32,running=4,min=741,d1=14507,d2=22702,q1=28881,d3=38277,d4=73729,md=111455,d6=168231,d7=257218,q3=319833,d8=390507,d9=616301,something-new=11,max=29917857" + desc_text = _make_descriptor({keyword: test_value}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(2712, getattr(desc, attr)[DirStats.COMPLETE]) + self.assertEquals(32, getattr(desc, attr)[DirStats.TIMEOUT]) + self.assertEquals(4, getattr(desc, attr)[DirStats.RUNNING]) + self.assertEquals(741, getattr(desc, attr)[DirStats.MIN]) + self.assertEquals(14507, getattr(desc, attr)[DirStats.D1]) + self.assertEquals(22702, getattr(desc, attr)[DirStats.D2]) + self.assertEquals(28881, getattr(desc, attr)[DirStats.Q1]) + self.assertEquals(38277, getattr(desc, attr)[DirStats.D3]) + self.assertEquals(73729, getattr(desc, attr)[DirStats.D4]) + self.assertEquals(111455, getattr(desc, attr)[DirStats.MD]) + self.assertEquals(168231, getattr(desc, attr)[DirStats.D6]) + self.assertEquals(257218, getattr(desc, attr)[DirStats.D7]) + self.assertEquals(319833, getattr(desc, attr)[DirStats.Q3]) + self.assertEquals(390507, getattr(desc, attr)[DirStats.D8]) + self.assertEquals(616301, getattr(desc, attr)[DirStats.D9]) + self.assertEquals(29917857, getattr(desc, attr)[DirStats.MAX]) + self.assertEquals(11, getattr(desc, unknown_attr)["something-new"]) + + test_entries = ( + "complete=-4", + "complete:4", + "complete=4.timeout=3", + ) + + for entry in test_entries: + desc_text = _make_descriptor({keyword: entry}) + desc = self._expect_invalid_attr(desc_text) + self.assertEqual({}, getattr(desc, attr)) + self.assertEqual({}, getattr(desc, unknown_attr)) + def test_percentage_lines(self): """ Uses valid and invalid data to tests lines of the form...

1 0

[stem/master] Parsing 'cell-circuits-per-decile' extrainfo lines
by atagar＠torproject.org 13 May '12

13 May '12

commit 153a98eca741d3185e7d20c5e3566763a54384bd Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 15:06:29 2012 -0700 Parsing 'cell-circuits-per-decile' extrainfo lines Handling the last of the 'cell-*' attributes. --- stem/descriptor/extrainfo_descriptor.py | 17 +++++++++++++++ test/unit/descriptor/extrainfo_descriptor.py | 28 +++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 7fa008e..a08d389 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -195,6 +195,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): cell_processed_cells (list) - measurement of processed cells per circuit cell_queued_cells (list) - measurement of queued cells per circuit cell_time_in_queue (list) - mean enqueued time in milliseconds for cells + cell_circuits_per_decile (int) - mean number of circuits in a deciles Directory Mirror Attributes: dir_stats_end (datetime) - end of the period when stats were gathered @@ -286,6 +287,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.cell_processed_cells = None self.cell_queued_cells = None self.cell_time_in_queue = None + self.cell_circuits_per_decile = None self.dir_stats_end = None self.dir_stats_interval = None @@ -393,6 +395,21 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value + elif keyword == "cell-circuits-per-decile": + # "cell-circuits-per-decile" num + + if not value.isdigit(): + if validate: + raise ValueError("Non-numeric cell-circuits-per-decile value: %s" % line) + else: + continue + + stat = int(value) + + if validate and stat < 0: + raise ValueError("Negative cell-circuits-per-decile value: %s" % line) + + self.cell_circuits_per_decile = stat elif keyword in ("dirreq-v2-resp", "dirreq-v3-resp", "dirreq-v2-direct-dl", "dirreq-v3-direct-dl", "dirreq-v2-tunneled-dl", "dirreq-v3-tunneled-dl"): recognized_counts = {} unrecognized_counts = {} diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 00013c0..f856341 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -129,7 +129,33 @@ class TestExtraInfoDescriptor(unittest.TestCase): for entry in test_entries: desc_text = _make_descriptor({"geoip-db-digest": entry}) - desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) + self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) + + def test_cell_circuits_per_decile(self): + """ + Parses the cell-circuits-per-decile line with valid and invalid data. + """ + + test_entries = ( + ("0", 0), + ("11", 11), + ) + + for entry in ("0", "11", "25"): + desc_text = _make_descriptor({"cell-circuits-per-decile": entry}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(int(entry), desc.cell_circuits_per_decile) + + test_entries = ( + "", + " ", + "-5", + "blarg", + ) + + for entry in test_entries: + desc_text = _make_descriptor({"cell-circuits-per-decile": entry}) + self._expect_invalid_attr(desc_text, "cell_circuits_per_decile") def test_dir_response_lines(self): """

1 0

[stem/master] Parsing 'dirreq-v2-resp' and 'dirreq-v3-resp' lines
by atagar＠torproject.org 13 May '12

13 May '12

commit f7527756275e71b82d426394a74fc0f3ac9824b8 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 20:10:12 2012 -0700 Parsing 'dirreq-v2-resp' and 'dirreq-v3-resp' lines These lines were a bit more substantial to implement since callers would want enums for their statuses, but not all statuses are necessarily known. Looks like the 'dirreq-v*-direct-dl' and 'dirreq-v*-tunneled-dl' lines will be similar. --- stem/descriptor/extrainfo_descriptor.py | 71 +++++++++++++++++++++---- test/integ/descriptor/extrainfo_descriptor.py | 9 ++- test/integ/descriptor/server_descriptor.py | 3 +- test/unit/descriptor/extrainfo_descriptor.py | 33 +++++++++++- 4 files changed, 99 insertions(+), 17 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 9b01ffc..d1967a9 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -16,6 +16,14 @@ Extra-info descriptors are available from a few sources... - tor metrics, at https://metrics.torproject.org/data.html - directory authorities and mirrors via their DirPort +DirResponses - known statuses for ExtraInfoDescriptor's dir_*_responses + |- OK - network status requests that were answered + |- NOT_ENOUGH_SIGS - network status wasn't signed by enough authorities + |- UNAVAILABLE - requested network status was unavailable + |- NOT_FOUND - requested network status was not found + |- NOT_MODIFIED - network status unmodified since If-Modified-Since time + +- BUSY - directory was busy + parse_file - Iterates over the extra-info descriptors in a file. ExtraInfoDescriptor - Tor extra-info descriptor. +- get_unrecognized_lines - lines with unrecognized content @@ -25,6 +33,17 @@ import re import datetime import stem.descriptor +import stem.util.enum + +# known statuses for dirreq-v2-resp and dirreq-v3-resp... +DirResponses = stem.util.enum.Enum( + ("OK", "ok"), + ("NOT_ENOUGH_SIGS", "not-enough-sigs"), + ("UNAVAILABLE", "unavailable"), + ("NOT_FOUND", "not-found"), + ("NOT_MODIFIED", "not-modified"), + ("BUSY", "busy"), +) # relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( @@ -161,6 +180,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_v3_ips (dict) - mapping of locales to rounded count of requester ips dir_v2_requests (dict) - mapping of locales to rounded count of requests dir_v3_requests (dict) - mapping of locales to rounded count of requests + dir_v2_responses (dict) - mapping of DirResponses to their rounded count + dir_v3_responses (dict) - mapping of DirResponses to their rounded count + dir_v2_responses_unknown (dict) - mapping of unrecognized statuses to their count + dir_v3_responses_unknown (dict) - mapping of unrecognized statuses to their count dir_v2_share (float) - percent of total directory traffic it expects to serve dir_v3_share (float) - percent of total directory traffic it expects to serve @@ -225,6 +248,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_v3_ips = None self.dir_v2_requests = None self.dir_v3_requests = None + self.dir_v2_responses = None + self.dir_v3_responses = None + self.dir_v2_responses_unknown = None + self.dir_v3_responses_unknown = None self.dir_v2_share = None self.dir_v3_share = None @@ -309,6 +336,33 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value + elif keyword in ("dirreq-v2-resp", "dirreq-v3-resp"): + recognized_counts = {} + unrecognized_counts = {} + error_msg = "%s lines should contain STATUS=COUNT mappings: %s" % (keyword, line) + + if value: + for entry in value.split(","): + if not "=" in entry: + if validate: raise ValueError(error_msg) + else: continue + + status, count = entry.split("=", 1) + + if count.isdigit(): + if status in DirResponses: + recognized_counts[status] = int(count) + else: + unrecognized_counts[status] = int(count) + elif validate: + raise ValueError(error_msg) + + if keyword == "dirreq-v2-resp": + self.dir_v2_responses = recognized_counts + self.dir_v2_responses_unknown = unrecognized_counts + else: + self.dir_v3_responses = recognized_counts + self.dir_v3_responses_unknown = unrecognized_counts elif keyword in ("dirreq-v2-share", "dirreq-v3-share"): # "<keyword>" num% @@ -385,6 +439,12 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): if validate: raise exc elif keyword in ("dirreq-v2-ips", "dirreq-v3-ips", "dirreq-v2-reqs", "dirreq-v3-reqs", "geoip-client-origins", "bridge-ips"): # "<keyword>" CC=N,CC=N,... + # + # The maxmind geoip (https://www.maxmind.com/app/iso3166) has numeric + # locale codes for some special values, for instance... + # A1,"Anonymous Proxy" + # A2,"Satellite Provider" + # ??,"Unknown" locale_usage = {} error_msg = "Entries in %s line should only be CC=N entries: %s" % (keyword, line) @@ -395,23 +455,12 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): if validate: raise ValueError(error_msg) else: continue - # The maxmind geoip has numeric locale codes for some special - # values, for instance... - # - # A1,"Anonymous Proxy" - # A2,"Satellite Provider" - # ??,"Unknown" - # - # https://www.maxmind.com/app/iso3166 - - locale, count = entry.split("=", 1) if re.match("^[a-zA-Z0-9\?]{2}$", locale) and count.isdigit(): locale_usage[locale] = int(count) elif validate: raise ValueError(error_msg) - if keyword == "dirreq-v2-ips": self.dir_v2_ips = locale_usage elif keyword == "dirreq-v3-ips": diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 75b26e0..841aa54 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -83,13 +83,16 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw #unrecognized_lines = desc.get_unrecognized_lines() unrecognized_lines = [] - if unrecognized_lines: + if desc.dir_v2_responses_unknown: + self.fail("Unrecognized statuses on dirreq-v2-resp lines: %s" % desc.dir_v2_responses_unknown) + elif desc.dir_v3_responses_unknown: + self.fail("Unrecognized statuses on dirreq-v3-resp lines: %s" % desc.dir_v3_responses_unknown) + elif unrecognized_lines: # TODO: This isn't actually a problem, and rather than failing we # should alert the user about these entries at the end of the tests # (along with new events, getinfo options, and such). For now though # there doesn't seem to be anything in practice to trigger this so # failing to get our attention if it does. - print "Unrecognized descriptor content: %s" % unrecognized_lines - self.fail() + self.fail("Unrecognized descriptor content: %s" % unrecognized_lines) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 3c07364..b5664c4 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -172,8 +172,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= # there doesn't seem to be anything in practice to trigger this so # failing to get our attention if it does. - print "Unrecognized descriptor content: %s" % unrecognized_lines - self.fail() + self.fail("Unrecognized descriptor content: %s" % unrecognized_lines) def test_non_ascii_descriptor(self): """ diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index d8849c0..b167ad4 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -4,7 +4,7 @@ Unit tests for stem.descriptor.extrainfo_descriptor. import datetime import unittest -from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor +from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor, DirResponses CRYPTO_BLOB = """ K5FSywk7qvw/boA4DQcqkls6Ize5vcBYfhQ8JnOeRQC9+uDxbnpm3qaYN9jZ8myj @@ -131,6 +131,37 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc_text = _make_descriptor({"geoip-db-digest": entry}) desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) + def test_dir_response_lines(self): + """ + Parses the dirreq-v2-resp and dirreq-v3-resp lines with valid and invalid + data. + """ + + for keyword in ("dirreq-v2-resp", "dirreq-v3-resp"): + attr = keyword.replace('-', '_').replace('dirreq', 'dir').replace('resp', 'responses') + unknown_attr = attr + "_unknown" + + test_value = "ok=0,unavailable=0,not-found=984,not-modified=0,something-new=7" + desc_text = _make_descriptor({keyword: test_value}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(0, getattr(desc, attr)[DirResponses.OK]) + self.assertEquals(0, getattr(desc, attr)[DirResponses.UNAVAILABLE]) + self.assertEquals(984, getattr(desc, attr)[DirResponses.NOT_FOUND]) + self.assertEquals(0, getattr(desc, attr)[DirResponses.NOT_MODIFIED]) + self.assertEquals(7, getattr(desc, unknown_attr)["something-new"]) + + test_entries = ( + "ok=-4", + "ok:4", + "ok=4.not-found=3", + ) + + for entry in test_entries: + desc_text = _make_descriptor({keyword: entry}) + desc = self._expect_invalid_attr(desc_text) + self.assertEqual({}, getattr(desc, attr)) + self.assertEqual({}, getattr(desc, unknown_attr)) + def test_percentage_lines(self): """ Uses valid and invalid data to tests lines of the form...

1 0

[stem/master] Citing ticket about negative cell-processed-cells values
by atagar＠torproject.org 13 May '12

13 May '12

commit 97fad7b6f7b6113fdb3e5ca8925224901748a98c Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 15:46:58 2012 -0700 Citing ticket about negative cell-processed-cells values Comment to explain why we're accepting negative values, and citing the related ticket. --- stem/descriptor/extrainfo_descriptor.py | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index a08d389..1363579 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -480,6 +480,11 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): if value: for entry in value.split(","): try: + # TODO: The description of the values sound like they should + # always be positive, but this is not always the case in + # practice... + # https://trac.torproject.org/projects/tor/ticket/5849 + entries.append(float(entry)) except ValueError: if validate: raise ValueError("Non-numeric entry in %s listing: %s" % (keyword, line))

1 0

[stem/master] Parsing dirreq-v2-share and dirreq-v3-share lines
by atagar＠torproject.org 13 May '12

13 May '12

commit e0ad461440361736b8ce340786433fe6dfaf44f2 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 15:39:15 2012 -0700 Parsing dirreq-v2-share and dirreq-v3-share lines Couple more extrainfo descriptor fields. Ye gods there a lot. --- stem/descriptor/extrainfo_descriptor.py | 21 ++++++++++++++++ test/unit/descriptor/extrainfo_descriptor.py | 33 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 61e00dd..58e2220 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -161,6 +161,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_v3_ips (dict) - mapping of locales to rounded count of requester ips dir_v2_reqs (dict) - mapping of locales to rounded count of requests dir_v3_reqs (dict) - mapping of locales to rounded count of requests + dir_v2_share (float) - percent of total directory traffic it expects to serve + dir_v3_share (float) - percent of total directory traffic it expects to serve Bytes read/written for directory mirroring dir_read_history_end (datetime) - end of the sampling interval @@ -223,6 +225,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_v3_ips = None self.dir_v2_reqs = None self.dir_v3_reqs = None + self.dir_v2_share = None + self.dir_v3_share = None self.dir_read_history_end = None self.dir_read_history_interval = None @@ -305,6 +309,23 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value + elif keyword in ("dirreq-v2-share", "dirreq-v3-share"): + # "<keyword>" num% + + try: + if not value.endswith("%"): raise ValueError() + percentage = float(value[:-1]) / 100 + + if validate and (percentage > 1 or percentage < 0): + raise ValueError() + + if keyword == "dirreq-v2-share": + self.dir_v2_share = percentage + elif keyword == "dirreq-v3-share": + self.dir_v3_share = percentage + except ValueError, exc: + if validate: + raise ValueError("Value can't be parsed as a percentage: %s" % line) elif keyword in ("published", "geoip-start-time"): # "<keyword>" YYYY-MM-DD HH:MM:SS diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 9732d4d..24034be 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -131,6 +131,39 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc_text = _make_descriptor({"geoip-db-digest": entry}) desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) + def test_percentage_lines(self): + """ + Uses valid and invalid data to tests lines of the form... + "<keyword>" num% + """ + + for keyword in ('dirreq-v2-share', 'dirreq-v3-share'): + attr = keyword.replace('-', '_').replace('dirreq', 'dir') + + test_entries = ( + ("0.00%", 0.0), + ("0.01%", 0.0001), + ("50%", 0.5), + ("100.0%", 1.0), + ) + + for test_value, expected_value in test_entries: + desc_text = _make_descriptor({keyword: test_value}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(expected_value, getattr(desc, attr)) + + test_entries = ( + ("", None), + (" ", None), + ("100", None), + ("100.1%", 1.001), + ("-5%", -0.05), + ) + + for entry, expected in test_entries: + desc_text = _make_descriptor({keyword: entry}) + self._expect_invalid_attr(desc_text, attr, expected) + def test_timestamp_lines(self): """ Uses valid and invalid data to tests lines of the form...

1 0

[stem/master] Parsing some more cell-* extrainfo lines
by atagar＠torproject.org 13 May '12

13 May '12

commit 8dd9d118977366ef8a17ad08a39809cbee735df4 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 14:47:40 2012 -0700 Parsing some more cell-* extrainfo lines Handling the cell-processed-cells, cell-queued-cells, and cell-time-in-queue entries which are all numeric lists. I was a little surprised to find negative cell-processed-cells values in the wild. --- stem/descriptor/extrainfo_descriptor.py | 24 ++++++++++++++++++ test/unit/descriptor/extrainfo_descriptor.py | 33 +++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 64a33b0..7fa008e 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -192,6 +192,9 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): Cell relaying statistics: cell_stats_end (datetime) - end of the period when stats were gathered cell_stats_interval (int) - length in seconds of the interval + cell_processed_cells (list) - measurement of processed cells per circuit + cell_queued_cells (list) - measurement of queued cells per circuit + cell_time_in_queue (list) - mean enqueued time in milliseconds for cells Directory Mirror Attributes: dir_stats_end (datetime) - end of the period when stats were gathered @@ -280,6 +283,9 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.cell_stats_end = None self.cell_stats_interval = None + self.cell_processed_cells = None + self.cell_queued_cells = None + self.cell_time_in_queue = None self.dir_stats_end = None self.dir_stats_interval = None @@ -449,6 +455,24 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): except ValueError, exc: if validate: raise ValueError("Value can't be parsed as a percentage: %s" % line) + elif keyword in ("cell-processed-cells", "cell-queued-cells", "cell-time-in-queue"): + # "<keyword>" num,...,num + + entries = [] + + if value: + for entry in value.split(","): + try: + entries.append(float(entry)) + except ValueError: + if validate: raise ValueError("Non-numeric entry in %s listing: %s" % (keyword, line)) + + if keyword == "cell-processed-cells": + self.cell_processed_cells = entries + elif keyword == "cell-queued-cells": + self.cell_queued_cells = entries + elif keyword == "cell-time-in-queue": + self.cell_time_in_queue = entries elif keyword in ("published", "geoip-start-time"): # "<keyword>" YYYY-MM-DD HH:MM:SS diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 92f37bc..00013c0 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -238,6 +238,37 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc_text = _make_descriptor({keyword: entry}) self._expect_invalid_attr(desc_text, attr, expected) + def test_number_list_lines(self): + """ + Uses valid and invalid data to tests lines of the form... + "<keyword>" num,...,num + """ + + for keyword in ('cell-processed-cells', 'cell-queued-cells', 'cell-time-in-queue'): + attr = keyword.replace('-', '_') + + test_entries = ( + ("", []), + (" ", []), + ("0,0,0", [0.0, 0.0, 0.0]), + ("2.3,-4.6,8.9,16.12,32.15", [2.3, -4.6, 8.9, 16.12, 32.15]), + ) + + for test_value, expected_value in test_entries: + desc_text = _make_descriptor({keyword: test_value}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(expected_value, getattr(desc, attr)) + + test_entries = ( + (",,11", [11.0]), + ("abc,5.7,def", [5.7]), + ("blarg", []), + ) + + for entry, expected in test_entries: + desc_text = _make_descriptor({keyword: entry}) + self._expect_invalid_attr(desc_text, attr, expected) + def test_timestamp_lines(self): """ Uses valid and invalid data to tests lines of the form... @@ -364,7 +395,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): for entry in test_entries: desc_text = _make_descriptor({keyword: entry}) - desc = self._expect_invalid_attr(desc_text, attr, {}) + self._expect_invalid_attr(desc_text, attr, {}) def _expect_invalid_attr(self, desc_text, attr = None, expected_value = None): """

1 0