May 2012 - tor-commits - lists.torproject.org

[stem/master] Parsing 'cell-circuits-per-decile' extrainfo lines
by atagar＠torproject.org 14 May '12

14 May '12

commit 153a98eca741d3185e7d20c5e3566763a54384bd Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 15:06:29 2012 -0700 Parsing 'cell-circuits-per-decile' extrainfo lines Handling the last of the 'cell-*' attributes. --- stem/descriptor/extrainfo_descriptor.py | 17 +++++++++++++++ test/unit/descriptor/extrainfo_descriptor.py | 28 +++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 7fa008e..a08d389 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -195,6 +195,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): cell_processed_cells (list) - measurement of processed cells per circuit cell_queued_cells (list) - measurement of queued cells per circuit cell_time_in_queue (list) - mean enqueued time in milliseconds for cells + cell_circuits_per_decile (int) - mean number of circuits in a deciles Directory Mirror Attributes: dir_stats_end (datetime) - end of the period when stats were gathered @@ -286,6 +287,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.cell_processed_cells = None self.cell_queued_cells = None self.cell_time_in_queue = None + self.cell_circuits_per_decile = None self.dir_stats_end = None self.dir_stats_interval = None @@ -393,6 +395,21 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value + elif keyword == "cell-circuits-per-decile": + # "cell-circuits-per-decile" num + + if not value.isdigit(): + if validate: + raise ValueError("Non-numeric cell-circuits-per-decile value: %s" % line) + else: + continue + + stat = int(value) + + if validate and stat < 0: + raise ValueError("Negative cell-circuits-per-decile value: %s" % line) + + self.cell_circuits_per_decile = stat elif keyword in ("dirreq-v2-resp", "dirreq-v3-resp", "dirreq-v2-direct-dl", "dirreq-v3-direct-dl", "dirreq-v2-tunneled-dl", "dirreq-v3-tunneled-dl"): recognized_counts = {} unrecognized_counts = {} diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 00013c0..f856341 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -129,7 +129,33 @@ class TestExtraInfoDescriptor(unittest.TestCase): for entry in test_entries: desc_text = _make_descriptor({"geoip-db-digest": entry}) - desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) + self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) + + def test_cell_circuits_per_decile(self): + """ + Parses the cell-circuits-per-decile line with valid and invalid data. + """ + + test_entries = ( + ("0", 0), + ("11", 11), + ) + + for entry in ("0", "11", "25"): + desc_text = _make_descriptor({"cell-circuits-per-decile": entry}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(int(entry), desc.cell_circuits_per_decile) + + test_entries = ( + "", + " ", + "-5", + "blarg", + ) + + for entry in test_entries: + desc_text = _make_descriptor({"cell-circuits-per-decile": entry}) + self._expect_invalid_attr(desc_text, "cell_circuits_per_decile") def test_dir_response_lines(self): """

1 0

[stem/master] Parsing 'dirreq-v2-resp' and 'dirreq-v3-resp' lines
by atagar＠torproject.org 14 May '12

14 May '12

commit f7527756275e71b82d426394a74fc0f3ac9824b8 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 20:10:12 2012 -0700 Parsing 'dirreq-v2-resp' and 'dirreq-v3-resp' lines These lines were a bit more substantial to implement since callers would want enums for their statuses, but not all statuses are necessarily known. Looks like the 'dirreq-v*-direct-dl' and 'dirreq-v*-tunneled-dl' lines will be similar. --- stem/descriptor/extrainfo_descriptor.py | 71 +++++++++++++++++++++---- test/integ/descriptor/extrainfo_descriptor.py | 9 ++- test/integ/descriptor/server_descriptor.py | 3 +- test/unit/descriptor/extrainfo_descriptor.py | 33 +++++++++++- 4 files changed, 99 insertions(+), 17 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 9b01ffc..d1967a9 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -16,6 +16,14 @@ Extra-info descriptors are available from a few sources... - tor metrics, at https://metrics.torproject.org/data.html - directory authorities and mirrors via their DirPort +DirResponses - known statuses for ExtraInfoDescriptor's dir_*_responses + |- OK - network status requests that were answered + |- NOT_ENOUGH_SIGS - network status wasn't signed by enough authorities + |- UNAVAILABLE - requested network status was unavailable + |- NOT_FOUND - requested network status was not found + |- NOT_MODIFIED - network status unmodified since If-Modified-Since time + +- BUSY - directory was busy + parse_file - Iterates over the extra-info descriptors in a file. ExtraInfoDescriptor - Tor extra-info descriptor. +- get_unrecognized_lines - lines with unrecognized content @@ -25,6 +33,17 @@ import re import datetime import stem.descriptor +import stem.util.enum + +# known statuses for dirreq-v2-resp and dirreq-v3-resp... +DirResponses = stem.util.enum.Enum( + ("OK", "ok"), + ("NOT_ENOUGH_SIGS", "not-enough-sigs"), + ("UNAVAILABLE", "unavailable"), + ("NOT_FOUND", "not-found"), + ("NOT_MODIFIED", "not-modified"), + ("BUSY", "busy"), +) # relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( @@ -161,6 +180,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_v3_ips (dict) - mapping of locales to rounded count of requester ips dir_v2_requests (dict) - mapping of locales to rounded count of requests dir_v3_requests (dict) - mapping of locales to rounded count of requests + dir_v2_responses (dict) - mapping of DirResponses to their rounded count + dir_v3_responses (dict) - mapping of DirResponses to their rounded count + dir_v2_responses_unknown (dict) - mapping of unrecognized statuses to their count + dir_v3_responses_unknown (dict) - mapping of unrecognized statuses to their count dir_v2_share (float) - percent of total directory traffic it expects to serve dir_v3_share (float) - percent of total directory traffic it expects to serve @@ -225,6 +248,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_v3_ips = None self.dir_v2_requests = None self.dir_v3_requests = None + self.dir_v2_responses = None + self.dir_v3_responses = None + self.dir_v2_responses_unknown = None + self.dir_v3_responses_unknown = None self.dir_v2_share = None self.dir_v3_share = None @@ -309,6 +336,33 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value + elif keyword in ("dirreq-v2-resp", "dirreq-v3-resp"): + recognized_counts = {} + unrecognized_counts = {} + error_msg = "%s lines should contain STATUS=COUNT mappings: %s" % (keyword, line) + + if value: + for entry in value.split(","): + if not "=" in entry: + if validate: raise ValueError(error_msg) + else: continue + + status, count = entry.split("=", 1) + + if count.isdigit(): + if status in DirResponses: + recognized_counts[status] = int(count) + else: + unrecognized_counts[status] = int(count) + elif validate: + raise ValueError(error_msg) + + if keyword == "dirreq-v2-resp": + self.dir_v2_responses = recognized_counts + self.dir_v2_responses_unknown = unrecognized_counts + else: + self.dir_v3_responses = recognized_counts + self.dir_v3_responses_unknown = unrecognized_counts elif keyword in ("dirreq-v2-share", "dirreq-v3-share"): # "<keyword>" num% @@ -385,6 +439,12 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): if validate: raise exc elif keyword in ("dirreq-v2-ips", "dirreq-v3-ips", "dirreq-v2-reqs", "dirreq-v3-reqs", "geoip-client-origins", "bridge-ips"): # "<keyword>" CC=N,CC=N,... + # + # The maxmind geoip (https://www.maxmind.com/app/iso3166) has numeric + # locale codes for some special values, for instance... + # A1,"Anonymous Proxy" + # A2,"Satellite Provider" + # ??,"Unknown" locale_usage = {} error_msg = "Entries in %s line should only be CC=N entries: %s" % (keyword, line) @@ -395,23 +455,12 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): if validate: raise ValueError(error_msg) else: continue - # The maxmind geoip has numeric locale codes for some special - # values, for instance... - # - # A1,"Anonymous Proxy" - # A2,"Satellite Provider" - # ??,"Unknown" - # - # https://www.maxmind.com/app/iso3166 - - locale, count = entry.split("=", 1) if re.match("^[a-zA-Z0-9\?]{2}$", locale) and count.isdigit(): locale_usage[locale] = int(count) elif validate: raise ValueError(error_msg) - if keyword == "dirreq-v2-ips": self.dir_v2_ips = locale_usage elif keyword == "dirreq-v3-ips": diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 75b26e0..841aa54 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -83,13 +83,16 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw #unrecognized_lines = desc.get_unrecognized_lines() unrecognized_lines = [] - if unrecognized_lines: + if desc.dir_v2_responses_unknown: + self.fail("Unrecognized statuses on dirreq-v2-resp lines: %s" % desc.dir_v2_responses_unknown) + elif desc.dir_v3_responses_unknown: + self.fail("Unrecognized statuses on dirreq-v3-resp lines: %s" % desc.dir_v3_responses_unknown) + elif unrecognized_lines: # TODO: This isn't actually a problem, and rather than failing we # should alert the user about these entries at the end of the tests # (along with new events, getinfo options, and such). For now though # there doesn't seem to be anything in practice to trigger this so # failing to get our attention if it does. - print "Unrecognized descriptor content: %s" % unrecognized_lines - self.fail() + self.fail("Unrecognized descriptor content: %s" % unrecognized_lines) diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 3c07364..b5664c4 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -172,8 +172,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= # there doesn't seem to be anything in practice to trigger this so # failing to get our attention if it does. - print "Unrecognized descriptor content: %s" % unrecognized_lines - self.fail() + self.fail("Unrecognized descriptor content: %s" % unrecognized_lines) def test_non_ascii_descriptor(self): """ diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index d8849c0..b167ad4 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -4,7 +4,7 @@ Unit tests for stem.descriptor.extrainfo_descriptor. import datetime import unittest -from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor +from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor, DirResponses CRYPTO_BLOB = """ K5FSywk7qvw/boA4DQcqkls6Ize5vcBYfhQ8JnOeRQC9+uDxbnpm3qaYN9jZ8myj @@ -131,6 +131,37 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc_text = _make_descriptor({"geoip-db-digest": entry}) desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) + def test_dir_response_lines(self): + """ + Parses the dirreq-v2-resp and dirreq-v3-resp lines with valid and invalid + data. + """ + + for keyword in ("dirreq-v2-resp", "dirreq-v3-resp"): + attr = keyword.replace('-', '_').replace('dirreq', 'dir').replace('resp', 'responses') + unknown_attr = attr + "_unknown" + + test_value = "ok=0,unavailable=0,not-found=984,not-modified=0,something-new=7" + desc_text = _make_descriptor({keyword: test_value}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(0, getattr(desc, attr)[DirResponses.OK]) + self.assertEquals(0, getattr(desc, attr)[DirResponses.UNAVAILABLE]) + self.assertEquals(984, getattr(desc, attr)[DirResponses.NOT_FOUND]) + self.assertEquals(0, getattr(desc, attr)[DirResponses.NOT_MODIFIED]) + self.assertEquals(7, getattr(desc, unknown_attr)["something-new"]) + + test_entries = ( + "ok=-4", + "ok:4", + "ok=4.not-found=3", + ) + + for entry in test_entries: + desc_text = _make_descriptor({keyword: entry}) + desc = self._expect_invalid_attr(desc_text) + self.assertEqual({}, getattr(desc, attr)) + self.assertEqual({}, getattr(desc, unknown_attr)) + def test_percentage_lines(self): """ Uses valid and invalid data to tests lines of the form...

1 0

[stem/master] Citing ticket about negative cell-processed-cells values
by atagar＠torproject.org 14 May '12

14 May '12

commit 97fad7b6f7b6113fdb3e5ca8925224901748a98c Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 15:46:58 2012 -0700 Citing ticket about negative cell-processed-cells values Comment to explain why we're accepting negative values, and citing the related ticket. --- stem/descriptor/extrainfo_descriptor.py | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index a08d389..1363579 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -480,6 +480,11 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): if value: for entry in value.split(","): try: + # TODO: The description of the values sound like they should + # always be positive, but this is not always the case in + # practice... + # https://trac.torproject.org/projects/tor/ticket/5849 + entries.append(float(entry)) except ValueError: if validate: raise ValueError("Non-numeric entry in %s listing: %s" % (keyword, line))

1 0

[stem/master] Parsing dirreq-v2-share and dirreq-v3-share lines
by atagar＠torproject.org 14 May '12

14 May '12

commit e0ad461440361736b8ce340786433fe6dfaf44f2 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 12 15:39:15 2012 -0700 Parsing dirreq-v2-share and dirreq-v3-share lines Couple more extrainfo descriptor fields. Ye gods there a lot. --- stem/descriptor/extrainfo_descriptor.py | 21 ++++++++++++++++ test/unit/descriptor/extrainfo_descriptor.py | 33 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 61e00dd..58e2220 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -161,6 +161,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_v3_ips (dict) - mapping of locales to rounded count of requester ips dir_v2_reqs (dict) - mapping of locales to rounded count of requests dir_v3_reqs (dict) - mapping of locales to rounded count of requests + dir_v2_share (float) - percent of total directory traffic it expects to serve + dir_v3_share (float) - percent of total directory traffic it expects to serve Bytes read/written for directory mirroring dir_read_history_end (datetime) - end of the sampling interval @@ -223,6 +225,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_v3_ips = None self.dir_v2_reqs = None self.dir_v3_reqs = None + self.dir_v2_share = None + self.dir_v3_share = None self.dir_read_history_end = None self.dir_read_history_interval = None @@ -305,6 +309,23 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line) self.geoip_db_digest = value + elif keyword in ("dirreq-v2-share", "dirreq-v3-share"): + # "<keyword>" num% + + try: + if not value.endswith("%"): raise ValueError() + percentage = float(value[:-1]) / 100 + + if validate and (percentage > 1 or percentage < 0): + raise ValueError() + + if keyword == "dirreq-v2-share": + self.dir_v2_share = percentage + elif keyword == "dirreq-v3-share": + self.dir_v3_share = percentage + except ValueError, exc: + if validate: + raise ValueError("Value can't be parsed as a percentage: %s" % line) elif keyword in ("published", "geoip-start-time"): # "<keyword>" YYYY-MM-DD HH:MM:SS diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 9732d4d..24034be 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -131,6 +131,39 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc_text = _make_descriptor({"geoip-db-digest": entry}) desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry) + def test_percentage_lines(self): + """ + Uses valid and invalid data to tests lines of the form... + "<keyword>" num% + """ + + for keyword in ('dirreq-v2-share', 'dirreq-v3-share'): + attr = keyword.replace('-', '_').replace('dirreq', 'dir') + + test_entries = ( + ("0.00%", 0.0), + ("0.01%", 0.0001), + ("50%", 0.5), + ("100.0%", 1.0), + ) + + for test_value, expected_value in test_entries: + desc_text = _make_descriptor({keyword: test_value}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(expected_value, getattr(desc, attr)) + + test_entries = ( + ("", None), + (" ", None), + ("100", None), + ("100.1%", 1.001), + ("-5%", -0.05), + ) + + for entry, expected in test_entries: + desc_text = _make_descriptor({keyword: entry}) + self._expect_invalid_attr(desc_text, attr, expected) + def test_timestamp_lines(self): """ Uses valid and invalid data to tests lines of the form...

1 0

[stem/master] Parsing some more cell-* extrainfo lines
by atagar＠torproject.org 14 May '12

14 May '12

commit 8dd9d118977366ef8a17ad08a39809cbee735df4 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 14:47:40 2012 -0700 Parsing some more cell-* extrainfo lines Handling the cell-processed-cells, cell-queued-cells, and cell-time-in-queue entries which are all numeric lists. I was a little surprised to find negative cell-processed-cells values in the wild. --- stem/descriptor/extrainfo_descriptor.py | 24 ++++++++++++++++++ test/unit/descriptor/extrainfo_descriptor.py | 33 +++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 64a33b0..7fa008e 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -192,6 +192,9 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): Cell relaying statistics: cell_stats_end (datetime) - end of the period when stats were gathered cell_stats_interval (int) - length in seconds of the interval + cell_processed_cells (list) - measurement of processed cells per circuit + cell_queued_cells (list) - measurement of queued cells per circuit + cell_time_in_queue (list) - mean enqueued time in milliseconds for cells Directory Mirror Attributes: dir_stats_end (datetime) - end of the period when stats were gathered @@ -280,6 +283,9 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.cell_stats_end = None self.cell_stats_interval = None + self.cell_processed_cells = None + self.cell_queued_cells = None + self.cell_time_in_queue = None self.dir_stats_end = None self.dir_stats_interval = None @@ -449,6 +455,24 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): except ValueError, exc: if validate: raise ValueError("Value can't be parsed as a percentage: %s" % line) + elif keyword in ("cell-processed-cells", "cell-queued-cells", "cell-time-in-queue"): + # "<keyword>" num,...,num + + entries = [] + + if value: + for entry in value.split(","): + try: + entries.append(float(entry)) + except ValueError: + if validate: raise ValueError("Non-numeric entry in %s listing: %s" % (keyword, line)) + + if keyword == "cell-processed-cells": + self.cell_processed_cells = entries + elif keyword == "cell-queued-cells": + self.cell_queued_cells = entries + elif keyword == "cell-time-in-queue": + self.cell_time_in_queue = entries elif keyword in ("published", "geoip-start-time"): # "<keyword>" YYYY-MM-DD HH:MM:SS diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 92f37bc..00013c0 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -238,6 +238,37 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc_text = _make_descriptor({keyword: entry}) self._expect_invalid_attr(desc_text, attr, expected) + def test_number_list_lines(self): + """ + Uses valid and invalid data to tests lines of the form... + "<keyword>" num,...,num + """ + + for keyword in ('cell-processed-cells', 'cell-queued-cells', 'cell-time-in-queue'): + attr = keyword.replace('-', '_') + + test_entries = ( + ("", []), + (" ", []), + ("0,0,0", [0.0, 0.0, 0.0]), + ("2.3,-4.6,8.9,16.12,32.15", [2.3, -4.6, 8.9, 16.12, 32.15]), + ) + + for test_value, expected_value in test_entries: + desc_text = _make_descriptor({keyword: test_value}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(expected_value, getattr(desc, attr)) + + test_entries = ( + (",,11", [11.0]), + ("abc,5.7,def", [5.7]), + ("blarg", []), + ) + + for entry, expected in test_entries: + desc_text = _make_descriptor({keyword: entry}) + self._expect_invalid_attr(desc_text, attr, expected) + def test_timestamp_lines(self): """ Uses valid and invalid data to tests lines of the form... @@ -364,7 +395,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): for entry in test_entries: desc_text = _make_descriptor({keyword: entry}) - desc = self._expect_invalid_attr(desc_text, attr, {}) + self._expect_invalid_attr(desc_text, attr, {}) def _expect_invalid_attr(self, desc_text, attr = None, expected_value = None): """

1 0

[stem/master] Parsing conn-bi-direct extrainfo lines
by atagar＠torproject.org 14 May '12

14 May '12

commit c7ea62037adef91fe11728caab75e19096c69d52 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 16:24:37 2012 -0700 Parsing conn-bi-direct extrainfo lines Another special snowflake extrainfo attribute. Parsing and added a test. --- stem/descriptor/extrainfo_descriptor.py | 34 +++++++++++++++++++++++ test/unit/descriptor/extrainfo_descriptor.py | 38 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 1363579..2b6997c 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -180,6 +180,13 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): geoip_db_digest (str) - sha1 of geoIP database file signature (str) - signature for this extrainfo descriptor (*) + conn_bi_direct_end (datetime) - end of the sampling interval + conn_bi_direct_interval (int) - seconds per interval + conn_bi_direct_below (int) - connections that read/wrote less than 20 KiB + conn_bi_direct_read (int) - connections that read at least 10x more than wrote + conn_bi_direct_write (int) - connections that wrote at least 10x more than read + conn_bi_direct_both (int) - remaining connections + Bytes read/written for relayed traffic: read_history_end (datetime) - end of the sampling interval read_history_interval (int) - seconds per interval @@ -274,6 +281,13 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.geoip_db_digest = None self.signature = None + self.conn_bi_direct_end = None + self.conn_bi_direct_interval = None + self.conn_bi_direct_below = None + self.conn_bi_direct_read = None + self.conn_bi_direct_write = None + self.conn_bi_direct_both = None + self.read_history_end = None self.read_history_interval = None self.read_history_values = None @@ -528,6 +542,26 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_stats_interval = interval except ValueError, exc: if validate: raise exc + elif keyword == "conn-bi-direct": + # "conn-bi-direct" YYYY-MM-DD HH:MM:SS (NSEC s) BELOW,READ,WRITE,BOTH + + try: + timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value) + stats = remainder.split(",") + + if len(stats) != 4 or not \ + (stats[0].isdigit() and stats[1].isdigit() and \ + stats[2].isdigit() and stats[3].isdigit()): + raise ValueError("conn-bi-direct line should end with four numeric values: %s" % line) + + self.conn_bi_direct_end = timestamp + self.conn_bi_direct_interval = interval + self.conn_bi_direct_below = int(stats[0]) + self.conn_bi_direct_read = int(stats[1]) + self.conn_bi_direct_write = int(stats[2]) + self.conn_bi_direct_both = int(stats[3]) + except ValueError, exc: + if validate: raise exc elif keyword in ("read-history", "write-history", "dirreq-read-history", "dirreq-write-history"): # "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM,NUM,NUM... try: diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index f856341..76a4344 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -231,6 +231,44 @@ class TestExtraInfoDescriptor(unittest.TestCase): self.assertEqual({}, getattr(desc, attr)) self.assertEqual({}, getattr(desc, unknown_attr)) + def test_conn_bi_direct(self): + """ + Parses the conn-bi-direct line with valid and invalid data. + """ + + desc_text = _make_descriptor({"conn-bi-direct": "2012-05-03 12:07:50 (500 s) 277431,12089,0,2134"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), desc.conn_bi_direct_end) + self.assertEquals(500, desc.conn_bi_direct_interval) + self.assertEquals(277431, desc.conn_bi_direct_below) + self.assertEquals(12089, desc.conn_bi_direct_read) + self.assertEquals(0, desc.conn_bi_direct_write) + self.assertEquals(2134, desc.conn_bi_direct_both) + + test_entries = ( + "", + "2012-05-03 ", + "2012-05-03", + "2012-05-03 12:07:60 (500 s)", + "2012-05-03 12:07:50 (500s)", + "2012-05-03 12:07:50 (500 s", + "2012-05-03 12:07:50 (500 )", + "2012-05-03 12:07:50 (500 s)11", + "2012-05-03 12:07:50 (500 s) 277431,12089,0", + "2012-05-03 12:07:50 (500 s) 277431,12089,0a,2134", + "2012-05-03 12:07:50 (500 s) -277431,12089,0,2134", + ) + + for entry in test_entries: + desc_text = _make_descriptor({"conn-bi-direct": entry}) + desc = self._expect_invalid_attr(desc_text) + self.assertEquals(None, desc.conn_bi_direct_end) + self.assertEquals(None, desc.conn_bi_direct_interval) + self.assertEquals(None, desc.conn_bi_direct_below) + self.assertEquals(None, desc.conn_bi_direct_read) + self.assertEquals(None, desc.conn_bi_direct_write) + self.assertEquals(None, desc.conn_bi_direct_both) + def test_percentage_lines(self): """ Uses valid and invalid data to tests lines of the form...

1 0

[stem/master] Parsing cell-stats-end lines
by atagar＠torproject.org 14 May '12

14 May '12

commit 00e7ce52acc2ddb8f45ccae29469d3463ba39010 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 14:13:57 2012 -0700 Parsing cell-stats-end lines Not much to this attribute. The next few are slightly different, though. --- stem/descriptor/extrainfo_descriptor.py | 18 ++++++++++++++---- test/unit/descriptor/extrainfo_descriptor.py | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 7008272..64a33b0 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -180,7 +180,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): geoip_db_digest (str) - sha1 of geoIP database file signature (str) - signature for this extrainfo descriptor (*) - Bytes read/written for relayed traffic + Bytes read/written for relayed traffic: read_history_end (datetime) - end of the sampling interval read_history_interval (int) - seconds per interval read_history_values (list) - bytes read during each interval @@ -189,6 +189,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): write_history_interval (int) - seconds per interval write_history_values (list) - bytes written during each interval + Cell relaying statistics: + cell_stats_end (datetime) - end of the period when stats were gathered + cell_stats_interval (int) - length in seconds of the interval + Directory Mirror Attributes: dir_stats_end (datetime) - end of the period when stats were gathered dir_stats_interval (int) - length in seconds of the interval @@ -214,7 +218,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_v2_tunneled_dl_unknown (dict) - mapping of unrecognized stats to their measurement dir_v3_tunneled_dl_unknown (dict) - mapping of unrecognized stats to their measurement - Bytes read/written for directory mirroring + Bytes read/written for directory mirroring: dir_read_history_end (datetime) - end of the sampling interval dir_read_history_interval (int) - seconds per interval dir_read_history_values (list) - bytes read during each interval @@ -274,6 +278,9 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.write_history_interval = None self.write_history_values = None + self.cell_stats_end = None + self.cell_stats_interval = None + self.dir_stats_end = None self.dir_stats_interval = None self.dir_v2_ips = None @@ -455,13 +462,16 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): except ValueError: if validate: raise ValueError("Timestamp on %s line wasn't parseable: %s" % (keyword, line)) - elif keyword in ("entry-stats-end", "bridge-stats-end", "dirreq-stats-end"): + elif keyword in ("cell-stats-end", "entry-stats-end", "bridge-stats-end", "dirreq-stats-end"): # "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) try: timestamp, interval, _ = _parse_timestamp_and_interval(keyword, value) - if keyword == "entry-stats-end": + if keyword == "cell-stats-end": + self.cell_stats_end = timestamp + self.cell_stats_interval = interval + elif keyword == "entry-stats-end": self.entry_stats_end = timestamp self.entry_stats_interval = interval elif keyword == "bridge-stats-end": diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 1ea19da..92f37bc 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -268,7 +268,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) """ - for keyword in ('entry-stats-end', 'bridge-stats-end', 'dirreq-stats-end'): + for keyword in ('cell-stats-end', 'entry-stats-end', 'bridge-stats-end', 'dirreq-stats-end'): end_attr = keyword.replace('-', '_').replace('dirreq', 'dir') interval_attr = end_attr[:-4] + "_interval"

1 0

[stem/master] Parsing entry-stats-end and entry-ips lines
by atagar＠torproject.org 14 May '12

14 May '12

commit 86bce1e8db0799efbca86af72a4ef3c0a27f9674 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 13:46:49 2012 -0700 Parsing entry-stats-end and entry-ips lines Handling a couple more extrainfo attributes. These were trivial thanks to earlier refactoring to make common value types easy to parse. --- stem/descriptor/extrainfo_descriptor.py | 20 +++++++++++++++++--- test/unit/descriptor/extrainfo_descriptor.py | 4 ++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 9493f2b..7008272 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -223,6 +223,11 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_write_history_interval (int) - seconds per interval dir_write_history_values (list) - bytes read during each interval + Guard Attributes: + entry_stats_end (datetime) - end of the period when stats were gathered + entry_stats_interval (int) - length in seconds of the interval + entry_ips (dict) - mapping of locales to rounded count of unique user ips + Bridge Attributes: bridge_stats_end (datetime) - end of the period when stats were gathered bridge_stats_interval (int) - length in seconds of the interval @@ -298,6 +303,10 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_write_history_interval = None self.dir_write_history_values = None + self.entry_stats_end = None + self.entry_stats_interval = None + self.entry_ips = None + self.bridge_stats_end = None self.bridge_stats_interval = None self.bridge_ips = None @@ -446,13 +455,16 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): except ValueError: if validate: raise ValueError("Timestamp on %s line wasn't parseable: %s" % (keyword, line)) - elif keyword in ("bridge-stats-end", "dirreq-stats-end"): + elif keyword in ("entry-stats-end", "bridge-stats-end", "dirreq-stats-end"): # "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) try: timestamp, interval, _ = _parse_timestamp_and_interval(keyword, value) - if keyword == "bridge-stats-end": + if keyword == "entry-stats-end": + self.entry_stats_end = timestamp + self.entry_stats_interval = interval + elif keyword == "bridge-stats-end": self.bridge_stats_end = timestamp self.bridge_stats_interval = interval elif keyword == "dirreq-stats-end": @@ -490,7 +502,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_write_history_values = history_values except ValueError, exc: if validate: raise exc - elif keyword in ("dirreq-v2-ips", "dirreq-v3-ips", "dirreq-v2-reqs", "dirreq-v3-reqs", "geoip-client-origins", "bridge-ips"): + elif keyword in ("dirreq-v2-ips", "dirreq-v3-ips", "dirreq-v2-reqs", "dirreq-v3-reqs", "geoip-client-origins", "entry-ips", "bridge-ips"): # "<keyword>" CC=N,CC=N,... # # The maxmind geoip (https://www.maxmind.com/app/iso3166) has numeric @@ -524,6 +536,8 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_v3_requests = locale_usage elif keyword == "geoip-client-origins": self.geoip_client_origins = locale_usage + elif keyword == "entry-ips": + self.entry_ips = locale_usage elif keyword == "bridge-ips": self.bridge_ips = locale_usage elif keyword == "router-signature": diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index e855a1d..1ea19da 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -268,7 +268,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) """ - for keyword in ('bridge-stats-end', 'dirreq-stats-end'): + for keyword in ('entry-stats-end', 'bridge-stats-end', 'dirreq-stats-end'): end_attr = keyword.replace('-', '_').replace('dirreq', 'dir') interval_attr = end_attr[:-4] + "_interval" @@ -342,7 +342,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): "<keyword>" CC=N,CC=N,... """ - for keyword in ('dirreq-v2-ips', 'dirreq-v3-ips', 'dirreq-v2-reqs', 'dirreq-v3-reqs', 'geoip-client-origins', 'bridge-ips'): + for keyword in ('dirreq-v2-ips', 'dirreq-v3-ips', 'dirreq-v2-reqs', 'dirreq-v3-reqs', 'geoip-client-origins', 'entry-ips', 'bridge-ips'): attr = keyword.replace('-', '_').replace('dirreq', 'dir').replace('reqs', 'requests') test_entries = (

1 0

[stem/master] Unused constructor annotations arg
by atagar＠torproject.org 14 May '12

14 May '12

commit 7d96e78e90e455019f674eff2a611449107eb109 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 17:02:28 2012 -0700 Unused constructor annotations arg Cached extrainfo descriptors don't have annotations, so excluding it from the constructor. Functionally it was already unused, just forgot to remove it from the args. Caught by pylint. --- stem/descriptor/extrainfo_descriptor.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index f0f3ee2..1c00d34 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -262,7 +262,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): default value, others are left as None if undefined """ - def __init__(self, raw_contents, validate = True, annotations = None): + def __init__(self, raw_contents, validate = True): """ Extra-info descriptor constructor, created from a relay's extra-info content (as provided by "GETINFO extra-info/digest/*", cached contents, and

1 0

[stem/master] Missing util import
by atagar＠torproject.org 14 May '12

14 May '12

commit 5bbf40dbc9a7374f555b4ec7aaf838be6156c9fc Author: Damian Johnson <atagar(a)torproject.org> Date: Sun May 13 16:57:53 2012 -0700 Missing util import This should have caused an error, but for some mysterious reason imports throughout stem have a wider scope than I'm intending (causing pylint to compain about lots of re-imports). Definitely an error - we don't want to rely on whatever buggy magic made it work. --- stem/descriptor/extrainfo_descriptor.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 9f71c3c..f0f3ee2 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -44,6 +44,7 @@ import datetime import stem.descriptor import stem.util.enum +import stem.util.connection # known statuses for dirreq-v2-resp and dirreq-v3-resp... DirResponses = stem.util.enum.Enum(

1 0