commit c33c4a0f0e6b98898622d0929e531d12abdce72a Author: Damian Johnson atagar@torproject.org Date: Thu May 10 09:29:04 2012 -0700

Parsing bridge specific extra-info params

Parsing, validation, and tests for bridge specific lines... - geoip-start-time - geoip-client-origins - bridge-stats-end - bridge-stats-ips

I haven't yet seen an actual bridge descriptor, and I'd like to add one as an integ test but that can come later. --- stem/descriptor/extrainfo_descriptor.py | 58 ++++++++++++++++++- stem/descriptor/server_descriptor.py | 3 +- test/unit/descriptor/extrainfo_descriptor.py | 81 ++++++++++++++++++++++++-- 3 files changed, 132 insertions(+), 10 deletions(-)

diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index d2ee780..380d022 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -165,6 +165,16 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): dir_write_history_interval (int) - seconds per interval dir_write_history_values (list) - bytes read during each interval (*)

+ Bridge-only Attributes: + bridge_stats_end (datetime.datetime) - end of the period when geoip + statistics were gathered + bridge_stats_end_interval (int) - length in seconds of th interval where + stats were gathered + bridge_ips (dict) - mapping of country codes to a rounded number of unique + ips from that region + geoip_start_time (datetime.datetime) - replaced by bridge_stats_end + geoip_client_origins (dict) - replaced by bridge_ips + (*) required fields, others are left as None if undefined """

@@ -215,6 +225,12 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): self.dir_write_history_interval = None self.dir_write_history_values = []

+ self.bridge_stats_end = None + self.bridge_stats_end_interval = None + self.bridge_ips = None + self.geoip_start_time = None + self.geoip_client_origins = None + self._unrecognized_lines = []

entries, first_keyword, last_keyword, _ = \ @@ -290,6 +306,45 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): raise ValueError("Geoip digest line had an invalid sha1 digest: %s" % line)

self.geoip_db_digest = value + elif keyword == "geoip-start-time": + # "geoip-start-time" YYYY-MM-DD HH:MM:SS + + try: + self.geoip_start_time = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + except ValueError: + if validate: + raise ValueError("Geoip start time line's time wasn't parseable: %s" % line) + elif keyword == "bridge-stats-end": + # "bridge-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) + + try: + timestamp, interval, _ = _parse_timestamp_and_interval(keyword, value) + self.bridge_stats_end = timestamp + self.bridge_stats_end_interval = interval + except ValueError, exc: + if validate: raise exc + elif keyword in ("geoip-client-origins", "bridge-ips"): + # "geoip-client-origins" CC=N,CC=N,... + + locale_usage = {} + error_msg = "Entries in %s line should only be CC=N entries: %s" % (keyword, line) + + for entry in value.split(","): + if not "=" in entry: + if validate: raise ValueError(error_msg) + else: continue + + locale, count = entry.split("=", 1) + + if re.match("^[a-zA-Z]{2}$", locale) and count.isdigit(): + locale_usage[locale] = int(count) + elif validate: + raise ValueError(error_msg) + + if keyword == "geoip-client-origins": + self.geoip_client_origins = locale_usage + elif keyword == "bridge-ips": + self.bridge_ips = locale_usage elif keyword in ("read-history", "write-history", "dirreq-read-history", "dirreq-write-history"): try: timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value) @@ -327,8 +382,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): # without fixing this one raise ValueError("BUG: unrecognized keyword '%s'" % keyword) except ValueError, exc: - if not validate: continue - else: raise exc + if validate: raise exc elif keyword == "router-signature": if validate and not block_contents: raise ValueError("Router signature line must be followed by a signature block: %s" % line) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 80f1538..2963a53 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -491,8 +491,7 @@ class ServerDescriptor(stem.descriptor.Descriptor): self.write_history_interval = interval self.write_history_values = history_values except ValueError, exc: - if not validate: continue - else: raise exc + if validate: raise exc else: self._unrecognized_lines.append(line)

diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 5a4a450..32732e3 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -2,6 +2,7 @@ Unit tests for stem.descriptor.extrainfo_descriptor. """

+import datetime import unittest from stem.descriptor.extrainfo_descriptor import ExtraInfoDescriptor

@@ -110,18 +111,13 @@ class TestExtraInfoDescriptor(unittest.TestCase):

def test_geoip_db_digest(self): """ - Parses a geoip-db-digest line with valid data. + Parses the geoip-db-digest line with valid and invalid data. """

geoip_db_digest = "916A3CA8B7DF61473D5AE5B21711F35F301CE9E8" desc_text = _make_descriptor({"geoip-db-digest": geoip_db_digest}) desc = ExtraInfoDescriptor(desc_text) self.assertEquals(geoip_db_digest, desc.geoip_db_digest) - - def test_geoip_db_digest_invalid(self): - """ - Parses the geoip-db-digest line with a variety of bad input. - """

test_entry = ( "", @@ -135,6 +131,79 @@ class TestExtraInfoDescriptor(unittest.TestCase): desc_text = _make_descriptor({"geoip-db-digest": entry}) desc = self._expect_invalid_attr(desc_text, "geoip_db_digest", entry)

+ def test_geoip_start_time(self): + """ + Parses the geoip-start-time line with valid and invalid data. + """ + + desc_text = _make_descriptor({"geoip-start-time": "2012-05-03 12:07:50"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), desc.geoip_start_time) + + test_entry = ( + "", + "2012-05-03 12:07:60", + "2012-05-03 ", + "2012-05-03", + ) + + for entry in test_entry: + desc_text = _make_descriptor({"geoip-start-time": entry}) + desc = self._expect_invalid_attr(desc_text, "geoip_start_time") + + def test_bridge_stats_end(self): + """ + Parses the bridge-stats-end line with valid and invalid data. + """ + + desc_text = _make_descriptor({"bridge-stats-end": "2012-05-03 12:07:50 (500 s)"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals(datetime.datetime(2012, 5, 3, 12, 7, 50), desc.bridge_stats_end) + self.assertEquals(500, desc.bridge_stats_end_interval) + + test_entry = ( + "", + "2012-05-03 12:07:60 (500 s)", + "2012-05-03 12:07:50 (500s)", + "2012-05-03 12:07:50 (500 s", + "2012-05-03 12:07:50 (500 )", + "2012-05-03 ", + "2012-05-03", + ) + + for entry in test_entry: + desc_text = _make_descriptor({"bridge-stats-end": entry}) + desc = self._expect_invalid_attr(desc_text, "bridge_stats_end") + + def test_bridge_ips(self): + """ + Parses both the bridge-ips and geoip-client-origins lines with valid and + invalid data. + """ + + # Testing both attributes since they contain the exact same data, + # geoip-client-origins was simply replaced by bridge-ips while adding an + # interval value for the period. + + for keyword in ('bridge-ips', 'geoip-client-origins'): + attr = keyword.replace('-', '_') + + desc_text = _make_descriptor({keyword: "uk=5,de=3,jp=2"}) + desc = ExtraInfoDescriptor(desc_text) + self.assertEquals({'uk': 5, 'de': 3, 'jp': 2}, getattr(desc, attr)) + + test_entry = ( + "", + "uk=-4", + "uki=4", + "uk:4", + "uk=4.de=3", + ) + + for entry in test_entry: + desc_text = _make_descriptor({keyword: entry}) + desc = self._expect_invalid_attr(desc_text, attr, {}) + def _expect_invalid_attr(self, desc_text, attr = None, expected_value = None): """ Asserts that construction will fail due to desc_text having a malformed