- tor-commits - lists.torproject.org

[stem/master] Adding a test for descriptor content from 2005
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 96678f167f210d10f65c14d977e9f764c4d70dbd Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 14 17:09:37 2012 -0700 Adding a test for descriptor content from 2005 Version 2 relay server descriptors were first introduced in 2005 so adding a descriptor from back then. This in theory doesn't exercise anything that the unit tests don't already do, but it still provides some good use cases (mostly around deprecated attributes). I ran over the full December 2005 metrics tarball and all descriptors were happily parsed. --- test/integ/descriptor/data/old_descriptor | 47 +++++++++++++++++++++++ test/integ/descriptor/server_descriptor.py | 56 ++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 0 deletions(-) diff --git a/test/integ/descriptor/data/old_descriptor b/test/integ/descriptor/data/old_descriptor new file mode 100644 index 0000000..5407aa8 --- /dev/null +++ b/test/integ/descriptor/data/old_descriptor @@ -0,0 +1,47 @@ +router krypton 212.37.39.59 8000 0 0 +platform Tor 0.1.0.14 on FreeBSD i386 +published 2005-12-16 18:01:03 +opt fingerprint 3E2F 63E2 356F 5231 8B53 6A12 B644 5373 808A 5D6C +uptime 64820 +bandwidth 102400 10485760 0 +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAOewVPvehUE/Iy06e/NmUyzwBdkL89DP3q+I68hJJsxQgd1Fqm2CVN4E +X0l3U14e3yyihr8xz1qmvQenWyKMx6B5ZjHrRAk774nTXzqqZMbcEKGnoWKTAgfC +SrZYD4YdO2vPIA3OIBLBvj6HlyZWPVulQANpETK1TOACKeqs6vlhAgMBAAE= +-----END RSA PUBLIC KEY----- +signing-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAJjsGNwSkG7xCzWEPTr2NBrC4k2Bx0SnTv8RfgUwqRg8rLss5CoZw7BV +GL3g6PNe13rWcu7rYg27mT81T4cehrl0omnx4FrhDPmnJY4d+yjUiLKujTYaijyZ +8ll4Mr1Ua3M2tTa+79sA8rXAg3zMf+BtTfIcK4avm+csipZ75r9BAgMBAAE= +-----END RSA PUBLIC KEY----- +opt write-history 2005-12-16 18:00:48 (900 s) 81,8848,8927,8927,83,8848,8931,8929,81,8846,8929,8931,81,8848,8929,8927,81,8846,8929,8929,83,8848,8931,8927,9544,23145,118733,12471330,23681225,32974322,36669050,50028191,60119352,43959405,104125876,93764839,67886465,56660787,69614721,49677518,72758978,69080350,68451562,56135957,66072248,44186182,63136414,78723340,84085493,73800911,61322609,54493047,40058835,35991325,33933884,43033805,30004913,31375439,29870384,39901793,24405790,46428634,32855040,31738054,32096150,34571772,47317095,43861120,1725009,2709813,3632379,7206137,4376659,3499688,2148089,2420819,3039400,3271931,2661625,2387027,943794,138137,167325,152659,120259,119828,120342,116812,118734,117279,116769,114764,115673,117709,117201,110668 +opt read-history 2005-12-16 18:00:48 (900 s) 20774,489973,510022,511163,20949,490665,509936,510266,20919,490429,512036,508640,20666,485937,506582,505071,20662,484684,507452,506131,20565,485822,505385,505766,28300,499433,627237,12392493,20716078,25654673,25415889,39581836,49991512,31545049,96363762,80390334,60930455,45885095,57137777,42643192,59698768,54573481,58379825,45862920,58781990,33448161,53507694,73363721,70825862,61283831,49325261,44674036,28451429,25009272,22895835,34914097,21103340,19156546,21131703,33036587,18445151,30464430,21857164,27329112,22507170,29536769,37525461,42026081,1778746,3200903,4137862,7712377,4397719,3980947,2650315,2440799,3522111,3774977,3165517,2407670,1427206,644034,672441,173113,609713,630447,632529,136977,611086,629716,629248,135385,607471,627380,625011,131334 +opt hibernating 1 +reject 169.254.0.0/255.255.0.0:* +reject 127.0.0.0/255.0.0.0:* +reject 192.168.0.0/255.255.0.0:* +reject 10.0.0.0/255.0.0.0:* +reject 172.16.0.0/255.240.0.0:* +accept *:20-22 +accept *:53 +accept *:79-81 +accept *:110 +accept *:143 +accept *:443 +accept *:706 +accept *:873 +accept *:993 +accept *:995 +accept *:6660-6669 +accept *:8008 +accept *:8080 +accept *:8888 +reject *:* +router-signature +-----BEGIN SIGNATURE----- +mHTlJGu2d2ZZgXfoI0CZBiLMCKbHox2n+Q3OGcivLj0kcYfJ/7/jk4o5ABRgyOHM +zJ7FnC2wSg19MB8jeAvKkxdd1hp0OZQp+tCRq/7p11+6E0qvJcio+RwdeZAb0dQf +suUOPIU/lHzfmraZMO3ovK4VjiWKQ7PgE1vAH7sTyZA= +-----END SIGNATURE----- diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index f1af13c..b974098 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -105,6 +105,62 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals([], desc.get_unrecognized_lines()) self.assertEquals("LHsnvqsEtOJFnYnKbVzRzF+Vpok", desc.digest()) + def test_old_descriptor(self): + """ + Parses a relay server descriptor from 2005. + """ + + descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "old_descriptor") + + descriptor_file = open(descriptor_path) + descriptor_contents = descriptor_file.read() + descriptor_file.close() + + desc = stem.descriptor.server_descriptor.RelayDescriptorV3(descriptor_contents) + self.assertEquals("krypton", desc.nickname) + self.assertEquals("3E2F63E2356F52318B536A12B6445373808A5D6C", desc.fingerprint) + self.assertEquals("212.37.39.59", desc.address) + self.assertEquals(8000, desc.or_port) + self.assertEquals(0, desc.socks_port) + self.assertEquals(0, desc.dir_port) + self.assertEquals("Tor 0.1.0.14 on FreeBSD i386", desc.platform) + self.assertEquals(stem.version.Version("0.1.0.14"), desc.tor_version) + self.assertEquals("FreeBSD i386", desc.operating_system) + self.assertEquals(64820, desc.uptime) + self.assertEquals(datetime.datetime(2005, 12, 16, 18, 1, 3), desc.published) + self.assertEquals(None, desc.contact) + self.assertEquals(None, desc.link_protocols) + self.assertEquals(None, desc.circuit_protocols) + self.assertEquals(True, desc.hibernating) + self.assertEquals(False, desc.allow_single_hop_exits) + self.assertEquals(False, desc.extra_info_cache) + self.assertEquals(None, desc.extra_info_digest) + self.assertEquals(None, desc.hidden_service_dir) + self.assertEquals([], desc.family) + self.assertEquals(102400, desc.average_bandwidth) + self.assertEquals(10485760, desc.burst_bandwidth) + self.assertEquals(0, desc.observed_bandwidth) + self.assertEquals(datetime.datetime(2005, 12, 16, 18, 0, 48), desc.read_history_end) + self.assertEquals(900, desc.read_history_interval) + self.assertEquals(datetime.datetime(2005, 12, 16, 18, 0, 48), desc.write_history_end) + self.assertEquals(900, desc.write_history_interval) + self.assertEquals([], desc.get_unrecognized_lines()) + + # The read-history and write-history lines are pretty long so just checking + # the initial contents for the line and parsed values. + + read_start = "2005-12-16 18:00:48 (900 s) 20774,489973,510022" + self.assertTrue(desc.read_history.startswith(read_start)) + + read_values_start = [20774, 489973, 510022, 511163, 20949] + self.assertEquals(read_values_start, desc.read_history_values[:5]) + + write_start = "2005-12-16 18:00:48 (900 s) 81,8848,8927,8927" + self.assertTrue(desc.write_history.startswith(write_start)) + + write_values_start = [81, 8848, 8927, 8927, 83, 8848, 8931, 8929, 81, 8846] + self.assertEquals(write_values_start, desc.write_history_values[:10]) + def test_cached_descriptor(self): """ Parses the cached descriptor file in our data directory, checking that it

1 0

[stem/master] Handling descriptor files in a depth first fashion
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 9f76969739eccf740da0f77378bcabc5672a85bb Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 14 18:30:40 2012 -0700 Handling descriptor files in a depth first fashion Directories enqueued all of the files that it contained prior to processing them which has a couple obvious disadvantages... - huge targets like the root directory or years worth of descriptors can consume lots of memory with the paths alone - this could easily cause us to have a huge startup time before we provided the caller any descriptors This was stupid, depth first parsing makes much more sense. --- stem/descriptor/reader.py | 55 +++++++++++++++++++++++--------------------- 1 files changed, 29 insertions(+), 26 deletions(-) diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 66ed65a..01f8809 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -327,36 +327,12 @@ class DescriptorReader: # adds all of the files that it contains for root, _, files in os.walk(target, followlinks = self._follow_links): for filename in files: - remaining_files.append(os.path.join(root, filename)) + self._handle_file(os.path.join(root, filename), new_processed_files) # this can take a while if, say, we're including the root directory if self._is_stopped.is_set(): break else: - # This is a file. Register its last modified timestamp and check if - # it's a file that we should skip. - - last_modified = int(os.stat(target).st_mtime) - last_used = self._processed_files.get(target) - new_processed_files[target] = last_modified - - if last_used and last_used >= last_modified: - self._notify_skip_listeners(target, AlreadyRead(last_modified, last_used)) - continue - - # The mimetypes module only checks the file extension. To actually - # check the content (like the 'file' command) we'd need something like - # pymagic (https://github.com/cloudburst/pymagic) - - target_type = mimetypes.guess_type(target) - - if target_type[0] in (None, 'text/plain'): - # either '.txt' or an unknown type - self._handle_descriptor_file(target) - elif tarfile.is_tarfile(target): - # handles gzip, bz2, and decompressed tarballs among others - self._handle_archive(target) - else: - self._notify_skip_listeners(target, UnrecognizedType(target_type)) + self._handle_file(target, new_processed_files) self._processed_files = new_processed_files @@ -377,6 +353,33 @@ class DescriptorReader: self._iter_notice.wait() self._iter_notice.clear() + def _handle_file(self, target, new_processed_files): + # This is a file. Register its last modified timestamp and check if + # it's a file that we should skip. + + last_modified = int(os.stat(target).st_mtime) + last_used = self._processed_files.get(target) + new_processed_files[target] = last_modified + + if last_used and last_used >= last_modified: + self._notify_skip_listeners(target, AlreadyRead(last_modified, last_used)) + return + + # The mimetypes module only checks the file extension. To actually + # check the content (like the 'file' command) we'd need something like + # pymagic (https://github.com/cloudburst/pymagic) + + target_type = mimetypes.guess_type(target) + + if target_type[0] in (None, 'text/plain'): + # either '.txt' or an unknown type + self._handle_descriptor_file(target) + elif tarfile.is_tarfile(target): + # handles gzip, bz2, and decompressed tarballs among others + self._handle_archive(target) + else: + self._notify_skip_listeners(target, UnrecognizedType(target_type)) + def _handle_descriptor_file(self, target): try: with open(target) as target_file:

1 0

[stem/master] Expanding attributes for read/write-history
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 62f34c67bcaaa7205b09cc7cf6201d78447c0a39 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 14 15:51:16 2012 -0700 Expanding attributes for read/write-history Parsing and validating read-history and write-history attributes. These are depricated fields, but still found on archived and extra-info descriptors. This includes a unit test to exercise the additions. --- stem/descriptor/server_descriptor.py | 64 +++++++++++++++++++++++++--- test/unit/descriptor/server_descriptor.py | 27 ++++++++++++ 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 9b74e2b..6892381 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -202,10 +202,18 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): average_bandwidth (int) - rate of traffic relay is willing to relay in bytes/s (*) burst_bandwidth (int) - rate of traffic relay is willing to burst to in bytes/s (*) observed_bandwidth (int) - estimated capacity of the relay based on usage in bytes/s (*) - read_history (str) - (deprecated) always unset - write_history (str) - (deprecated) always unset eventdns (bool) - (deprecated) always unset (*) + read_history (str) - (deprecated) read-history line, always unset + read_history_end (datetime.datetime) - (deprecated) end of the sampling interval + read_history_interval (int) - (deprecated) seconds per interval + read_history_values (list) - (deprecated) bytes read during each interval (*) + + write_history (str) - (deprecated) write-history line, always unset + write_history_end (datetime.datetime) - (deprecated) end of the sampling interval + write_history_interval (int) - (deprecated) seconds per interval + write_history_values (list) - (deprecated) bytes written during each interval (*) + (*) required fields, others are left as None if undefined """ @@ -254,9 +262,18 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): self.average_bandwidth = None self.burst_bandwidth = None self.observed_bandwidth = None + self.eventdns = True + self.read_history = None + self.read_history_end = None + self.read_history_interval = None + self.read_history_values = [] + self.write_history = None - self.eventdns = True + self.write_history_end = None + self.write_history_interval = None + self.write_history_values = [] + self._unrecognized_lines = [] self._annotation_lines = annotations if annotations else [] @@ -474,12 +491,45 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): raise ValueError("Protocols line did not match the expected pattern: %s" % line) elif keyword == "family": self.family = value.split(" ") - elif keyword == "read-history": - self.read_history = value - elif keyword == "write-history": - self.write_history = value elif keyword == "eventdns": self.eventdns = value == "1" + elif keyword in ("read-history", "write-history"): + is_read = keyword == "read-history" + + if is_read: self.read_history = value + else: self.write_history = value + + value_match = re.match("^(.*) $([0-9]+) s$ (.*)$", value) + + if not value_match: + if not validate: continue + raise ValueError("Malformed %s line: %s" % (keyword, line)) + + end_value, interval_value, history_values = value_match.groups() + + try: + end_datetime = datetime.datetime.strptime(end_value, "%Y-%m-%d %H:%M:%S") + + if is_read: self.read_history_end = end_datetime + else: self.write_history_end = end_datetime + except ValueError: + if validate: + raise ValueError("%s line's time wasn't parseable: %s" % (keyword, line)) + + if interval_value.isdigit(): + if is_read: self.read_history_interval = int(interval_value) + else: self.write_history_interval = int(interval_value) + elif validate: + raise ValueError("%s line's interval wasn't a number: %s" % (keyword, line)) + + for sampling in history_values.split(","): + if sampling.isdigit(): + if is_read: self.read_history_values.append(int(sampling)) + else: self.write_history_values.append(int(sampling)) + else: + if validate: + raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) + else: break else: self._unrecognized_lines.append(line) diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 89a28c6..5c4cfce 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -222,6 +222,33 @@ class TestServerDescriptor(unittest.TestCase): desc_text = _make_descriptor({"published": "2012-01-01"}) self._expect_invalid_attr(desc_text, "published") + def test_read_and_write_history(self): + """ + Parses a read-history and write-history entry. This is now a depricated + field for relay server descriptors but is still found in archives and + extra-info descriptors. + """ + + for field in ("read-history", "write-history"): + value = "2005-12-16 18:00:48 (900 s) 81,8848,8927,8927,83,8848" + desc_text = _make_descriptor({"opt %s" % field: value}) + desc = RelayDescriptorV3(desc_text) + + if field == "read-history": + attr = (desc.read_history, desc.read_history_end, + desc.read_history_interval, desc.read_history_values) + else: + attr = (desc.write_history, desc.write_history_end, + desc.write_history_interval, desc.write_history_values) + + expected_end = datetime.datetime(2005, 12, 16, 18, 0, 48) + expected_values = [81, 8848, 8927, 8927, 83, 8848] + + self.assertEquals(value, attr[0]) + self.assertEquals(expected_end, attr[1]) + self.assertEquals(900, attr[2]) + self.assertEquals(expected_values, attr[3]) + def test_annotations(self): """ Checks that content before a descriptor are parsed as annotations.

1 0

[stem/master] Handling empty read/write-history value listings
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 707897bea765b3d403dea9eaf4a3a0788a455bd0 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 14 17:23:03 2012 -0700 Handling empty read/write-history value listings Bug where relay server descriptors with read/write-history lines but no values on them would fail validation. Added a unit test for this and fixed the bug. --- stem/descriptor/server_descriptor.py | 17 +++++++++-------- test/unit/descriptor/server_descriptor.py | 13 +++++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 6892381..780e7b5 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -522,14 +522,15 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): elif validate: raise ValueError("%s line's interval wasn't a number: %s" % (keyword, line)) - for sampling in history_values.split(","): - if sampling.isdigit(): - if is_read: self.read_history_values.append(int(sampling)) - else: self.write_history_values.append(int(sampling)) - else: - if validate: - raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) - else: break + if history_values != '': + for sampling in history_values.split(","): + if sampling.isdigit(): + if is_read: self.read_history_values.append(int(sampling)) + else: self.write_history_values.append(int(sampling)) + else: + if validate: + raise ValueError("%s line has non-numeric values: %s" % (keyword, line)) + else: break else: self._unrecognized_lines.append(line) diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 5c4cfce..d170e96 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -249,6 +249,19 @@ class TestServerDescriptor(unittest.TestCase): self.assertEquals(900, attr[2]) self.assertEquals(expected_values, attr[3]) + def test_read_history_empty(self): + """ + Parses a read-history with an empty value. + """ + + value = "2005-12-17 01:23:11 (900 s) " + desc_text = _make_descriptor({"opt read-history": value}) + desc = RelayDescriptorV3(desc_text) + self.assertEquals(value, desc.read_history) + self.assertEquals(datetime.datetime(2005, 12, 17, 1, 23, 11), desc.read_history_end) + self.assertEquals(900, desc.read_history_interval) + self.assertEquals([], desc.read_history_values) + def test_annotations(self): """ Checks that content before a descriptor are parsed as annotations.

1 0

[stem/master] Merge server descriptor changes suggested by Karsten
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit ac3f6271e0896cbdb2e668576f43670b260bfa2b Merge: 809b341 9f76969 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 14 19:12:01 2012 -0700 Merge server descriptor changes suggested by Karsten stem/descriptor/reader.py | 89 ++-- stem/descriptor/server_descriptor.py | 663 +++++++++++++++++------ stem/util/connection.py | 40 ++- test/integ/descriptor/data/bridge_descriptor | 13 + test/integ/descriptor/data/non-ascii_descriptor | 29 + test/integ/descriptor/data/old_descriptor | 47 ++ test/integ/descriptor/reader.py | 10 +- test/integ/descriptor/server_descriptor.py | 164 ++++++- test/unit/descriptor/server_descriptor.py | 181 ++++++- test/unit/util/connection.py | 28 + 10 files changed, 1016 insertions(+), 248 deletions(-)

1 0

[stem/master] Making descriptor reader skip tests more resilient
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 04176384995a89555067ddecf4cf6a7fe882ce39 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 14 17:03:28 2012 -0700 Making descriptor reader skip tests more resilient Couple problems with the skip tests for the descriptor reader: - A bug in the test for skipping unmodified files caused the test to rely on the order in which descriptors were read. It had a check that should have caught this, but rather than "assertEqual(1, len(skipped_files))" it had "assertTrue(1, len(skipped_files))". Fixed the check and narrowed the test to just the single descriptor file that it was supposed to test against. - The test which checks that we properly skip and report non-descriptor content would fail if other files are added to the data directory. This is as it should be, but the error message wasn't helpful and I've encountered this a lot due to vim '.swp' files. --- test/integ/descriptor/reader.py | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index 1444dbb..5c84704 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -302,7 +302,11 @@ class TestDescriptorReader(unittest.TestCase): self.assertTrue(2, len(skip_listener.results)) for skip_path, skip_exception in skip_listener.results: - self.assertTrue(os.path.basename(skip_path) in ("riddle", "tiny.png")) + if skip_path.endswith(".swp"): continue # skip vim temp files + + if not os.path.basename(skip_path) in ("riddle", "tiny.png"): + self.fail("Unexpected non-descriptor content: %s" % skip_path) + self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType)) def test_skip_listener_already_read(self): @@ -317,14 +321,14 @@ class TestDescriptorReader(unittest.TestCase): initial_processed_files = {test_path: sys.maxint} skip_listener = SkipListener() - reader = stem.descriptor.reader.DescriptorReader([DESCRIPTOR_TEST_DATA]) + reader = stem.descriptor.reader.DescriptorReader([test_path]) reader.register_skip_listener(skip_listener.listener) reader.set_processed_files(initial_processed_files) self.assertEquals(initial_processed_files, reader.get_processed_files()) with reader: list(reader) # iterates over all of the descriptors - self.assertTrue(1, len(skip_listener.results)) + self.assertEquals(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path)

1 0

[stem/master] Test for descriptor with non-ascii content
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit dd6b71c2b4fe27c33f82ba45e405be9c3fabd409 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Mar 29 08:54:22 2012 -0700 Test for descriptor with non-ascii content Test suggestion by Karsten. This reads a descriptor with non-ascii content (hiragana and something else), checking that the results match expected values. --- test/integ/descriptor/data/non-ascii_descriptor | 29 ++++++++++++++++ test/integ/descriptor/server_descriptor.py | 41 +++++++++++++++++++++++ 2 files changed, 70 insertions(+), 0 deletions(-) diff --git a/test/integ/descriptor/data/non-ascii_descriptor b/test/integ/descriptor/data/non-ascii_descriptor new file mode 100644 index 0000000..7758276 --- /dev/null +++ b/test/integ/descriptor/data/non-ascii_descriptor @@ -0,0 +1,29 @@ +router torrelay389752132 130.243.230.116 9001 0 0 +platform Tor 0.2.2.35 (git-4f42b0a93422f70e) on Linux x86_64 +opt protocols Link 1 2 Circuit 1 +published 2012-03-21 16:28:14 +opt fingerprint FEBC 7F99 2AC4 18BB E42B C13F E94E FCFE 6549 197E +uptime 3103848 +bandwidth 81920 102400 84275 +opt extra-info-digest 51E9FD0DA7C235D8C0250BAFB6E1ABB5F1EF9F04 +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAKTHzdEAocB7JhpB5iDaNYgHujJ+0mtoahZOAqgZ5BWVjw8fwt0UXEtO +BRPzNigNyAP1UWKHvgelBy49wvuC9twoBejCCYjcu9z2RmbBvYcyZRzphvHJqBpW +k3Rx75up+wsuBzhfwSYr7W+T+WkDQvz49RFPpns6Ef0qFpQ1TlHxAgMBAAE= +-----END RSA PUBLIC KEY----- +signing-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAMSmtutGlXVdvh/IC4TyhQpgSajxrZItC2lS5/70Vr4uLevryPlBgVrW +35CHxKYaj0MAOfkJQ0/OvTaXe7hlaCLrDDXScaH/XEDurcWrynsdzomsCvn/6VJ+ +xZFszt2Dn5myXKMvYy3j1oevC4iDaZXwxgpwx/UMJsFn7GOUPFYbAgMBAAE= +-----END RSA PUBLIC KEY----- +opt hidden-service-dir +contact 2048R/F171EC1F Johan Blåbäck こんにちは +reject *:* +router-signature +-----BEGIN SIGNATURE----- +q3Tw41+miuycnXowX/k6CCOcHMjw0BCDjW56Wh/eHoICmVb/hBJdtuzTaorWHLWp +OoTa4Sy4OrGFL+ldzajGC8+oqMvrYudiIxbJWmH3NXFyd7ZeEdnHzHxNOU8p1+X+ +hFwdOCEvzvvTbOuS2DwDt+TU8rljZunZfcMWgXktAD0= +-----END SIGNATURE----- diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index f950c4a..8cf0ae4 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -130,4 +130,45 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= print "Unrecognized descriptor content: %s" % unrecognized_lines self.fail() + + def test_non_ascii_descriptor(self): + """ + Parses a descriptor with non-ascii content. + """ + + descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "non-ascii_descriptor") + + descriptor_file = open(descriptor_path) + descriptor_contents = descriptor_file.read() + descriptor_file.close() + + expected_published = datetime.datetime(2012, 3, 21, 16, 28, 14) + expected_contact = "2048R/F171EC1F Johan Bl\xc3\xa5b\xc3\xa4ck \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf" + + desc = stem.descriptor.server_descriptor.ServerDescriptorV3(descriptor_contents) + self.assertEquals("torrelay389752132", desc.nickname) + self.assertEquals("FEBC7F992AC418BBE42BC13FE94EFCFE6549197E", desc.fingerprint) + self.assertEquals("130.243.230.116", desc.address) + self.assertEquals(9001, desc.or_port) + self.assertEquals(0, desc.socks_port) + self.assertEquals(0, desc.dir_port) + self.assertEquals("Tor 0.2.2.35 (git-4f42b0a93422f70e) on Linux x86_64", desc.platform) + self.assertEquals(stem.version.Version("0.2.2.35"), desc.tor_version) + self.assertEquals("Linux x86_64", desc.operating_system) + self.assertEquals(3103848, desc.uptime) + self.assertEquals(expected_published, desc.published) + self.assertEquals(expected_contact, desc.contact) + self.assertEquals(["1", "2"], desc.link_protocols) + self.assertEquals(["1"], desc.circuit_protocols) + self.assertEquals(False, desc.hibernating) + self.assertEquals(False, desc.allow_single_hop_exits) + self.assertEquals(False, desc.extra_info_cache) + self.assertEquals("51E9FD0DA7C235D8C0250BAFB6E1ABB5F1EF9F04", desc.extra_info_digest) + self.assertEquals(["2"], desc.hidden_service_dir) + self.assertEquals([], desc.family) + self.assertEquals(81920, desc.average_bandwidth) + self.assertEquals(102400, desc.burst_bandwidth) + self.assertEquals(84275, desc.observed_bandwidth) + self.assertEquals(["reject *:*"], desc.exit_policy) + self.assertEquals([], desc.get_unrecognized_lines())

1 0

[stem/master] Splitting relay and bridge server descritors
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 0f0eac50b59e33d5a25b428ddb9b0d16be023a28 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Apr 9 09:11:13 2012 -0700 Splitting relay and bridge server descritors Making separate classes for relay and bridge v3 server descriptors. This is the more proper object model and allows us to easily handle their differences in terms of parsing and validation. This includes the implementation and an integ test, but the implementation isn't yet complete (missing or-address parsing) and we still need unit tests. The current unit tests pass but the integ test fails due to or-address. --- stem/descriptor/server_descriptor.py | 470 +++++++++++++++++--------- test/integ/descriptor/data/bridge_descriptor | 13 + test/integ/descriptor/server_descriptor.py | 50 +++- test/unit/descriptor/server_descriptor.py | 22 +- 4 files changed, 383 insertions(+), 172 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 3460740..6895e5b 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -9,10 +9,14 @@ etc). This information is provided from a few sources... parse_file_v3 - Iterates over the server descriptors in a file. ServerDescriptorV3 - Tor server descriptor, version 3. + | |- RelayDescriptorV3 - Server descriptor for a relay. + | | +- is_valid - checks the signature against the descriptor content + | | + | +- BridgeDescriptorV3 - Scrubbed server descriptor for a bridge. + | |- get_unrecognized_lines - lines with unrecognized content |- get_annotations - dictionary of content prior to the descriptor entry - |- get_annotation_lines - lines that provided the annotations - +- is_valid - checks the signature against the descriptor content + +- get_annotation_lines - lines that provided the annotations """ import re @@ -24,16 +28,13 @@ import stem.util.connection import stem.util.tor_tools import stem.util.log as log -ENTRY_START = "router" -ENTRY_END = "router-signature" - KEYWORD_CHAR = "a-zA-Z0-9-" WHITESPACE = " \t" KEYWORD_LINE = re.compile("^([%s]+)[%s]*(.*)$" % (KEYWORD_CHAR, WHITESPACE)) PGP_BLOCK_START = re.compile("^-----BEGIN ([%s%s]+)-----$" % (KEYWORD_CHAR, WHITESPACE)) PGP_BLOCK_END = "-----END %s-----" -# entries must have exactly one of the following +# relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( "router", "bandwidth", @@ -71,7 +72,7 @@ def parse_file_v3(descriptor_file, validate = True): True, skips these checks otherwise Returns: - iterator for ServerDescriptorV3 instances in the file + iterator for RelayDescriptorV3 instances in the file Raises: ValueError if the contents is malformed and validate is True @@ -94,17 +95,17 @@ def parse_file_v3(descriptor_file, validate = True): # Metrics descriptor files are the same, but lack any annotations. The # following simply does the following... # - # - parse as annotations until we get to ENTRY_START - # - parse as descriptor content until we get to ENTRY_END followed by the - # end of the signature block + # - parse as annotations until we get to "router" + # - parse as descriptor content until we get to "router-signature" followed + # by the end of the signature block # - construct a descriptor and provide it back to the caller # # Any annotations after the last server descriptor is ignored (never provided # to the caller). while True: - annotations = _read_until_keyword(ENTRY_START, descriptor_file) - descriptor_content = _read_until_keyword(ENTRY_END, descriptor_file) + annotations = _read_until_keyword("router", descriptor_file) + descriptor_content = _read_until_keyword("router-signature", descriptor_file) # we've reached the 'router-signature', now include the pgp style block block_end_prefix = PGP_BLOCK_END.split(' ', 1)[0] @@ -115,7 +116,7 @@ def parse_file_v3(descriptor_file, validate = True): annotations = map(str.strip, annotations) descriptor_text = "".join(descriptor_content) - descriptor = ServerDescriptorV3(descriptor_text, validate, annotations) + descriptor = RelayDescriptorV3(descriptor_text, validate, annotations) yield descriptor else: break # done parsing descriptors @@ -153,47 +154,9 @@ def _read_until_keyword(keyword, descriptor_file, inclusive = False): return content -def _get_pseudo_pgp_block(remaining_contents): - """ - Checks if given contents begins with a pseudo-Open-PGP-style block and, if - so, pops it off and provides it back to the caller. - - Arguments: - remaining_contents (list) - lines to be checked for a public key block - - Returns: - str with the armor wrapped contents or None if it doesn't exist - - Raises: - ValueError if the contents starts with a key block but it's malformed (for - instance, if it lacks an ending line) - """ - - if not remaining_contents: - return None # nothing left - - block_match = PGP_BLOCK_START.match(remaining_contents[0]) - - if block_match: - block_type = block_match.groups()[0] - block_lines = [] - - while True: - if not remaining_contents: - raise ValueError("Unterminated pgp style block") - - line = remaining_contents.pop(0) - block_lines.append(line) - - if line == PGP_BLOCK_END % block_type: - return "\n".join(block_lines) - else: - return None - class ServerDescriptorV3(stem.descriptor.Descriptor): """ - Version 3 server descriptor, as specified in... - https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt + Common parent for version 3 server descriptors. Attributes: nickname (str) - relay's nickname (*) @@ -201,7 +164,7 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): address (str) - IPv4 address of the relay (*) or_port (int) - port used for relaying (*) socks_port (int) - (deprecated) always zero (*) - dir_port (int) - deprecated port used for descriptor mirroring (*) + dir_port (int) - port used for descriptor mirroring (*) platform (str) - operating system and tor version tor_version (stem.version.Version) - version of tor operating_system (str) - relay's operating system @@ -223,14 +186,11 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): read_history (str) - (deprecated) always unset write_history (str) - (deprecated) always unset eventdns (bool) - (deprecated) always unset (*) - onion_key (str) - key used to encrypt EXTEND cells (*) - signing_key (str) - relay's long-term identity key (*) - signature (str) - signature for this descriptor (*) (*) required fields, others are left as None if undefined """ - def __init__(self, contents, validate = True, annotations = None): + def __init__(self, raw_contents, annotations): """ Version 3 server descriptor constructor, created from an individual relay's descriptor content (as provided by "GETINFO desc/*", cached descriptors, @@ -241,7 +201,7 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): malformed data. Arguments: - contents (str) - descriptor content provided by the relay + raw_contents (str) - descriptor content provided by the relay validate (bool) - checks the validity of the descriptor's content if True, skips these checks otherwise annotations (list) - lines that appeared prior to the descriptor @@ -250,7 +210,7 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): ValueError if the contents is malformed and validate is True """ - stem.descriptor.Descriptor.__init__(self, contents) + stem.descriptor.Descriptor.__init__(self, raw_contents) self.nickname = None self.fingerprint = None @@ -278,9 +238,6 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): self.read_history = None self.write_history = None self.eventdns = True - self.onion_key = None - self.signing_key = None - self.signature = None # TODO: Until we have a proper ExitPolicy class this is just a list of the # exit policy strings... @@ -301,70 +258,50 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): else: self._annotation_lines = [] self._annotation_dict = {} + + def get_unrecognized_lines(self): + return list(self._unrecognized_lines) + + def get_annotations(self): + """ + Provides content that appeard prior to the descriptor. If this comes from + the cached-descriptors file then this commonly contains content like... + + @downloaded-at 2012-03-18 21:18:29 + @source "173.254.216.66" + + Returns: + dict with the key/value pairs in our annotations + """ + + return self._annotation_dict + + def get_annotation_lines(self): + """ + Provides the lines of content that appeared prior to the descriptor. This + is the same as the get_annotations() results, but with the unparsed lines + and ordering retained. + + Returns: + list with the lines of annotation that came before this descriptor + """ + + return self._annotation_lines + + def _parse(self, entries, validate): + """ + Parses a series of 'keyword => (value, pgp block)' mappings and applies + them as attributes. + + Arguments: + entries (dict) - descriptor contents to be applied + validate (bool) - checks the validity of descriptor content if True + + Raises: + ValueError if an error occures in validation + """ - # A descriptor contains a series of 'keyword lines' which are simply a - # keyword followed by an optional value. Lines can also be followed by a - # signature block. - # - # We care about the ordering of 'accept' and 'reject' entries because this - # influences the resulting exit policy, but for everything else the order - # does not matter so breaking it into key / value pairs. - - entries = {} - remaining_contents = contents.split("\n") - first_entry, last_entry = remaining_contents[0], remaining_contents[0] - while remaining_contents: - line = remaining_contents.pop(0) - - # last line can be empty - if not line and not remaining_contents: continue - last_entry = line - - # Some lines have an 'opt ' for backward compatability. They should be - # ignored. This prefix is being removed in... - # https://trac.torproject.org/projects/tor/ticket/5124 - - if line.startswith("opt "): line = line[4:] - - line_match = KEYWORD_LINE.match(line) - - if not line_match: - if not validate: continue - raise ValueError("Line contains invalid characters: %s" % line) - - keyword, value = line_match.groups() - - try: - block_contents = _get_pseudo_pgp_block(remaining_contents) - except ValueError, exc: - if not validate: continue - raise exc - - if keyword in ("accept", "reject"): - self.exit_policy.append("%s %s" % (keyword, value)) - elif keyword in entries: - entries[keyword].append((value, block_contents)) - else: - entries[keyword] = [(value, block_contents)] - # validates restrictions about the entries - if validate: - for keyword in REQUIRED_FIELDS: - if not keyword in entries: - raise ValueError("Descriptor must have a '%s' entry" % keyword) - - for keyword in SINGLE_FIELDS + REQUIRED_FIELDS: - if keyword in entries and len(entries[keyword]) > 1: - raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) - - if not first_entry.startswith(ENTRY_START): - raise ValueError("Descriptor must start with a '%s' entry" % ENTRY_START) - elif not last_entry.startswith(ENTRY_END): - raise ValueError("Descriptor must end with a '%s' entry" % ENTRY_END) - elif not self.exit_policy: - raise ValueError("Descriptor must have at least one 'accept' or 'reject' entry") - - # parse all the entries into our attributes for keyword, values in entries.items(): # most just work with the first (and only) value value, block_contents = values[0] @@ -493,21 +430,6 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): raise ValueError("Uptime line must have an integer value: %s" % value) self.uptime = int(value) - elif keyword == "onion-key": - if validate and not block_contents: - raise ValueError("Onion key line must be followed by a public key: %s" % line) - - self.onion_key = block_contents - elif keyword == "signing-key": - if validate and not block_contents: - raise ValueError("Signing key line must be followed by a public key: %s" % line) - - self.signing_key = block_contents - elif keyword == "router-signature": - if validate and not block_contents: - raise ValueError("Router signature line must be followed by a signature block: %s" % line) - - self.signature = block_contents elif keyword == "contact": self.contact = value elif keyword == "protocols": @@ -533,34 +455,72 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): else: self._unrecognized_lines.append(line) - def get_unrecognized_lines(self): - return list(self._unrecognized_lines) - - def get_annotations(self): + def _check_constraints(self, contents): """ - Provides content that appeard prior to the descriptor. If this comes from - the cached-descriptors file then this commonly contains content like... + Does a basic check that the entries conform to this descriptor type's + constraints. - @downloaded-at 2012-03-18 21:18:29 - @source "173.254.216.66" + Arguments: + contents (_DescriptorContents) - contents to be validated - Returns: - dict with the key/value pairs in our annotations + Raises: + ValueError if an issue arises in validation """ - return self._annotation_dict + required_fields = self._required_fields() + if required_fields: + for keyword in required_fields: + if not keyword in contents.entries: + raise ValueError("Descriptor must have a '%s' entry" % keyword) + + single_fields = self._single_fields() + if single_fields: + for keyword in self._single_fields(): + if keyword in contents.entries and len(contents.entries[keyword]) > 1: + raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) + + first_keyword = self._first_keyword() + if first_keyword and not contents.first_keyword == first_keyword: + raise ValueError("Descriptor must start with a '%s' entry" % first_keyword) + + last_keyword = self._last_keyword() + if last_keyword and not contents.last_keyword == self._last_keyword(): + raise ValueError("Descriptor must end with a '%s' entry" % last_keyword) + + if not self.exit_policy: + raise ValueError("Descriptor must have at least one 'accept' or 'reject' entry") - def get_annotation_lines(self): - """ - Provides the lines of content that appeared prior to the descriptor. This - is the same as the get_annotations() results, but with the unparsed lines - and ordering retained. + # Constraints that the descriptor must meet to be valid. These can be None if + # not applicable. + + def _required_fields(self): return None + def _single_fields(self): return None + def _first_keyword(self): return None + def _last_keyword(self): return None + +class RelayDescriptorV3(ServerDescriptorV3): + """ + Version 3 server descriptor, as specified in... + https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt + + Attributes: + onion_key (str) - key used to encrypt EXTEND cells (*) + signing_key (str) - relay's long-term identity key (*) + signature (str) - signature for this descriptor (*) - Returns: - list with the lines of annotation that came before this descriptor - """ + (*) required fields, others are left as None if undefined + """ + + def __init__(self, raw_contents, validate = True, annotations = None): + ServerDescriptorV3.__init__(self, raw_contents, annotations) + self.onion_key = None + self.signing_key = None + self.signature = None - return self._annotation_lines + contents = _DescriptorContents(raw_contents, validate) + self.exit_policy = contents.exit_policy + self._parse(contents.entries, validate) + if validate: self._check_constraints(contents) def is_valid(self): """ @@ -571,4 +531,196 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): """ raise NotImplementedError # TODO: implement + + def _parse(self, entries, validate): + entries = dict(entries) # shallow copy since we're destructive + + # handles fields only in server descriptors + for keyword, values in entries.items(): + value, block_contents = values[0] + + if keyword == "onion-key": + if validate and not block_contents: + raise ValueError("Onion key line must be followed by a public key: %s" % line) + + self.onion_key = block_contents + del entries["onion-key"] + elif keyword == "signing-key": + if validate and not block_contents: + raise ValueError("Signing key line must be followed by a public key: %s" % line) + + self.signing_key = block_contents + del entries["signing-key"] + elif keyword == "router-signature": + if validate and not block_contents: + raise ValueError("Router signature line must be followed by a signature block: %s" % line) + + self.signature = block_contents + del entries["router-signature"] + + ServerDescriptorV3._parse(self, entries, validate) + + def _required_fields(self): + return REQUIRED_FIELDS + + def _single_fields(self): + return REQUIRED_FIELDS + SINGLE_FIELDS + + def _first_keyword(self): + return "router" + + def _last_keyword(self): + return "router-signature" + +class BridgeDescriptorV3(ServerDescriptorV3): + """ + Version 3 bridge descriptor, as specified in... + https://metrics.torproject.org/formats.html#bridgedesc + """ + + def __init__(self, raw_contents, validate = True, annotations = None): + ServerDescriptorV3.__init__(self, raw_contents, annotations) + + contents = _DescriptorContents(raw_contents, validate) + self.exit_policy = contents.exit_policy + self._parse(contents.entries, validate) + if validate: self._check_constraints(contents) + + def _parse(self, entries, validate): + ServerDescriptorV3._parse(self, entries, validate) + + if validate: + # checks that we properly scrubbed fields + if self.nickname != "Unnamed": + raise ValueError("Router line's nickname should be scrubbed to be 'Unnamed': %s" % self.nickname) + elif not self.address.startswith("10."): + raise ValueError("Router line's address should be scrubbed to be '10.x.x.x': %s" % self.address) + elif self.contact and self.contact != "somebody": + raise ValueError("Contact line should be scrubbed to be 'somebody', but instead had '%s'" % self.contact) + + for line in self.get_unrecognized_lines(): + if line.startswith("onion-key "): + raise ValueError("Bridge descriptors should have their onion-key scrubbed: %s" % self.onion_key) + elif line.startswith("signing-key "): + raise ValueError("Bridge descriptors should have their signing-key scrubbed: %s" % self.signing_key) + elif line.startswith("router-signature "): + raise ValueError("Bridge descriptors should have their signature scrubbed: %s" % self.signature) + + def _required_fields(self): + # bridge required fields are the same as a relay descriptor, minus items + # excluded according to the format page + + excluded_fields = ( + "onion-key", + "signing-key", + "router-signature", + ) + + return filter(lambda e: not e in excluded_fields, REQUIRED_FIELDS) + + def _single_fields(self): + return self._required_fields() + SINGLE_FIELDS + + def _first_keyword(self): + return "router" + +class _DescriptorContents: + """ + Initial breakup of the server descriptor contents to make parsing easier + later. + + A descriptor contains a series of 'keyword lines' which are simply a keyword + followed by an optional value. Lines can also be followed by a signature + block. + + We care about the ordering of 'accept' and 'reject' entries because this + influences the resulting exit policy, but for everything else the order does + not matter so breaking it into key / value pairs. + + Attributes: + entries (dict) - keyword => (value, pgp key) entries + first_keyword (str) - keyword of the first line + last_keyword (str) - keyword of the last line + exit_policy (list) - lines containing the exit policy + """ + + def __init__(self, raw_contents, validate): + self.entries = {} + self.first_keyword = None + self.last_keyword = None + self.exit_policy = [] + remaining_lines = raw_contents.split("\n") + + while remaining_lines: + line = remaining_lines.pop(0) + + # last line can be empty + if not line and not remaining_lines: continue + + # Some lines have an 'opt ' for backward compatability. They should be + # ignored. This prefix is being removed in... + # https://trac.torproject.org/projects/tor/ticket/5124 + + if line.startswith("opt "): line = line[4:] + + line_match = KEYWORD_LINE.match(line) + + if not line_match: + if not validate: continue + raise ValueError("Line contains invalid characters: %s" % line) + + keyword, value = line_match.groups() + + if not self.first_keyword: self.first_keyword = keyword + self.last_keyword = keyword + + try: + block_contents = _get_pseudo_pgp_block(remaining_lines) + except ValueError, exc: + if not validate: continue + raise exc + + if keyword in ("accept", "reject"): + self.exit_policy.append("%s %s" % (keyword, value)) + elif keyword in self.entries: + self.entries[keyword].append((value, block_contents)) + else: + self.entries[keyword] = [(value, block_contents)] + +def _get_pseudo_pgp_block(remaining_contents): + """ + Checks if given contents begins with a pseudo-Open-PGP-style block and, if + so, pops it off and provides it back to the caller. + + Arguments: + remaining_contents (list) - lines to be checked for a public key block + + Returns: + str with the armor wrapped contents or None if it doesn't exist + + Raises: + ValueError if the contents starts with a key block but it's malformed (for + instance, if it lacks an ending line) + """ + + if not remaining_contents: + return None # nothing left + + block_match = PGP_BLOCK_START.match(remaining_contents[0]) + + if block_match: + block_type = block_match.groups()[0] + block_lines = [] + + while True: + if not remaining_contents: + raise ValueError("Unterminated pgp style block") + + line = remaining_contents.pop(0) + block_lines.append(line) + + if line == PGP_BLOCK_END % block_type: + return "\n".join(block_lines) + else: + return None diff --git a/test/integ/descriptor/data/bridge_descriptor b/test/integ/descriptor/data/bridge_descriptor new file mode 100644 index 0000000..d28ccfa --- /dev/null +++ b/test/integ/descriptor/data/bridge_descriptor @@ -0,0 +1,13 @@ +router Unnamed 10.45.227.253 9001 0 0 +or-address [fd9f:2e19:3bcf::02:9970]:9001 +platform Tor 0.2.3.12-alpha (git-800942b4176ca31c) on Linux x86_64 +opt protocols Link 1 2 Circuit 1 +published 2012-03-22 17:34:38 +opt fingerprint AE54 E28E D069 CDF4 5F30 09F9 63EE 3B3D 6FA2 6A2E +uptime 186 +bandwidth 409600 819200 5120 +opt extra-info-digest 134F81F7A0D270B85FCD481DD10CEA34BA7B15C9 +family $CE396C72A3D0880F74C064FEA79D68C15BD380B9 $AB8B00C00B1347BA80A88E548FAC9EDF701D7D0E $8C8A470D7C23151665A7B84E75E89FCC205A3304 +opt hidden-service-dir +contact somebody +reject *:* diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 8cf0ae4..19b2a6d 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -61,7 +61,7 @@ dskLSPz8beUW7bzwDjR6EVNGpyoZde83Ejvau+5F2c6cGnlu91fiZN3suE88iE6e Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= -----END SIGNATURE-----""" - desc = stem.descriptor.server_descriptor.ServerDescriptorV3(descriptor_contents) + desc = stem.descriptor.server_descriptor.RelayDescriptorV3(descriptor_contents) self.assertEquals("caerSidi", desc.nickname) self.assertEquals("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", desc.fingerprint) self.assertEquals("71.35.133.197", desc.address) @@ -145,7 +145,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= expected_published = datetime.datetime(2012, 3, 21, 16, 28, 14) expected_contact = "2048R/F171EC1F Johan Bl\xc3\xa5b\xc3\xa4ck \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf" - desc = stem.descriptor.server_descriptor.ServerDescriptorV3(descriptor_contents) + desc = stem.descriptor.server_descriptor.RelayDescriptorV3(descriptor_contents) self.assertEquals("torrelay389752132", desc.nickname) self.assertEquals("FEBC7F992AC418BBE42BC13FE94EFCFE6549197E", desc.fingerprint) self.assertEquals("130.243.230.116", desc.address) @@ -171,4 +171,50 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals(84275, desc.observed_bandwidth) self.assertEquals(["reject *:*"], desc.exit_policy) self.assertEquals([], desc.get_unrecognized_lines()) + + def test_bridge_descriptor(self): + """ + Parses a bridge descriptor. + """ + + descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "bridge_descriptor") + + descriptor_file = open(descriptor_path) + descriptor_contents = descriptor_file.read() + descriptor_file.close() + + expected_published = datetime.datetime(2012, 3, 22, 17, 34, 38) + + expected_family = [ + "$CE396C72A3D0880F74C064FEA79D68C15BD380B9", + "$AB8B00C00B1347BA80A88E548FAC9EDF701D7D0E", + "$8C8A470D7C23151665A7B84E75E89FCC205A3304", + ] + + desc = stem.descriptor.server_descriptor.BridgeDescriptorV3(descriptor_contents) + self.assertEquals("Unnamed", desc.nickname) + self.assertEquals("AE54E28ED069CDF45F3009F963EE3B3D6FA26A2E", desc.fingerprint) + self.assertEquals("10.45.227.253", desc.address) + self.assertEquals(9001, desc.or_port) + self.assertEquals(0, desc.socks_port) + self.assertEquals(0, desc.dir_port) + self.assertEquals("Tor 0.2.3.12-alpha (git-800942b4176ca31c) on Linux x86_64", desc.platform) + self.assertEquals(stem.version.Version("0.2.3.12-alpha"), desc.tor_version) + self.assertEquals("Linux x86_64", desc.operating_system) + self.assertEquals(186, desc.uptime) + self.assertEquals(expected_published, desc.published) + self.assertEquals("somebody", desc.contact) + self.assertEquals(["1", "2"], desc.link_protocols) + self.assertEquals(["1"], desc.circuit_protocols) + self.assertEquals(False, desc.hibernating) + self.assertEquals(False, desc.allow_single_hop_exits) + self.assertEquals(False, desc.extra_info_cache) + self.assertEquals("134F81F7A0D270B85FCD481DD10CEA34BA7B15C9", desc.extra_info_digest) + self.assertEquals(["2"], desc.hidden_service_dir) + self.assertEquals(expected_family, desc.family) + self.assertEquals(409600, desc.average_bandwidth) + self.assertEquals(819200, desc.burst_bandwidth) + self.assertEquals(5120, desc.observed_bandwidth) + self.assertEquals(["reject *:*"], desc.exit_policy) + self.assertEquals([], desc.get_unrecognized_lines()) diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index cde6714..42325ff 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -7,7 +7,7 @@ import StringIO import unittest import stem.descriptor.server_descriptor -from stem.descriptor.server_descriptor import ServerDescriptorV3 +from stem.descriptor.server_descriptor import RelayDescriptorV3 CRYPTO_BLOB = """ MIGJAoGBAJv5IIWQ+WDWYUdyA/0L8qbIkEVH/cwryZWoIaPAzINfrw1WfNZGtBmg @@ -63,7 +63,7 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor() - desc = ServerDescriptorV3(desc_text) + desc = RelayDescriptorV3(desc_text) self.assertEquals("caerSidi", desc.nickname) self.assertEquals("71.35.133.197", desc.address) @@ -78,7 +78,7 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor({"opt": "contact www.atagar.com/contact/"}) - desc = ServerDescriptorV3(desc_text) + desc = RelayDescriptorV3(desc_text) self.assertEquals("www.atagar.com/contact/", desc.contact) def test_unrecognized_line(self): @@ -87,7 +87,7 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor({"pepperjack": "is oh so tasty!"}) - desc = ServerDescriptorV3(desc_text) + desc = RelayDescriptorV3(desc_text) self.assertEquals(["pepperjack is oh so tasty!"], desc.get_unrecognized_lines()) def test_proceeding_line(self): @@ -168,12 +168,12 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor({"platform": ""}) - desc = ServerDescriptorV3(desc_text, validate = False) + desc = RelayDescriptorV3(desc_text, validate = False) self.assertEquals("", desc.platform) # does the same but with 'platform ' replaced with 'platform' desc_text = desc_text.replace("platform ", "platform") - desc = ServerDescriptorV3(desc_text, validate = False) + desc = RelayDescriptorV3(desc_text, validate = False) self.assertEquals("", desc.platform) def test_protocols_no_circuit_versions(self): @@ -195,7 +195,7 @@ class TestServerDescriptor(unittest.TestCase): desc_text = _make_descriptor({"published": "2012-02-29 04:03:19"}) expected_published = datetime.datetime(2012, 2, 29, 4, 3, 19) - self.assertEquals(expected_published, ServerDescriptorV3(desc_text).published) + self.assertEquals(expected_published, RelayDescriptorV3(desc_text).published) def test_published_no_time(self): """ @@ -242,10 +242,10 @@ class TestServerDescriptor(unittest.TestCase): for attr in stem.descriptor.server_descriptor.REQUIRED_FIELDS: desc_text = _make_descriptor(exclude = [attr]) - self.assertRaises(ValueError, ServerDescriptorV3, desc_text) + self.assertRaises(ValueError, RelayDescriptorV3, desc_text) # check that we can still construct it without validation - desc = ServerDescriptorV3(desc_text, validate = False) + desc = RelayDescriptorV3(desc_text, validate = False) # for one of them checks that the corresponding values are None if attr == "router": @@ -262,8 +262,8 @@ class TestServerDescriptor(unittest.TestCase): value when we're constructed without validation. """ - self.assertRaises(ValueError, ServerDescriptorV3, desc_text) - desc = ServerDescriptorV3(desc_text, validate = False) + self.assertRaises(ValueError, RelayDescriptorV3, desc_text) + desc = RelayDescriptorV3(desc_text, validate = False) if attr: # check that the invalid attribute matches the expected value when

1 0

[stem/master] Server descriptor reformatting
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 57ed21a26d440d32e01924cbc027d97665fe8dbc Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Apr 10 09:03:39 2012 -0700 Server descriptor reformatting Cleaning up the recent changes to split apart relay and bridge descriptors, dropping an unnecessary parser class and rearranging things to be more readable. --- stem/descriptor/server_descriptor.py | 204 +++++++++++++++++++--------------- 1 files changed, 112 insertions(+), 92 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index a0f688c..8548cf2 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -238,31 +238,28 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): self.read_history = None self.write_history = None self.eventdns = True + self._unrecognized_lines = [] + + self._annotation_lines = annotations if annotations else [] + self._annotation_dict = None # cached breakdown of key/value mappings # TODO: Until we have a proper ExitPolicy class this is just a list of the # exit policy strings... self.exit_policy = [] - self._unrecognized_lines = [] + # A descriptor contains a series of 'keyword lines' which are simply a + # keyword followed by an optional value. Lines can also be followed by a + # signature block. + # + # We care about the ordering of 'accept' and 'reject' entries because this + # influences the resulting exit policy, but for everything else the order + # does not matter so breaking it into key / value pairs. - if annotations: - self._annotation_lines = annotations - self._annotation_dict = {} - - for line in annotations: - if " " in line: - key, value = line.split(" ", 1) - self._annotation_dict[key] = value - else: self._annotation_dict[line] = None - else: - self._annotation_lines = [] - self._annotation_dict = {} - - contents = _DescriptorContents(raw_contents, validate) - self.exit_policy = contents.exit_policy - self._parse(contents.entries, validate) - if validate: self._check_constraints(contents) + entries, first_keyword, last_keyword, self.exit_policy = \ + _get_descriptor_components(raw_contents, validate) + self._parse(entries, validate) + if validate: self._check_constraints(entries, first_keyword, last_keyword) def get_unrecognized_lines(self): return list(self._unrecognized_lines) @@ -279,6 +276,17 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): dict with the key/value pairs in our annotations """ + if self._annotation_dict == None: + annotation_dict = {} + + for line in self._annotation_lines: + if " " in line: + key, value = line.split(" ", 1) + annotation_dict[key] = value + else: annotation_dict[line] = None + + self._annotation_dict = annotation_dict + return self._annotation_dict def get_annotation_lines(self): @@ -306,7 +314,6 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): ValueError if an error occures in validation """ - for keyword, values in entries.items(): # most just work with the first (and only) value value, block_contents = values[0] @@ -460,13 +467,15 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): else: self._unrecognized_lines.append(line) - def _check_constraints(self, contents): + def _check_constraints(self, entries, first_keyword, last_keyword): """ Does a basic check that the entries conform to this descriptor type's constraints. Arguments: - contents (_DescriptorContents) - contents to be validated + entries (dict) - keyword => (value, pgp key) entries + first_keyword (str) - keyword of the first line + last_keyword (str) - keyword of the last line Raises: ValueError if an issue arises in validation @@ -475,21 +484,21 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): required_fields = self._required_fields() if required_fields: for keyword in required_fields: - if not keyword in contents.entries: + if not keyword in entries: raise ValueError("Descriptor must have a '%s' entry" % keyword) single_fields = self._single_fields() if single_fields: for keyword in self._single_fields(): - if keyword in contents.entries and len(contents.entries[keyword]) > 1: + if keyword in entries and len(entries[keyword]) > 1: raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword) - first_keyword = self._first_keyword() - if first_keyword and not contents.first_keyword == first_keyword: + expected_first_keyword = self._first_keyword() + if expected_first_keyword and not first_keyword == expected_first_keyword: raise ValueError("Descriptor must start with a '%s' entry" % first_keyword) - last_keyword = self._last_keyword() - if last_keyword and not contents.last_keyword == self._last_keyword(): + expected_last_keyword = self._last_keyword() + if expected_last_keyword and not last_keyword == expected_last_keyword: raise ValueError("Descriptor must end with a '%s' entry" % last_keyword) if not self.exit_policy: @@ -581,23 +590,28 @@ class BridgeDescriptorV3(ServerDescriptorV3): def _parse(self, entries, validate): ServerDescriptorV3._parse(self, entries, validate) + if validate: self._check_scrubbing() + + def _check_scrubbing(self): + """ + Checks that our attributes have been scrubbed in accordance with the bridge + descriptor specification. + """ - if validate: - # checks that we properly scrubbed fields - if self.nickname != "Unnamed": - raise ValueError("Router line's nickname should be scrubbed to be 'Unnamed': %s" % self.nickname) - elif not self.address.startswith("10."): - raise ValueError("Router line's address should be scrubbed to be '10.x.x.x': %s" % self.address) - elif self.contact and self.contact != "somebody": - raise ValueError("Contact line should be scrubbed to be 'somebody', but instead had '%s'" % self.contact) - - for line in self.get_unrecognized_lines(): - if line.startswith("onion-key "): - raise ValueError("Bridge descriptors should have their onion-key scrubbed: %s" % self.onion_key) - elif line.startswith("signing-key "): - raise ValueError("Bridge descriptors should have their signing-key scrubbed: %s" % self.signing_key) - elif line.startswith("router-signature "): - raise ValueError("Bridge descriptors should have their signature scrubbed: %s" % self.signature) + if self.nickname != "Unnamed": + raise ValueError("Router line's nickname should be scrubbed to be 'Unnamed': %s" % self.nickname) + elif not self.address.startswith("10."): + raise ValueError("Router line's address should be scrubbed to be '10.x.x.x': %s" % self.address) + elif self.contact and self.contact != "somebody": + raise ValueError("Contact line should be scrubbed to be 'somebody', but instead had '%s'" % self.contact) + + for line in self.get_unrecognized_lines(): + if line.startswith("onion-key "): + raise ValueError("Bridge descriptors should have their onion-key scrubbed: %s" % line) + elif line.startswith("signing-key "): + raise ValueError("Bridge descriptors should have their signing-key scrubbed: %s" % line) + elif line.startswith("router-signature "): + raise ValueError("Bridge descriptors should have their signature scrubbed: %s" % line) def _required_fields(self): # bridge required fields are the same as a relay descriptor, minus items @@ -617,10 +631,9 @@ class BridgeDescriptorV3(ServerDescriptorV3): def _first_keyword(self): return "router" -class _DescriptorContents: +def _get_descriptor_components(raw_contents, validate): """ - Initial breakup of the server descriptor contents to make parsing easier - later. + Initial breakup of the server descriptor contents to make parsing easier. A descriptor contains a series of 'keyword lines' which are simply a keyword followed by an optional value. Lines can also be followed by a signature @@ -630,55 +643,62 @@ class _DescriptorContents: influences the resulting exit policy, but for everything else the order does not matter so breaking it into key / value pairs. - Attributes: - entries (dict) - keyword => (value, pgp key) entries - first_keyword (str) - keyword of the first line - last_keyword (str) - keyword of the last line - exit_policy (list) - lines containing the exit policy + Arguments: + raw_contents (str) - descriptor content provided by the relay + validate (bool) - checks the validity of the descriptor's content if + True, skips these checks otherwise + + Returns: + tuple with the following attributes... + entries (dict) - keyword => (value, pgp key) entries + first_keyword (str) - keyword of the first line + last_keyword (str) - keyword of the last line + exit_policy (list) - lines containing the exit policy """ - def __init__(self, raw_contents, validate): - self.entries = {} - self.first_keyword = None - self.last_keyword = None - self.exit_policy = [] - remaining_lines = raw_contents.split("\n") + entries = {} + first_keyword = None + last_keyword = None + exit_policy = [] + remaining_lines = raw_contents.split("\n") + + while remaining_lines: + line = remaining_lines.pop(0) - while remaining_lines: - line = remaining_lines.pop(0) - - # last line can be empty - if not line and not remaining_lines: continue - - # Some lines have an 'opt ' for backward compatability. They should be - # ignored. This prefix is being removed in... - # https://trac.torproject.org/projects/tor/ticket/5124 - - if line.startswith("opt "): line = line[4:] - - line_match = KEYWORD_LINE.match(line) - - if not line_match: - if not validate: continue - raise ValueError("Line contains invalid characters: %s" % line) - - keyword, value = line_match.groups() - - if not self.first_keyword: self.first_keyword = keyword - self.last_keyword = keyword - - try: - block_contents = _get_pseudo_pgp_block(remaining_lines) - except ValueError, exc: - if not validate: continue - raise exc - - if keyword in ("accept", "reject"): - self.exit_policy.append("%s %s" % (keyword, value)) - elif keyword in self.entries: - self.entries[keyword].append((value, block_contents)) - else: - self.entries[keyword] = [(value, block_contents)] + # last line can be empty + if not line and not remaining_lines: continue + + # Some lines have an 'opt ' for backward compatability. They should be + # ignored. This prefix is being removed in... + # https://trac.torproject.org/projects/tor/ticket/5124 + + if line.startswith("opt "): line = line[4:] + + line_match = KEYWORD_LINE.match(line) + + if not line_match: + if not validate: continue + raise ValueError("Line contains invalid characters: %s" % line) + + keyword, value = line_match.groups() + + if not first_keyword: first_keyword = keyword + last_keyword = keyword + + try: + block_contents = _get_pseudo_pgp_block(remaining_lines) + except ValueError, exc: + if not validate: continue + raise exc + + if keyword in ("accept", "reject"): + exit_policy.append("%s %s" % (keyword, value)) + elif keyword in entries: + entries[keyword].append((value, block_contents)) + else: + entries[keyword] = [(value, block_contents)] + + return entries, first_keyword, last_keyword, exit_policy def _get_pseudo_pgp_block(remaining_contents): """

1 0

[stem/master] Stop tracking last-modified timestamps if file disappears
by atagar＠torproject.org 14 Apr '12

14 Apr '12

commit 57398f137882e22d5ca278a7944212ec62e4d5a1 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Mar 29 09:07:44 2012 -0700 Stop tracking last-modified timestamps if file disappears Improvement suggested by Karsten. For long running processes with a rotating pool of descriptors we could accumulate a lot of 'path => last modified' entries, so only including the entries for files that still exist. --- stem/descriptor/reader.py | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index c81fe6e..bbedefc 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -226,7 +226,8 @@ class DescriptorReader: absolute path (str) => last modified unix timestamp (int) - This includes entries set through the set_processed_files() method. + This includes entries set through the set_processed_files() method. After + each run is reset to only the files that were present during that run. Returns: dict with the absolute paths and unix timestamp for the last modified @@ -305,6 +306,7 @@ class DescriptorReader: self._reader_thread = None def _read_descriptor_files(self): + new_processed_files = {} remaining_files = list(self._targets) while remaining_files and not self._is_stopped.is_set(): @@ -328,12 +330,11 @@ class DescriptorReader: last_modified = int(os.stat(target).st_mtime) last_used = self._processed_files.get(target) + new_processed_files[target] = last_modified if last_used and last_used >= last_modified: self._notify_skip_listeners(target, AlreadyRead(last_modified, last_used)) continue - else: - self._processed_files[target] = last_modified # The mimetypes module only checks the file extension. To actually # check the content (like the 'file' command) we'd need something like @@ -350,6 +351,7 @@ class DescriptorReader: else: self._notify_skip_listeners(target, UnrecognizedType(target_type)) + self._processed_files = new_processed_files self._enqueue_descriptor(FINISHED) self._iter_notice.set()

1 0