commit 301401360337e4c02a5fd5e4e8520cc1ecf88633 Author: Damian Johnson atagar@torproject.org Date: Fri May 18 10:25:49 2012 -0700

Utilising metrics descriptor type annotations

Using the @type annotations added by Karsten to the metrics descriptors as per ticket 5651. --- stem/descriptor/__init__.py | 52 ++++++++++++++------ stem/descriptor/server_descriptor.py | 14 ----- test/integ/descriptor/data/bridge_descriptor | 1 + test/integ/descriptor/data/cr_in_contact_line | 1 + test/integ/descriptor/data/descriptor_archive.tar | Bin 20480 -> 20480 bytes .../descriptor/data/descriptor_archive.tar.bz2 | Bin 3322 -> 3352 bytes .../descriptor/data/descriptor_archive.tar.gz | Bin 2844 -> 2874 bytes test/integ/descriptor/data/example_descriptor | 1 + test/integ/descriptor/data/extrainfo_descriptor | 1 + test/integ/descriptor/data/negative_uptime | 1 + test/integ/descriptor/data/non-ascii_descriptor | 1 + test/integ/descriptor/data/old_descriptor | 1 + test/integ/descriptor/extrainfo_descriptor.py | 1 + test/integ/descriptor/reader.py | 2 + test/integ/descriptor/server_descriptor.py | 6 ++ 15 files changed, 52 insertions(+), 30 deletions(-)

diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index f3a5983..1de1dac 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -35,30 +35,50 @@ def parse_file(path, descriptor_file): IOError if unable to read from the descriptor_file """

- import stem.descriptor.extrainfo_descriptor import stem.descriptor.server_descriptor + import stem.descriptor.extrainfo_descriptor

# The tor descriptor specifications do not provide a reliable method for # identifying a descriptor file's type and version so we need to guess - # based on... - # - its filename for resources from the tor data directory - # - first line of our contents for files provided by metrics + # based on its filename. Metrics descriptors, however, can be identified + # by an annotation on their first line... + # https://trac.torproject.org/5651

- filename = os.path.basename(path) - first_line = descriptor_file.readline() - descriptor_file.seek(0) + # Cached descriptor handling. These contain mulitple descriptors per file.

- if filename == "cached-descriptors" or first_line.startswith("router "): - for desc in stem.descriptor.server_descriptor.parse_file(descriptor_file): - desc._set_path(path) - yield desc - elif filename == "cached-extrainfo" or first_line.startswith("extra-info "): - for desc in stem.descriptor.extrainfo_descriptor.parse_file(descriptor_file): + filename, file_parser = os.path.basename(path), None + + if filename == "cached-descriptors": + file_parser = stem.descriptor.server_descriptor.parse_file + elif filename == "cached-extrainfo": + file_parser = stem.descriptor.extrainfo_descriptor.parse_file + + if file_parser: + for desc in file_parser(descriptor_file): desc._set_path(path) yield desc - else: - # unrecognized descriptor type - raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line)) + + return + + # Metrics descriptor handling. These contain a single descriptor per file. + + first_line, desc = descriptor_file.readline().strip(), None + + if first_line == "@type server-descriptor 1.0": + desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_file.read()) + elif first_line == "@type bridge-server-descriptor 1.0": + desc = stem.descriptor.server_descriptor.BridgeDescriptor(descriptor_file.read()) + elif first_line in ("@type extra-info 1.0", "@type bridge-extra-info 1.0"): + desc = stem.descriptor.extrainfo_descriptor.ExtraInfoDescriptor(descriptor_file.read()) + + if desc: + desc._set_path(path) + yield desc + return + + # Not recognized as a descriptor file. + + raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line))

class Descriptor: """ diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index d347b15..9a91d60 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -80,20 +80,6 @@ def parse_file(descriptor_file, validate = True): IOError if the file can't be read """

- # Handler for bridge descriptors - # - # Bridge descriptors are scrubbed so their nickname is 'Unnamed' and their - # ip address is in the 10.x.x.x space, which is normally reserved for private - # networks. Bridge descriptors only come from metrics so a file only contains - # a single descriptor. - - first_line = descriptor_file.readline() - descriptor_file.seek(0) - - if first_line.startswith("router Unnamed 10."): - yield BridgeDescriptor(descriptor_file.read()) - return - # Handler for relay descriptors # # Cached descriptors consist of annotations followed by the descriptor diff --git a/test/integ/descriptor/data/bridge_descriptor b/test/integ/descriptor/data/bridge_descriptor index d28ccfa..07dffe2 100644 --- a/test/integ/descriptor/data/bridge_descriptor +++ b/test/integ/descriptor/data/bridge_descriptor @@ -1,3 +1,4 @@ +@type bridge-server-descriptor 1.0 router Unnamed 10.45.227.253 9001 0 0 or-address [fd9f:2e19:3bcf::02:9970]:9001 platform Tor 0.2.3.12-alpha (git-800942b4176ca31c) on Linux x86_64 diff --git a/test/integ/descriptor/data/cr_in_contact_line b/test/integ/descriptor/data/cr_in_contact_line index f1d93fb..ca6636c 100644 --- a/test/integ/descriptor/data/cr_in_contact_line +++ b/test/integ/descriptor/data/cr_in_contact_line @@ -1,3 +1,4 @@ +@type server-descriptor 1.0 router pogonip 75.5.248.48 9001 0 0 platform Tor 0.1.2.17 on Darwin i386 published 2007-09-03 10:15:53 diff --git a/test/integ/descriptor/data/descriptor_archive.tar b/test/integ/descriptor/data/descriptor_archive.tar index 2c40716..55d29d1 100644 Binary files a/test/integ/descriptor/data/descriptor_archive.tar and b/test/integ/descriptor/data/descriptor_archive.tar differ diff --git a/test/integ/descriptor/data/descriptor_archive.tar.bz2 b/test/integ/descriptor/data/descriptor_archive.tar.bz2 index ba1f239..2950c57 100644 Binary files a/test/integ/descriptor/data/descriptor_archive.tar.bz2 and b/test/integ/descriptor/data/descriptor_archive.tar.bz2 differ diff --git a/test/integ/descriptor/data/descriptor_archive.tar.gz b/test/integ/descriptor/data/descriptor_archive.tar.gz index 63a6a57..325adb7 100644 Binary files a/test/integ/descriptor/data/descriptor_archive.tar.gz and b/test/integ/descriptor/data/descriptor_archive.tar.gz differ diff --git a/test/integ/descriptor/data/example_descriptor b/test/integ/descriptor/data/example_descriptor index 5ab4f57..6ee442c 100644 --- a/test/integ/descriptor/data/example_descriptor +++ b/test/integ/descriptor/data/example_descriptor @@ -1,3 +1,4 @@ +@type server-descriptor 1.0 router caerSidi 71.35.133.197 9001 0 0 platform Tor 0.2.1.30 on Linux x86_64 opt protocols Link 1 2 Circuit 1 diff --git a/test/integ/descriptor/data/extrainfo_descriptor b/test/integ/descriptor/data/extrainfo_descriptor index 4525afe..ecaabe0 100644 --- a/test/integ/descriptor/data/extrainfo_descriptor +++ b/test/integ/descriptor/data/extrainfo_descriptor @@ -1,3 +1,4 @@ +@type extra-info 1.0 extra-info NINJA B2289C3EAB83ECD6EB916A2F481A02E6B76A0A48 published 2012-05-05 17:03:50 write-history 2012-05-05 17:02:45 (900 s) 1082368,19456,50176,272384,485376,1850368,1132544,1790976,2459648,4091904,6310912,13701120,3209216,3871744,7873536,5440512,7287808,10561536,9979904,11247616,11982848,7590912,10611712,20728832,38534144,6839296,3173376,16678912 diff --git a/test/integ/descriptor/data/negative_uptime b/test/integ/descriptor/data/negative_uptime index 4140d75..388d44b 100644 --- a/test/integ/descriptor/data/negative_uptime +++ b/test/integ/descriptor/data/negative_uptime @@ -1,3 +1,4 @@ +@type server-descriptor 1.0 router TipTor 62.99.247.83 9001 0 9030 platform Tor 0.1.1.25 on Darwin Power Macintosh published 2006-12-18 22:42:40 diff --git a/test/integ/descriptor/data/non-ascii_descriptor b/test/integ/descriptor/data/non-ascii_descriptor index 7758276..2cd2a6b 100644 --- a/test/integ/descriptor/data/non-ascii_descriptor +++ b/test/integ/descriptor/data/non-ascii_descriptor @@ -1,3 +1,4 @@ +@type server-descriptor 1.0 router torrelay389752132 130.243.230.116 9001 0 0 platform Tor 0.2.2.35 (git-4f42b0a93422f70e) on Linux x86_64 opt protocols Link 1 2 Circuit 1 diff --git a/test/integ/descriptor/data/old_descriptor b/test/integ/descriptor/data/old_descriptor index 5407aa8..8be82f9 100644 --- a/test/integ/descriptor/data/old_descriptor +++ b/test/integ/descriptor/data/old_descriptor @@ -1,3 +1,4 @@ +@type server-descriptor 1.0 router krypton 212.37.39.59 8000 0 0 platform Tor 0.1.0.14 on FreeBSD i386 published 2005-12-16 18:01:03 diff --git a/test/integ/descriptor/extrainfo_descriptor.py b/test/integ/descriptor/extrainfo_descriptor.py index 479ff01..a753d27 100644 --- a/test/integ/descriptor/extrainfo_descriptor.py +++ b/test/integ/descriptor/extrainfo_descriptor.py @@ -19,6 +19,7 @@ class TestExtraInfoDescriptor(unittest.TestCase): descriptor_path = test.integ.descriptor.get_resource("extrainfo_descriptor")

descriptor_file = open(descriptor_path) + descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close()

diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index 81635e0..0ae882e 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -51,6 +51,7 @@ def _get_raw_tar_descriptors(): for tar_entry in tar_file: if tar_entry.isfile(): entry = tar_file.extractfile(tar_entry) + entry.readline() # strip header raw_descriptors.append(entry.read()) entry.close()

@@ -149,6 +150,7 @@ class TestDescriptorReader(unittest.TestCase):

descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor") with open(descriptor_path) as descriptor_file: + descriptor_file.readline() # strip header descriptor_entries.append(descriptor_file.read())

# running this test multiple times to flush out concurrency issues diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 5d7e62d..afe3c93 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -21,6 +21,7 @@ class TestServerDescriptor(unittest.TestCase): descriptor_path = test.integ.descriptor.get_resource("example_descriptor")

descriptor_file = open(descriptor_path) + descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close()

@@ -92,6 +93,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_path = test.integ.descriptor.get_resource("old_descriptor")

descriptor_file = open(descriptor_path) + descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close()

@@ -176,6 +178,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_path = test.integ.descriptor.get_resource("non-ascii_descriptor")

descriptor_file = open(descriptor_path) + descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close()

@@ -217,6 +220,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_path = test.integ.descriptor.get_resource("cr_in_contact_line")

descriptor_file = open(descriptor_path) + descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close()

@@ -244,6 +248,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_path = test.integ.descriptor.get_resource("negative_uptime")

descriptor_file = open(descriptor_path) + descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close()

@@ -267,6 +272,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_path = test.integ.descriptor.get_resource("bridge_descriptor")

descriptor_file = open(descriptor_path) + descriptor_file.readline() # strip header descriptor_contents = descriptor_file.read() descriptor_file.close()