commit 9183f64f2085d0f22d66b6cc381b6e66eef227f7 Author: Damian Johnson atagar@torproject.org Date: Sun Jan 20 13:44:28 2013 -0800
Parsing multiple descriptors from non-cached files
We were parsing single descriptors from metrics archives because... well, that's what they contained. This is unintuitive to users making then parsing their own descriptors, so always using the _parse_file() functions when able.
An extra advantage is that all server descriptors parsed through parse_file() will support server annotations (so things like "@downloaded-at 2012-09-17 17:44:50" will no longer trip it up).
https://trac.torproject.org/8005 --- stem/descriptor/__init__.py | 12 ++- stem/descriptor/extrainfo_descriptor.py | 8 ++- stem/descriptor/server_descriptor.py | 9 ++- .../descriptor/data/metrics_server_desc_multiple | 87 ++++++++++++++++++++ test/integ/descriptor/server_descriptor.py | 19 ++++ 5 files changed, 127 insertions(+), 8 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index f1fb86e..f6d76c0 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -158,16 +158,20 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto import stem.descriptor.networkstatus
if descriptor_type == "server-descriptor" and major_version == 1: - yield stem.descriptor.server_descriptor.RelayDescriptor(descriptor_file.read()) + for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = False): + yield desc elif descriptor_type == "bridge-server-descriptor" and major_version == 1: - yield stem.descriptor.server_descriptor.BridgeDescriptor(descriptor_file.read()) + for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = True): + yield desc elif descriptor_type == "extra-info" and major_version == 1: - yield stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor(descriptor_file.read()) + for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = False): + yield desc elif descriptor_type == "bridge-extra-info" and major_version == 1: # version 1.1 introduced a 'transport' field... # https://trac.torproject.org/6257
- yield stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor(descriptor_file.read()) + for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = True): + yield desc elif descriptor_type == "network-status-2" and major_version == 1: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV2
diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index de6b5c9..75e974e 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -133,11 +133,12 @@ SINGLE_FIELDS = ( )
-def _parse_file(descriptor_file, validate = True): +def _parse_file(descriptor_file, is_bridge = False, validate = True): """ Iterates over the extra-info descriptors in a file.
:param file descriptor_file: file with descriptor content + :param bool is_bridge: parses the file as being a bridge descriptor :param bool validate: checks the validity of the descriptor's content if **True**, skips these checks otherwise
@@ -157,7 +158,10 @@ def _parse_file(descriptor_file, validate = True): extrainfo_content += stem.descriptor._read_until_keywords(block_end_prefix, descriptor_file, True)
if extrainfo_content: - yield RelayExtraInfoDescriptor("".join(extrainfo_content), validate) + if is_bridge: + yield BridgeExtraInfoDescriptor("".join(extrainfo_content), validate) + else: + yield RelayExtraInfoDescriptor("".join(extrainfo_content), validate) else: break # done parsing file
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 1a10dd1..a4548bf 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -71,11 +71,12 @@ SINGLE_FIELDS = ( )
-def _parse_file(descriptor_file, validate = True): +def _parse_file(descriptor_file, is_bridge = False, validate = True): """ Iterates over the server descriptors in a file.
:param file descriptor_file: file with descriptor content + :param bool is_bridge: parses the file as being a bridge descriptor :param bool validate: checks the validity of the descriptor's content if **True**, skips these checks otherwise
@@ -125,7 +126,11 @@ def _parse_file(descriptor_file, validate = True): annotations = map(str.strip, annotations)
descriptor_text = "".join(descriptor_content) - yield RelayDescriptor(descriptor_text, validate, annotations) + + if is_bridge: + yield BridgeDescriptor(descriptor_text, validate, annotations) + else: + yield RelayDescriptor(descriptor_text, validate, annotations) else: break # done parsing descriptors
diff --git a/test/integ/descriptor/data/metrics_server_desc_multiple b/test/integ/descriptor/data/metrics_server_desc_multiple new file mode 100644 index 0000000..aa3d044 --- /dev/null +++ b/test/integ/descriptor/data/metrics_server_desc_multiple @@ -0,0 +1,87 @@ +@type server-descriptor 1.0 +router anonion 31.54.58.167 443 0 0 +platform Tor 0.2.3.22-rc on Windows XP +opt protocols Link 1 2 Circuit 1 +published 2012-09-17 07:28:01 +opt fingerprint 9A5E C5BB 8665 17E5 3962 AF4D 3E77 6536 694B 069E +uptime 0 +bandwidth 8388608 8388608 442368 +opt extra-info-digest 0DED759EDA005BFD735E91D3E2DBF22D2D685F12 +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAM+vu//ysOgcbCwri0RnunH8xNwswnTPbbusad7im2lXJuPjMbTFlkcn +3yPxbL1aD3p/PHPfLKOOPcFDVlCma/N8tJ6CEeJn6pZcrvgsU/d1EASQyrhpXFgL +oH/+zfg4ir52dZp4gS03P16i9m4HB4YTewo62cIuZoZKq1smReMzAgMBAAE= +-----END RSA PUBLIC KEY----- +signing-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALSrtq1TqL5TImtctKm/5CvZHb54n5JpkmD8jtmdajgpHF0eaSaaw/Qw +IKaoQn17etuGzbeXF1vgphv2T4xyIm04MiLOszbLGXK1Fsa5uAO+qs1bIy6mLpwg +XrYhyuddYyw1s4U5ZNBNK/0rlwo7hoIZjxcej0AvTGewFx+Rv6UlAgMBAAE= +-----END RSA PUBLIC KEY----- +opt hidden-service-dir +contact anonion at nym dot hush dot com +reject 0.0.0.0/8:* +reject 169.254.0.0/16:* +reject 127.0.0.0/8:* +reject 192.168.0.0/16:* +reject 10.0.0.0/8:* +reject 172.16.0.0/12:* +reject 31.54.58.167:* +accept *:53 +accept *:80 +accept *:443 +accept *:3128 +accept *:8080 +accept *:8000 +accept *:50 +accept *:51 +accept *:389 +accept *:636 +accept *:2123 +accept *:2152 +accept *:11370 +accept *:11371 +accept *:2301 +accept *:2381 +accept *:6518 +accept *:9100 +accept *:1433 +accept *:1434 +accept *:3389 +accept *:5901 +reject *:* +router-signature +-----BEGIN SIGNATURE----- +XzcWkD6QV6CHtcdKuM6mVGGl+m9JsmLtYhpPMxRANxbRLMLaI1+XkO58QS0zvl6f +0vhmifJhJSu+FV+mOkg/aCsPyleSMd8vLOjCzhVv0yCzcn252ujsjGc0HrmyPE1R +p2bQE2u2SpgLKw6oylyr6qLcZkGIWDYhnnyZTg2lXm8= +-----END SIGNATURE----- +router Unnamed 122.60.235.157 9001 0 0 +platform Tor 0.2.2.37 (git-fce6eb1c44e87bc2) on Darwin x86_64 +opt protocols Link 1 2 Circuit 1 +published 2012-09-17 14:57:28 +opt fingerprint 5366 F1D1 9875 9F88 94EA 6E5F F768 C667 F59A FD24 +uptime 542717 +bandwidth 32768 65536 58012 +opt extra-info-digest 33C8C462BFA5A9C6E825A0A717BD63072AB68E59 +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAKCt4oloDEzaHKzuljXGIJujbTbQfl/XZPjc/84eWp8Ka2Vmpa9DL3cV ++bO3wttgA6ZtkGUB6d6AHcfOoRT7tP/wBeQSHSDxh9OmKzGKZiHB76HNxVny8aJd +ngVPgMmolXajdnzCIAIRLdUW6SHvIinyeghLuVSwLZU+eEN5XSXrAgMBAAE= +-----END RSA PUBLIC KEY----- +signing-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAJ5avJflNjXZjGukC0aACUeTMHsQEHsQKCFPwXFVBCX6gDmrfivqT+EB +efT8XTktk44EFJbCAWj65kuj+aoXBfUvrr21at4WAEJ33BgmitwCrzwkKWVX/wLM ++NMIGObvs88HYyaTsoyZ/TToLE5nPQJj77bNyabHgNvYEXyFxooJAgMBAAE= +-----END RSA PUBLIC KEY----- +opt hidden-service-dir +reject *:* +router-signature +-----BEGIN SIGNATURE----- +m0Q/dr4HvyGShKG8DdKyabcTO4lZtGMXNGX+TEM1D43+5mw4D5UgiIdaK9a2tMQA +sBGFCymv3ZHpmPRtOhSMRafwcZDYXeCSdvZXmfGzpyIRrBde3uRIJj97eqA+oOHd +5TlnK2Z0tF+M1bUD0xYHOde5a/lAaiQaw5+mley6i6M= +-----END SIGNATURE----- diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index dd7dcd7..0258dfa 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -9,6 +9,7 @@ import os import unittest
import stem.control +import stem.descriptor import stem.descriptor.server_descriptor import stem.exit_policy import stem.version @@ -89,6 +90,24 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEquals([], desc.get_unrecognized_lines()) self.assertEquals("2C7B27BEAB04B4E2459D89CA6D5CD1CC5F95A689", desc.digest())
+ def test_metrics_descriptor_multiple(self): + """ + Parses and checks our results against a server descriptor from metrics. + """ + + descriptor_path = test.integ.descriptor.get_resource("metrics_server_desc_multiple") + + with open(descriptor_path) as descriptor_file: + descriptors = list(stem.descriptor.parse_file(descriptor_file, "server-descriptor 1.0")) + + self.assertEquals(2, len(descriptors)) + + self.assertEquals("anonion", descriptors[0].nickname) + self.assertEquals("9A5EC5BB866517E53962AF4D3E776536694B069E", descriptors[0].fingerprint) + + self.assertEquals("Unnamed", descriptors[1].nickname) + self.assertEquals("5366F1D198759F8894EA6E5FF768C667F59AFD24", descriptors[1].fingerprint) + def test_old_descriptor(self): """ Parses a relay server descriptor from 2005.