commit 1fa75186657a6b5f1f839314ed666ccf618abc7a Author: Damian Johnson atagar@torproject.org Date: Sun Nov 11 17:11:38 2018 -0800

Add a descriptor type_annotation method

Interesting feature request from irl...

https://trac.torproject.org/projects/tor/ticket/28397

We can't knowledgeably provide a version number (those come from CollecTor, for instance to specify which bridge scrubbing specification metrics used). But we can certainly provide a valid annotation. --- docs/change_log.rst | 1 + stem/descriptor/__init__.py | 59 ++++++++++++++++++---- stem/descriptor/extrainfo_descriptor.py | 4 ++ stem/descriptor/hidden_service_descriptor.py | 2 + stem/descriptor/microdescriptor.py | 2 + stem/descriptor/networkstatus.py | 17 +++++++ stem/descriptor/server_descriptor.py | 4 ++ stem/descriptor/tordnsel.py | 2 + test/unit/descriptor/extrainfo_descriptor.py | 4 ++ test/unit/descriptor/hidden_service_descriptor.py | 1 + test/unit/descriptor/microdescriptor.py | 2 + .../descriptor/networkstatus/bridge_document.py | 1 + test/unit/descriptor/networkstatus/document_v2.py | 1 + test/unit/descriptor/networkstatus/document_v3.py | 4 ++ .../descriptor/networkstatus/key_certificate.py | 1 + test/unit/descriptor/server_descriptor.py | 2 + test/unit/descriptor/tordnsel.py | 2 + 17 files changed, 99 insertions(+), 10 deletions(-)

diff --git a/docs/change_log.rst b/docs/change_log.rst index 01f3f2a4..f8db0b52 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -51,6 +51,7 @@ The following are only available within Stem's `git repository

* **Descriptors**

+ * Added :func:`~stem.descriptor.Descriptor.type_annotation` method (:trac:`28397`) * DescriptorDownloader crashed if **use_mirrors** is set (:trac:`28393`) * Don't download from Serge, a bridge authority that frequently timeout

diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index a9860140..13c69f11 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -125,6 +125,24 @@ DocumentHandler = stem.util.enum.UppercaseEnum( )

+class TypeAnnotation(collections.namedtuple('TypeAnnotation', ['name', 'major_version', 'minor_version'])): + """ + `Tor metrics type annotation + https://metrics.torproject.org/collector.html#relay-descriptors`_. The + string representation is the header annotation, for example "@type + server-descriptor 1.0". + + .. versionadded:: 1.8.0 + + :var str name: name of the descriptor type + :var int major_version: major version number + :var int minor_version: minor version number + """ + + def __str__(self): + return '@type %s %s.%s' % (self.name, self.major_version, self.minor_version) + + class SigningKey(collections.namedtuple('SigningKey', ['private', 'public', 'public_digest'])): """ Key used by relays to sign their server and extrainfo descriptors. @@ -333,30 +351,30 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto # Parses descriptor files from metrics, yielding individual descriptors. This # throws a TypeError if the descriptor_type or version isn't recognized.

- if descriptor_type == 'server-descriptor' and major_version == 1: + if descriptor_type == stem.descriptor.server_descriptor.RelayDescriptor.TYPE_ANNOTATION_NAME and major_version == 1: for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = False, validate = validate, **kwargs): yield desc - elif descriptor_type == 'bridge-server-descriptor' and major_version == 1: + elif descriptor_type == stem.descriptor.server_descriptor.BridgeDescriptor.TYPE_ANNOTATION_NAME and major_version == 1: for desc in stem.descriptor.server_descriptor._parse_file(descriptor_file, is_bridge = True, validate = validate, **kwargs): yield desc - elif descriptor_type == 'extra-info' and major_version == 1: + elif descriptor_type == stem.descriptor.extrainfo_descriptor.RelayExtraInfoDescriptor.TYPE_ANNOTATION_NAME and major_version == 1: for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = False, validate = validate, **kwargs): yield desc - elif descriptor_type == 'microdescriptor' and major_version == 1: + elif descriptor_type == stem.descriptor.microdescriptor.Microdescriptor.TYPE_ANNOTATION_NAME and major_version == 1: for desc in stem.descriptor.microdescriptor._parse_file(descriptor_file, validate = validate, **kwargs): yield desc - elif descriptor_type == 'bridge-extra-info' and major_version == 1: + elif descriptor_type == stem.descriptor.extrainfo_descriptor.BridgeExtraInfoDescriptor.TYPE_ANNOTATION_NAME and major_version == 1: # version 1.1 introduced a 'transport' field... # https://trac.torproject.org/6257

for desc in stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, is_bridge = True, validate = validate, **kwargs): yield desc - elif descriptor_type == 'network-status-2' and major_version == 1: + elif descriptor_type == stem.descriptor.networkstatus.NetworkStatusDocumentV2.TYPE_ANNOTATION_NAME and major_version == 1: document_type = stem.descriptor.networkstatus.NetworkStatusDocumentV2

for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler, **kwargs): yield desc - elif descriptor_type == 'dir-key-certificate-3' and major_version == 1: + elif descriptor_type == stem.descriptor.networkstatus.KeyCertificate.TYPE_ANNOTATION_NAME and major_version == 1: for desc in stem.descriptor.networkstatus._parse_file_key_certs(descriptor_file, validate = validate, **kwargs): yield desc elif descriptor_type in ('network-status-consensus-3', 'network-status-vote-3') and major_version == 1: @@ -369,17 +387,17 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto

for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs): yield desc - elif descriptor_type == 'bridge-network-status' and major_version == 1: + elif descriptor_type == stem.descriptor.networkstatus.BridgeNetworkStatusDocument.TYPE_ANNOTATION_NAME and major_version == 1: document_type = stem.descriptor.networkstatus.BridgeNetworkStatusDocument

for desc in stem.descriptor.networkstatus._parse_file(descriptor_file, document_type, validate = validate, document_handler = document_handler, **kwargs): yield desc - elif descriptor_type == 'tordnsel' and major_version == 1: + elif descriptor_type == stem.descriptor.tordnsel.TorDNSEL.TYPE_ANNOTATION_NAME and major_version == 1: document_type = stem.descriptor.tordnsel.TorDNSEL

for desc in stem.descriptor.tordnsel._parse_file(descriptor_file, validate = validate, **kwargs): yield desc - elif descriptor_type == 'hidden-service-descriptor' and major_version == 1: + elif descriptor_type == stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor.TYPE_ANNOTATION_NAME and major_version == 1: document_type = stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor

for desc in stem.descriptor.hidden_service_descriptor._parse_file(descriptor_file, validate = validate, **kwargs): @@ -617,6 +635,7 @@ class Descriptor(object):

ATTRIBUTES = {} # mapping of 'attribute' => (default_value, parsing_function) PARSER_FOR_LINE = {} # line keyword to its associated parsing function + TYPE_ANNOTATION_NAME = None

def __init__(self, contents, lazy_load = False): self._path = None @@ -675,6 +694,26 @@ class Descriptor(object):

return cls(cls.content(attr, exclude, sign), validate = validate)

+ def type_annotation(self): + """ + Provides the `Tor metrics annotation + https://metrics.torproject.org/collector.html#relay-descriptors`_ of this + descriptor type. For example, "@type server-descriptor 1.0" for server + descriptors. + + Please note that the version number component is specific to CollecTor, + and for the moment hardcode as 1.0. This may change in the future. + + .. versionadded:: 1.8.0 + + :returns: :class:`~stem.descriptor.TypeAnnotation` with our type information + """ + + if self.TYPE_ANNOTATION_NAME is not None: + return TypeAnnotation(self.TYPE_ANNOTATION_NAME, 1, 0) + else: + raise NotImplementedError('%s does not have a @type annotation' % type(self).__name__) + def get_path(self): """ Provides the absolute path that we loaded this descriptor from. diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 485b3063..8d8894d1 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -903,6 +903,8 @@ class RelayExtraInfoDescriptor(ExtraInfoDescriptor): Added the ed25519_certificate and ed25519_signature attributes. """

+ TYPE_ANNOTATION_NAME = 'extra-info' + ATTRIBUTES = dict(ExtraInfoDescriptor.ATTRIBUTES, **{ 'ed25519_certificate': (None, _parse_identity_ed25519_line), 'ed25519_signature': (None, _parse_router_sig_ed25519_line), @@ -963,6 +965,8 @@ class BridgeExtraInfoDescriptor(ExtraInfoDescriptor): Added the ed25519_certificate_hash and router_digest_sha256 attributes. """

+ TYPE_ANNOTATION_NAME = 'bridge-extra-info' + ATTRIBUTES = dict(ExtraInfoDescriptor.ATTRIBUTES, **{ 'ed25519_certificate_hash': (None, _parse_master_key_ed25519_line), 'router_digest_sha256': (None, _parse_router_digest_sha256_line), diff --git a/stem/descriptor/hidden_service_descriptor.py b/stem/descriptor/hidden_service_descriptor.py index 83618dd8..a7cc0e3d 100644 --- a/stem/descriptor/hidden_service_descriptor.py +++ b/stem/descriptor/hidden_service_descriptor.py @@ -213,6 +213,8 @@ class HiddenServiceDescriptor(Descriptor): Added the **skip_crypto_validation** constructor argument. """

+ TYPE_ANNOTATION_NAME = 'hidden-service-descriptor' + ATTRIBUTES = { 'descriptor_id': (None, _parse_rendezvous_service_descriptor_line), 'version': (None, _parse_version_line), diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index 731e8453..74a01071 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -233,6 +233,8 @@ class Microdescriptor(Descriptor): Added the protocols attribute. """

+ TYPE_ANNOTATION_NAME = 'microdescriptor' + ATTRIBUTES = { 'onion_key': (None, _parse_onion_key_line), 'ntor_onion_key': (None, _parse_ntor_onion_key_line), diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 57098e81..63483c94 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -65,6 +65,7 @@ import stem.version from stem.descriptor import ( PGP_BLOCK_END, Descriptor, + TypeAnnotation, DocumentHandler, _descriptor_content, _descriptor_components, @@ -442,6 +443,8 @@ class NetworkStatusDocumentV2(NetworkStatusDocument): a default value, others are left as **None** if undefined """

+ TYPE_ANNOTATION_NAME = 'network-status-2' + ATTRIBUTES = { 'version': (None, _parse_network_status_version_line), 'hostname': (None, _parse_dir_source_line), @@ -1088,6 +1091,16 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): self.routers = dict((desc.fingerprint, desc) for desc in router_iter) self._footer(document_file, validate)

+ def type_annotation(self): + if not self.is_microdescriptor: + return TypeAnnotation('network-status-consensus-3' if not self.is_vote else 'network-status-vote-3', 1, 0) + else: + # Directory authorities do not issue a 'microdescriptor consensus' vote, + # so unlike the above there isn't a 'network-status-microdesc-vote-3' + # counterpart here. + + return TypeAnnotation('network-status-microdesc-consensus-3', 1, 0) + def validate_signatures(self, key_certs): """ Validates we're properly signed by the signing certificates. @@ -1613,6 +1626,8 @@ class KeyCertificate(Descriptor): ***** mandatory attribute """

+ TYPE_ANNOTATION_NAME = 'dir-key-certificate-3' + ATTRIBUTES = { 'version': (None, _parse_dir_key_certificate_version_line), 'address': (None, _parse_dir_address_line), @@ -1766,6 +1781,8 @@ class BridgeNetworkStatusDocument(NetworkStatusDocument): :var datetime published: time when the document was published """

+ TYPE_ANNOTATION_NAME = 'bridge-network-status' + def __init__(self, raw_content, validate = False): super(BridgeNetworkStatusDocument, self).__init__(raw_content)

diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index d212459e..0d12f875 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -793,6 +793,8 @@ class RelayDescriptor(ServerDescriptor): Added the **skip_crypto_validation** constructor argument. """

+ TYPE_ANNOTATION_NAME = 'server-descriptor' + ATTRIBUTES = dict(ServerDescriptor.ATTRIBUTES, **{ 'certificate': (None, _parse_identity_ed25519_line), 'ed25519_certificate': (None, _parse_identity_ed25519_line), @@ -997,6 +999,8 @@ class BridgeDescriptor(ServerDescriptor): descriptors). """

+ TYPE_ANNOTATION_NAME = 'bridge-server-descriptor' + ATTRIBUTES = dict(ServerDescriptor.ATTRIBUTES, **{ 'ed25519_certificate_hash': (None, _parse_master_key_ed25519_for_hash_line), 'router_digest_sha256': (None, _parse_router_digest_sha256_line), diff --git a/stem/descriptor/tordnsel.py b/stem/descriptor/tordnsel.py index b573b79c..c4aba296 100644 --- a/stem/descriptor/tordnsel.py +++ b/stem/descriptor/tordnsel.py @@ -60,6 +60,8 @@ class TorDNSEL(Descriptor): a default value, others are left as **None** if undefined """

+ TYPE_ANNOTATION_NAME = 'tordnsel' + def __init__(self, raw_contents, validate): super(TorDNSEL, self).__init__(raw_contents) raw_contents = stem.util.str_tools._to_unicode(raw_contents) diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index d649040a..f3252d4e 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -72,6 +72,8 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw dir_write_values_start = [0, 0, 0, 227328, 349184, 382976, 738304] self.assertEqual(dir_write_values_start, desc.dir_write_history_values[:7])

+ self.assertEqual('@type extra-info 1.0', str(desc.type_annotation())) + def test_metrics_bridge_descriptor(self): """ Parses and checks our results against an extrainfo bridge descriptor from @@ -133,6 +135,8 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw self.assertEqual({}, desc.dir_v2_responses_unknown) self.assertEqual({}, desc.dir_v2_responses_unknown)

+ self.assertEqual('@type bridge-extra-info 1.0', str(desc.type_annotation())) + @test.require.cryptography def test_descriptor_signing(self): RelayExtraInfoDescriptor.create(sign = True) diff --git a/test/unit/descriptor/hidden_service_descriptor.py b/test/unit/descriptor/hidden_service_descriptor.py index 23067c62..437366a2 100644 --- a/test/unit/descriptor/hidden_service_descriptor.py +++ b/test/unit/descriptor/hidden_service_descriptor.py @@ -424,6 +424,7 @@ class TestHiddenServiceDescriptor(unittest.TestCase): self.assertEqual([], desc.introduction_points_auth) self.assertEqual(b'', desc.introduction_points_content) self.assertEqual([], desc.introduction_points()) + self.assertEqual('@type hidden-service-descriptor 1.0', str(desc.type_annotation()))

def test_unrecognized_line(self): """ diff --git a/test/unit/descriptor/microdescriptor.py b/test/unit/descriptor/microdescriptor.py index 5f245619..bb4b91a2 100644 --- a/test/unit/descriptor/microdescriptor.py +++ b/test/unit/descriptor/microdescriptor.py @@ -74,6 +74,8 @@ class TestMicrodescriptor(unittest.TestCase): self.assertEqual({b'@last-listed': b'2013-02-24 00:18:36'}, router.get_annotations()) self.assertEqual([b'@last-listed 2013-02-24 00:18:36'], router.get_annotation_lines())

+ self.assertEqual('@type microdescriptor 1.0', str(router.type_annotation())) + def test_minimal_microdescriptor(self): """ Basic sanity check that we can parse a microdescriptor with minimal diff --git a/test/unit/descriptor/networkstatus/bridge_document.py b/test/unit/descriptor/networkstatus/bridge_document.py index d027c94d..97e3e178 100644 --- a/test/unit/descriptor/networkstatus/bridge_document.py +++ b/test/unit/descriptor/networkstatus/bridge_document.py @@ -51,6 +51,7 @@ class TestBridgeNetworkStatusDocument(unittest.TestCase): self.assertEqual(datetime.datetime(2012, 6, 1, 4, 7, 4), document.published) self.assertEqual({}, document.routers) self.assertEqual([], document.get_unrecognized_lines()) + self.assertEqual('@type bridge-network-status 1.0', str(document.type_annotation()))

def test_document(self): """ diff --git a/test/unit/descriptor/networkstatus/document_v2.py b/test/unit/descriptor/networkstatus/document_v2.py index a02ea7a5..7dcc235b 100644 --- a/test/unit/descriptor/networkstatus/document_v2.py +++ b/test/unit/descriptor/networkstatus/document_v2.py @@ -56,6 +56,7 @@ TpQQk3nNQF8z6UIvdlvP+DnJV4izWVkQEZgUZgIVM0E= self.assertEqual([], document.get_unrecognized_lines())

self.assertEqual(3, len(document.routers)) + self.assertEqual('@type network-status-2 1.0', str(document.type_annotation()))

router1 = document.routers['719BE45DE224B607C53707D0E2143E2D423E74CF'] self.assertEqual('moria2', router1.nickname) diff --git a/test/unit/descriptor/networkstatus/document_v3.py b/test/unit/descriptor/networkstatus/document_v3.py index 18d9088a..e4258dab 100644 --- a/test/unit/descriptor/networkstatus/document_v3.py +++ b/test/unit/descriptor/networkstatus/document_v3.py @@ -121,6 +121,7 @@ ci356fosgLiM1sVqCUkNdA== self.assertEqual([], document.consensus_methods) self.assertEqual(None, document.published) self.assertEqual([], document.get_unrecognized_lines()) + self.assertEqual('@type network-status-consensus-3 1.0', str(document.type_annotation()))

router = document.routers['348225F83C854796B2DD6364E65CB189B33BD696'] self.assertEqual('test002r', router.nickname) @@ -254,6 +255,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEqual('178.218.213.229', router.address) self.assertEqual(80, router.or_port) self.assertEqual(None, router.dir_port) + self.assertEqual('@type network-status-vote-3 1.0', str(document.type_annotation()))

authority = document.directory_authorities[0] self.assertEqual(1, len(document.directory_authorities)) @@ -1143,6 +1145,8 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertTrue(entry1 in document.routers.values()) self.assertTrue(entry2 in document.routers.values())

+ self.assertEqual('@type network-status-microdesc-consensus-3 1.0', str(document.type_annotation())) + # try with an invalid RouterStatusEntry

entry3 = RouterStatusEntryMicroV3(RouterStatusEntryMicroV3.content({'r': 'ugabuga'}), False) diff --git a/test/unit/descriptor/networkstatus/key_certificate.py b/test/unit/descriptor/networkstatus/key_certificate.py index 0da8da11..985610fb 100644 --- a/test/unit/descriptor/networkstatus/key_certificate.py +++ b/test/unit/descriptor/networkstatus/key_certificate.py @@ -89,6 +89,7 @@ PPc3r7zKlL/jEGHwz+C7kE88HIvkVnKLLn//40b6HxitHSOCkZ1vtp8YyXae6xnU self.assertEqual(expected_signing_key, cert.signing_key) self.assertEqual(expected_crosscert, cert.crosscert) self.assertEqual(expected_key_cert, cert.certification) + self.assertEqual('@type dir-key-certificate-3 1.0', str(cert.type_annotation())) self.assertEqual([], cert.get_unrecognized_lines())

def test_metrics_certificate(self): diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 6e6d27d4..ec1af553 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -155,6 +155,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEqual([], desc.get_unrecognized_lines()) self.assertEqual('2C7B27BEAB04B4E2459D89CA6D5CD1CC5F95A689', desc.digest())

+ self.assertEqual('@type server-descriptor 1.0', str(desc.type_annotation())) self.assertEqual(['2'], desc.hidden_service_dir) # obsolete field

def test_metrics_descriptor_multiple(self): @@ -436,6 +437,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertFalse(hasattr(desc, 'ed25519_certificate')) self.assertEqual('lgIuiAJCoXPRwWoHgG4ZAoKtmrv47aPr4AsbmESj8AA', desc.ed25519_certificate_hash) self.assertEqual('OB/fqLD8lYmjti09R+xXH/D4S2qlizxdZqtudnsunxE', desc.router_digest_sha256) + self.assertEqual('@type bridge-server-descriptor 1.0', str(desc.type_annotation())) self.assertEqual([], desc.get_unrecognized_lines())

def test_cr_in_contact_line(self): diff --git a/test/unit/descriptor/tordnsel.py b/test/unit/descriptor/tordnsel.py index f6cf07ff..fbc17442 100644 --- a/test/unit/descriptor/tordnsel.py +++ b/test/unit/descriptor/tordnsel.py @@ -86,3 +86,5 @@ class TestTorDNSELDescriptor(unittest.TestCase): self.assertTrue(is_valid_fingerprint(desc.fingerprint)) self.assertEqual('030B22437D99B2DB2908B747B6962EAD13AB4038', desc.fingerprint) self.assertEqual(0, len(desc.exit_addresses)) + + self.assertEqual('@type tordnsel 1.0', str(desc.type_annotation()))