commit 51750b88553158d581bd1860a4c140d8f1336f9e Author: Damian Johnson atagar@torproject.org Date: Thu Mar 5 09:33:39 2015 -0800

Support for consensus' new package attribute

Support for the newly added pacakge field...

https://gitweb.torproject.org/torspec.git/commit/?id=ab6453476066fd1bf5c8cb5...

These don't appear to be in the actual consensus yet, but likely will soon. I still have some questions about this field but this parsing apprach should work for now.

https://trac.torproject.org/projects/tor/ticket/15157 --- docs/change_log.rst | 3 +- stem/descriptor/hidden_service_descriptor.py | 2 + stem/descriptor/microdescriptor.py | 3 ++ stem/descriptor/networkstatus.py | 55 ++++++++++++++++++++- test/mocking.py | 1 + test/unit/descriptor/networkstatus/document_v3.py | 42 ++++++++++++++++ 6 files changed, 104 insertions(+), 2 deletions(-)

diff --git a/docs/change_log.rst b/docs/change_log.rst index c181fea..cdd7311 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -60,6 +60,7 @@ conversion (:trac:`14075`). * Lazy-loading descriptors, improving performance by 25-70% depending on what type it is (:trac:`14011`) * Added `support for hidden service descriptors <api/descriptor/hidden_service_descriptor.html>`_ (:trac:`15004`) * The :class:`~stem.descriptor.networkstatus.DirectoryAuthority` 'fingerprint' attribute was actually its 'v3ident' + * Added consensus' new package attribute (:spec:`ab64534`) * Updating Faravahar's address (:trac:`14487`)

* **Utilities** @@ -155,7 +156,7 @@ among numerous other improvements and fixes. Released on **June 1st, 2014**. * **Descriptors**

* Added tarfile support to :func:`~stem.descriptor.__init__.parse_file` (:trac:`10977`) - * Added microdescriptor's new identity and identity_type attributes (:spec:`22cda72`) + * Added microdescriptor's new identifier and identifier_type attributes (:spec:`22cda72`)

* **Utilities**

diff --git a/stem/descriptor/hidden_service_descriptor.py b/stem/descriptor/hidden_service_descriptor.py index 73f5edf..1292189 100644 --- a/stem/descriptor/hidden_service_descriptor.py +++ b/stem/descriptor/hidden_service_descriptor.py @@ -14,6 +14,8 @@ the HSDir flag. ::

HiddenServiceDescriptor - Tor hidden service descriptor. + +.. versionadded:: 1.4.0 """

# TODO: Add a description for how to retrieve them when tor supports that diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index f4ffe94..ffbec43 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -196,6 +196,9 @@ class Microdescriptor(Descriptor): :var str identifier: base64 encoded identity digest, this is only used for collision prevention (:trac:`11743`)

***** attribute is required when we're parsed with validation + + .. versionchanged:: 1.1.0 + Added the identifier and identifier_type attributes. """

ATTRIBUTES = { diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index f465e6f..a162e2e 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -47,8 +47,19 @@ For more information see :func:`~stem.descriptor.__init__.DocumentHandler`... KeyCertificate - Certificate used to authenticate an authority DocumentSignature - Signature of a document by a directory authority DirectoryAuthority - Directory authority as defined in a v3 network status document + + +.. data:: PackageVersion + + Latest recommended version of a package that's available. + + :var str name: name of the package + :var str version: latest recommended version + :var str url: package's url + :var dict digests: mapping of digest types to their value """

+import collections import io

import stem.descriptor.router_status_entry @@ -75,6 +86,13 @@ from stem.descriptor.router_status_entry import ( RouterStatusEntryMicroV3, )

+PackageVersion = collections.namedtuple('PackageVersion', [ + 'name', + 'version', + 'url', + 'digests', +]) + # Version 2 network status document fields, tuples of the form... # (keyword, is_mandatory)

@@ -109,6 +127,7 @@ HEADER_STATUS_DOCUMENT_FIELDS = ( ('voting-delay', True, True, True), ('client-versions', True, True, False), ('server-versions', True, True, False), + ('package', True, True, False), ('known-flags', True, True, True), ('flag-thresholds', True, False, False), ('params', True, True, False), @@ -638,6 +657,30 @@ def _parse_footer_directory_signature_line(descriptor, entries): descriptor.signatures = signatures

+def _parse_package_line(descriptor, entries): + package_versions = [] + + for value, _, _ in entries['package']: + value_comp = value.split() + + if len(value_comp) < 3: + raise ValueError("'package' must at least have a 'PackageName Version URL': %s" % value) + + name, version, url = value_comp[:3] + digests = {} + + for digest_entry in value_comp[3:]: + if '=' not in digest_entry: + raise ValueError("'package' digest entries should be 'key=value' pairs: %s" % value) + + key, value = digest_entry.split('=', 1) + digests[key] = value + + package_versions.append(PackageVersion(name, version, url, digests)) + + descriptor.packages = package_versions + + _parse_header_valid_after_line = _parse_timestamp_line('valid-after', 'valid_after') _parse_header_fresh_until_line = _parse_timestamp_line('fresh-until', 'fresh_until') _parse_header_valid_until_line = _parse_timestamp_line('valid-until', 'valid_until') @@ -669,6 +712,7 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): signatures from all authorities :var list client_versions: list of recommended client tor versions :var list server_versions: list of recommended server tor versions + :var list packages: ***** list of :data:`~stem.descriptor.networkstatus.PackageVersion` entries :var list known_flags: ***** list of :data:`~stem.Flag` for the router's flags :var dict params: ***** dict of parameter(**str**) => value(**int**) mappings :var list directory_authorities: ***** list of :class:`~stem.descriptor.networkstatus.DirectoryAuthority` @@ -689,6 +733,9 @@ class NetworkStatusDocumentV3(NetworkStatusDocument):

***** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + + .. versionchanged:: 1.4.0 + Added the packages attribute. """

ATTRIBUTES = { @@ -707,6 +754,7 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): 'dist_delay': (None, _parse_header_voting_delay_line), 'client_versions': ([], _parse_header_client_versions_line), 'server_versions': ([], _parse_header_server_versions_line), + 'packages': ([], _parse_package_line), 'known_flags': ([], _parse_header_known_flags_line), 'flag_thresholds': ({}, _parse_header_flag_thresholds_line), 'params': ({}, _parse_header_parameters_line), @@ -727,6 +775,7 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): 'voting-delay': _parse_header_voting_delay_line, 'client-versions': _parse_header_client_versions_line, 'server-versions': _parse_header_server_versions_line, + 'package': _parse_package_line, 'known-flags': _parse_header_known_flags_line, 'flag-thresholds': _parse_header_flag_thresholds_line, 'params': _parse_header_parameters_line, @@ -820,7 +869,7 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): # all known header fields can only appear once except

for keyword, values in list(entries.items()): - if len(values) > 1 and keyword in HEADER_FIELDS: + if len(values) > 1 and keyword in HEADER_FIELDS and keyword != 'package': raise ValueError("Network status documents can only have a single '%s' line, got %i" % (keyword, len(values)))

if self._default_params: @@ -1072,6 +1121,10 @@ class DirectoryAuthority(Descriptor): authority's key certificate

***** mandatory attribute + + .. versionchanged:: 1.4.0 + Renamed our 'fingerprint' attribute to 'v3ident' (prior attribute exists + for backward compatability, but is deprecated). """

ATTRIBUTES = { diff --git a/test/mocking.py b/test/mocking.py index 156ac4a..27b0921 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -173,6 +173,7 @@ NETWORK_STATUS_DOCUMENT_HEADER = ( ('voting-delay', '300 300'), ('client-versions', None), ('server-versions', None), + ('package', None), ('known-flags', 'Authority BadExit Exit Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid'), ('params', None), ) diff --git a/test/unit/descriptor/networkstatus/document_v3.py b/test/unit/descriptor/networkstatus/document_v3.py index 2f847d7..f440bbf 100644 --- a/test/unit/descriptor/networkstatus/document_v3.py +++ b/test/unit/descriptor/networkstatus/document_v3.py @@ -15,6 +15,7 @@ from stem.descriptor.networkstatus import ( HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, + PackageVersion, DirectoryAuthority, NetworkStatusDocumentV3, _parse_file, @@ -125,6 +126,7 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEqual(expected_versions, document.client_versions) self.assertEqual(expected_versions, document.server_versions) self.assertEqual(expected_flags, set(document.known_flags)) + self.assertEqual([], document.packages) self.assertEqual({'CircuitPriorityHalflifeMsec': 30000, 'bwauthpid': 1}, document.params)

self.assertEqual(12, document.consensus_method) @@ -248,6 +250,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEqual([], document.client_versions) self.assertEqual([], document.server_versions) self.assertEqual(expected_flags, set(document.known_flags)) + self.assertEqual([], document.packages) self.assertEqual({'CircuitPriorityHalflifeMsec': 30000, 'bwauthpid': 1}, document.params)

self.assertEqual(None, document.consensus_method) @@ -325,6 +328,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEqual([], document.client_versions) self.assertEqual([], document.server_versions) self.assertEqual(expected_known_flags, document.known_flags) + self.assertEqual([], document.packages) self.assertEqual({}, document.flag_thresholds) self.assertEqual(DEFAULT_PARAMS, document.params) self.assertEqual((), document.directory_authorities) @@ -359,6 +363,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEqual([], document.client_versions) self.assertEqual([], document.server_versions) self.assertEqual(expected_known_flags, document.known_flags) + self.assertEqual([], document.packages) self.assertEqual({}, document.flag_thresholds) self.assertEqual(DEFAULT_PARAMS, document.params) self.assertEqual({}, document.bandwidth_weights) @@ -726,6 +731,43 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= document = NetworkStatusDocumentV3(content, False) self.assertEqual([], getattr(document, attr))

+ def test_packages(self): + """ + Parse the package line. These can appear multiple times, and have any + number of digests. + """ + + test_values = ( + (['Stem 1.3.0 https://stem.torproject.org/'], + [PackageVersion('Stem', '1.3.0', 'https://stem.torproject.org/', {})]), + (['Stem 1.3.0 https://stem.torproject.org/ sha1=5d676c8124b4be1f52ddc8e15ca143cad211eeb4 md5=600ad5e2fc4caf585c1bdaaa532b7e82'], + [PackageVersion('Stem', '1.3.0', 'https://stem.torproject.org/', {'sha1': '5d676c8124b4be1f52ddc8e15ca143cad211eeb4', 'md5': '600ad5e2fc4caf585c1bdaaa532b7e82'})]), + (['Stem 1.3.0 https://stem.torproject.org/', 'Txtorcon 0.13.0 https://github.com/meejah/txtorcon'], + [PackageVersion('Stem', '1.3.0', 'https://stem.torproject.org/', {}), + PackageVersion('Txtorcon', '0.13.0', 'https://github.com/meejah/txtorcon', {})]), + ) + + for test_value, expected_value in test_values: + document = get_network_status_document_v3({'package': '\npackage '.join(test_value)}) + self.assertEqual(expected_value, document.packages) + + test_values = ( + '', + ' ', + 'Stem', + 'Stem 1.3.0', + 'Stem 1.3.0 https://stem.torproject.org/ keyword_field', + 'Stem 1.3.0 https://stem.torproject.org/ keyword_field key=value', + 'Stem 1.3.0 https://stem.torproject.org/ key=value keyword_field', + ) + + for test_value in test_values: + content = get_network_status_document_v3({'package': test_value}, content = True) + self.assertRaises(ValueError, NetworkStatusDocumentV3, content, True) + + document = NetworkStatusDocumentV3(content, False) + self.assertEqual([], document.packages) + def test_known_flags(self): """ Parses some known-flag entries. Just exercising the field, there's not much