tor-commits March 2015

tor-commits@lists.torproject.org

23 participants
1889 discussions

[bridgedb/develop] Condense code for getting statefile absolute path in b.persistent.State.
by isis＠torproject.org 01 Mar '15

01 Mar '15

commit 7de653e942237f65743323e26bf018f4b696a591 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 24 22:58:33 2015 +0000 Condense code for getting statefile absolute path in b.persistent.State. These checks were unnecessary overhead, since the path expansion functions won't do anything if there's nothing to be done. --- lib/bridgedb/persistent.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index ad4398d..1b2507c 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -124,11 +124,7 @@ class State(jelly.Jellyable): :param string statefile: The filename of the statefile. """ - if filename.startswith('~'): - filename = os.path.expanduser(filename) - if not os.path.isabs(filename): - filename = os.path.abspath(filename) - + filename = os.path.abspath(os.path.expanduser(filename)) logging.debug("Setting statefile to '%s'" % filename) self._statefile = filename

1 0

[bridgedb/develop] Decrease logging for email signing.
by isis＠torproject.org 01 Mar '15

01 Mar '15

commit 6557fd3bf7fd87a7c0e769747647b08666269d36 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 24 08:49:48 2015 +0000 Decrease logging for email signing. --- lib/bridgedb/email/autoresponder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/bridgedb/email/autoresponder.py b/lib/bridgedb/email/autoresponder.py index e96ad91..726856c 100644 --- a/lib/bridgedb/email/autoresponder.py +++ b/lib/bridgedb/email/autoresponder.py @@ -357,7 +357,6 @@ class EmailResponse(object): sig = self.gpgSign(body) if sig: body = sig - print("Signed email body:\n%s" % body) # XXX self.writelines(body)

1 0

[bridgedb/develop] Add more unittests for bridgedb.util module.
by isis＠torproject.org 01 Mar '15

01 Mar '15

commit 145db3c76f60cf47fb76b6c82049d99ebc2368d6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 24 08:33:23 2015 +0000 Add more unittests for bridgedb.util module. --- lib/bridgedb/test/test_util.py | 42 ++++++++++++++++++++++++++++++++++++++++ lib/bridgedb/util.py | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_util.py b/lib/bridgedb/test/test_util.py index 065c401..272c5b7 100644 --- a/lib/bridgedb/test/test_util.py +++ b/lib/bridgedb/test/test_util.py @@ -15,6 +15,7 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import logging import os from twisted.mail.smtp import Address @@ -43,6 +44,15 @@ class MiscLoggingUtilTests(unittest.TestCase): self.assertEqual(len(logHandlers), 1) self.assertTrue('console' not in logHandlers) + def test_getLogHandlers_disable_logfile(self): + """util._getLogHandlers() should return ['console'] if stderr logging + is disabled. + """ + logHandlers = util._getLogHandlers(logToFile=False) + self.assertIsInstance(logHandlers, list) + self.assertEqual(len(logHandlers), 1) + self.assertTrue('rotating' not in logHandlers) + def test_getRotatingFileHandler(self): """_getRotatingFileHandler() should create a file with 0600 permissions (os.ST_WRITE | os.ST_APPEND). @@ -100,3 +110,35 @@ class LevenshteinDistanceTests(unittest.TestCase): fuzzyMatch = 4 distance = util.levenshteinDistance(email1.local, email2.local) self.assertLessEqual(distance, fuzzyMatch) + + +class JustifiedLogFormatterTests(unittest.TestCase): + """Unittests for :class:`bridgedb.util.JustifiedLogFormatter`.""" + + def setUp(self): + # name, level, path, lineno, message, args, exc_info + self.record = logging.LogRecord('name', logging.INFO, '/foo/bar/baz', + 12345, 'This is a message', None, None) + + def test_util_JustifiedLogFormatter(self): + formatter = util.JustifiedLogFormatter() + self.assertIsInstance(formatter, logging.Formatter) + + def test_util_JustifiedLogFormatter_logThreads(self): + formatter = util.JustifiedLogFormatter(logThreads=True) + self.assertIsInstance(formatter, logging.Formatter) + + def test_util_JustifiedLogFormatter_formatCallingFuncName(self): + formatter = util.JustifiedLogFormatter() + record = formatter._formatCallingFuncName(self.record) + self.assertIsInstance(formatter, logging.Formatter) + self.assertIsInstance(record, logging.LogRecord) + + def test_util_JustifiedLogFormatter_format(self): + formatter = util.JustifiedLogFormatter() + formatted = formatter.format(self.record) + self.assertIsInstance(formatter, logging.Formatter) + self.assertIsInstance(formatted, basestring) + self.assertNotEqual(formatted, '') + self.assertTrue('INFO' in formatted) + self.assertTrue('This is a message' in formatted) diff --git a/lib/bridgedb/util.py b/lib/bridgedb/util.py index 63fd5e6..4253d4e 100644 --- a/lib/bridgedb/util.py +++ b/lib/bridgedb/util.py @@ -62,7 +62,7 @@ def _getRotatingFileHandler(filename, mode='a', maxBytes=1000000, backupCount=0, os.chown(filename, uid, gid) try: os.chmod(filename, os.ST_WRITE | os.ST_APPEND) - except AttributeError: + except AttributeError: # pragma: no cover logging.error(""" XXX FIXME: Travis chokes on `os.ST_WRITE` saying that the module doesn't have that attribute, for some reason:

1 0

[bridgedb/develop] Refactor b.persistent.State with problems found in additional tests.
by isis＠torproject.org 01 Mar '15

01 Mar '15

commit cda620bc8a4fac397f07a3bc76fe25a5f8abbcf4 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 24 21:04:09 2015 +0000 Refactor b.persistent.State with problems found in additional tests. * ADD documentation that `b.persistent.State.load` can raise a MissingState exception. * REMOVE an `else` clause in b.persistent.State.load() that was impossible to hit because the `elif` clause right before it should always succeed or raise an AttributeError. * ADD error handling for when pickle.load() is given an opened, but empty file, and an EOFError is raised. * CHANGE b.persistent.State.load() to only raise a MissingState exception at the end if there were errors detected during the function execution. * REMOVE unused `err` variable from b.persistent.State.save(). --- lib/bridgedb/persistent.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index 1c0d3df..ad4398d 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -158,6 +158,7 @@ class State(jelly.Jellyable): def load(self, statefile=None): """Load a previously saved statefile. + :raises MissingState: If there was any error loading the **statefile**. :rtype: :class:`State` or None :returns: The state, loaded from :attr:`State.STATEFILE`, or None if an error occurred. @@ -176,8 +177,6 @@ class State(jelly.Jellyable): fh = open(statefile, 'r') elif not statefile.closed: fh = statefile - else: - raise TypeError("Nothing worked.") except (IOError, OSError) as error: # pragma: no cover err += "There was an error reading statefile " err += "'{0}':\n{1}".format(statefile, error) @@ -186,14 +185,19 @@ class State(jelly.Jellyable): err += "\n\t{0}\nstatefile type = '{1}'".format( error.message, type(statefile)) else: - status = pickle.load(fh) - quo = jelly.unjelly(status) - if fh is not None: - fh.close() - if quo: - return quo + try: + status = pickle.load(fh) + except EOFError: + err += "The statefile %s was empty." % fh.name + else: + quo = jelly.unjelly(status) + if fh is not None: + fh.close() + if quo: + return quo - raise MissingState(err) + if err: + raise MissingState(err) def save(self, statefile=None): """Save state as a pickled jelly to a file on disk.""" @@ -204,8 +208,6 @@ class State(jelly.Jellyable): logging.debug("Saving state to: \t'%s'" % statefile) fh = None - err = '' - try: fh = open(statefile, 'w') except (IOError, OSError) as error: # pragma: no cover

1 0

[bridgedb/develop] Fix improper assignment to statefile @property in b.persistent.State.
by isis＠torproject.org 01 Mar '15

01 Mar '15

commit 92df000d063ab9ea9229d66d3956cd78b5ed1ca8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 24 20:57:24 2015 +0000 Fix improper assignment to statefile @property in b.persistent.State. --- lib/bridgedb/persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index f728bdd..1c0d3df 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -100,7 +100,7 @@ class State(jelly.Jellyable): self.key = None if 'STATEFILE' in kwargs: - self.statefile(kwargs['STATEFILE']) + self.statefile = kwargs['STATEFILE'] for key, value in kwargs.items(): self.__dict__[key] = value

1 0

[bridgedb/develop] Add additional unittests for uncovered portions of bridgedb.persistent.
by isis＠torproject.org 01 Mar '15

01 Mar '15

commit 09350c2af6c3248f5543740796f3e73d23b3f7aa Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 24 20:47:18 2015 +0000 Add additional unittests for uncovered portions of bridgedb.persistent. --- lib/bridgedb/persistent.py | 13 ++--- lib/bridgedb/test/test_persistentSaveAndLoad.py | 68 ++++++++++++++++++++++- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/lib/bridgedb/persistent.py b/lib/bridgedb/persistent.py index 8c65e78..f728bdd 100644 --- a/lib/bridgedb/persistent.py +++ b/lib/bridgedb/persistent.py @@ -17,7 +17,7 @@ import os.path try: import cPickle as pickle -except (ImportError, NameError): +except (ImportError, NameError): # pragma: no cover import pickle from twisted.python.reflect import safe_repr @@ -148,7 +148,7 @@ class State(jelly.Jellyable): fh.close() os.unlink(self._statefile) self._statefile = None - except (IOError, OSError) as error: + except (IOError, OSError) as error: # pragma: no cover logging.error("There was an error deleting the statefile: '%s'" % self._statefile) @@ -178,7 +178,7 @@ class State(jelly.Jellyable): fh = statefile else: raise TypeError("Nothing worked.") - except (IOError, OSError) as error: + except (IOError, OSError) as error: # pragma: no cover err += "There was an error reading statefile " err += "'{0}':\n{1}".format(statefile, error) except (AttributeError, TypeError) as error: @@ -208,10 +208,9 @@ class State(jelly.Jellyable): try: fh = open(statefile, 'w') - except MissingState as error: - err += error.message - except (IOError, OSError) as error: - err += "Error writing state file to '%s': %s" % (statefile, error) + except (IOError, OSError) as error: # pragma: no cover + logging.warn("Error writing state file to '%s': %s" + % (statefile, error)) else: try: pickle.dump(jelly.jelly(self), fh) diff --git a/lib/bridgedb/test/test_persistentSaveAndLoad.py b/lib/bridgedb/test/test_persistentSaveAndLoad.py index 7919df3..67585f6 100644 --- a/lib/bridgedb/test/test_persistentSaveAndLoad.py +++ b/lib/bridgedb/test/test_persistentSaveAndLoad.py @@ -67,6 +67,45 @@ class StateSaveAndLoadTests(unittest.TestCase): if savedStatefile: self.assertTrue(os.path.isfile(str(savedStatefile))) + def test_init_with_STATEFILE(self): + config = self.config + setattr(config, 'STATEFILE', '~/foo.state') + state = persistent.State(**config.__dict__) + self.loadedStateAssertions(state) + statefile = state.statefile + self.assertTrue(statefile.endswith('foo.state')) + + def test_init_without_config(self): + state = persistent.State(None) + self.loadedStateAssertions(state) + + def test_init_with_config(self): + state = persistent.State(self.config) + self.loadedStateAssertions(state) + + def test_get_statefile(self): + statefile = self.state._get_statefile() + self.assertIsInstance(statefile, basestring) + + def test_set_statefile(self): + self.state._set_statefile('~/bar.state') + statefile = self.state._get_statefile() + self.assertIsInstance(statefile, basestring) + + def test_set_statefile_new_dir(self): + config = self.config + setattr(config, 'STATEFILE', 'statefiles/foo.state') + state = persistent.State(**config.__dict__) + self.loadedStateAssertions(state) + statefile = state.statefile + self.assertTrue(statefile.endswith('foo.state')) + + def test_del_statefile(self): + self.state._set_statefile('baz.state') + self.state._del_statefile() + statefile = self.state._get_statefile() + self.assertIsNone(statefile) + def test_save(self): self.state.save() self.savedStateAssertions() @@ -93,5 +132,32 @@ class StateSaveAndLoadTests(unittest.TestCase): def test_load(self): self.state.save() - loadedState = persistent.load() + loadedState = persistent.load() + self.loadedStateAssertions(loadedState) + + def test_load_with_state(self): + loadedState = persistent.load(self.state) self.loadedStateAssertions(loadedState) + + def test_load_with_None(self): + persistent._setState(None) + self.assertRaises(persistent.MissingState, + persistent.load, None) + + def test_load_with_statefile(self): + self.assertRaises(persistent.MissingState, + self.state.load, 'quux.state') + + def test_load_with_statefile_opened(self): + fh = open('quux.state', 'w+') + self.assertRaises(persistent.MissingState, self.state.load, fh) + fh.close() + + def test_load_with_statefile_object(self): + self.assertRaises(persistent.MissingState, self.state.load, object) + + def test_load_without_statefile(self): + persistent._setState(None) + self.state.statefile = None + self.assertRaises(persistent.MissingState, + persistent.load)

1 0

[bridgedb/develop] Avoid having a test for b.persistent.State write to $HOME.
by isis＠torproject.org 01 Mar '15

01 Mar '15

commit 783a8f18e990608ebb915a0bcc4096b34646bc04 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Feb 24 22:56:50 2015 +0000 Avoid having a test for b.persistent.State write to $HOME. --- lib/bridgedb/test/test_persistentSaveAndLoad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/test/test_persistentSaveAndLoad.py b/lib/bridgedb/test/test_persistentSaveAndLoad.py index 67585f6..6d6584d 100644 --- a/lib/bridgedb/test/test_persistentSaveAndLoad.py +++ b/lib/bridgedb/test/test_persistentSaveAndLoad.py @@ -88,7 +88,7 @@ class StateSaveAndLoadTests(unittest.TestCase): self.assertIsInstance(statefile, basestring) def test_set_statefile(self): - self.state._set_statefile('~/bar.state') + self.state._set_statefile('bar.state') statefile = self.state._get_statefile() self.assertIsInstance(statefile, basestring)

1 0

[stem/master] Missing API doc resource
by atagar＠torproject.org 01 Mar '15

01 Mar '15

commit 45abe23c71c38bb57cf903d9e82cf68d2d554f2e Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Feb 28 21:17:50 2015 -0800 Missing API doc resource Oops, forgot to add this new file. --- docs/api/descriptor/hidden_service_descriptor.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/api/descriptor/hidden_service_descriptor.rst b/docs/api/descriptor/hidden_service_descriptor.rst new file mode 100644 index 0000000..145203e --- /dev/null +++ b/docs/api/descriptor/hidden_service_descriptor.rst @@ -0,0 +1,5 @@ +Hidden Service Descriptor +========================= + +.. automodule:: stem.descriptor.hidden_service_descriptor +

1 0

[stem/master] Initial hidden service descriptor parser
by atagar＠torproject.org 01 Mar '15

01 Mar '15

commit 380074b53311962690b92670bd640de697b8d9e2 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Feb 23 15:07:20 2015 -0800 Initial hidden service descriptor parser Quick initial parser for hidden service descriptors... https://trac.torproject.org/projects/tor/ticket/15004 Things left to do are... * Issue tickets for some of these 'TODO' notes. * Support the introduction-points field. It's a bit special. * Unit tests for malformed content. * Validate that the signature matches the descriptor content. --- stem/descriptor/__init__.py | 7 + stem/descriptor/hidden_service_descriptor.py | 157 ++++++++++++++++++++ test/settings.cfg | 1 + .../unit/descriptor/data/hidden_service_duckduckgo | 60 ++++++++ test/unit/descriptor/data/hidden_service_facebook | 60 ++++++++ test/unit/descriptor/hidden_service_descriptor.py | 114 ++++++++++++++ 6 files changed, 399 insertions(+) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index abcdab2..87f7f73 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -122,6 +122,7 @@ def parse_file(descriptor_file, descriptor_type = None, validate = False, docume torperf 1.0 **unsupported** bridge-pool-assignment 1.0 **unsupported** tordnsel 1.0 :class:`~stem.descriptor.tordnsel.TorDNSEL` + hidden-service-descriptor 1.0 :class:`~stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor` ========================================= ===== If you're using **python 3** then beware that the open() function defaults to @@ -307,6 +308,11 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto for desc in stem.descriptor.tordnsel._parse_file(descriptor_file, validate = validate, **kwargs): yield desc + elif descriptor_type == 'hidden-service-descriptor' and major_version == 1: + document_type = stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor + + for desc in stem.descriptor.hidden_service_descriptor._parse_file(descriptor_file, validate = validate, **kwargs): + yield desc else: raise TypeError("Unrecognized metrics descriptor format. type: '%s', version: '%i.%i'" % (descriptor_type, major_version, minor_version)) @@ -720,3 +726,4 @@ import stem.descriptor.extrainfo_descriptor import stem.descriptor.networkstatus import stem.descriptor.microdescriptor import stem.descriptor.tordnsel +import stem.descriptor.hidden_service_descriptor diff --git a/stem/descriptor/hidden_service_descriptor.py b/stem/descriptor/hidden_service_descriptor.py new file mode 100644 index 0000000..c915f0e --- /dev/null +++ b/stem/descriptor/hidden_service_descriptor.py @@ -0,0 +1,157 @@ +# Copyright 2015, Damian Johnson and The Tor Project +# See LICENSE for licensing information + +""" +Parsing for Tor hidden service descriptors as described in Tor's `rend-spec +<https://gitweb.torproject.org/torspec.git/tree/rend-spec.txt>`_. + +Unlike other descriptor types these describe a hidden service rather than a +relay. They're created by the service, and can only be fetched via relays with +the HSDir flag. + +**Module Overview:** + +:: + + HiddenServiceDescriptor - Tor hidden service descriptor. +""" + +# TODO: Add a description for how to retrieve them when tor supports that (#14847). + +# TODO: We should add a '@type hidden-service-descriptor 1.0' annotation to +# CollecTor... +# +# https://collector.torproject.org/formats.html + +from stem.descriptor import ( + PGP_BLOCK_END, + Descriptor, + _get_descriptor_components, + _read_until_keywords, + _value, + _parse_simple_line, + _parse_timestamp_line, + _parse_key_block, +) + +REQUIRED_FIELDS = ( + 'rendezvous-service-descriptor', + 'version', + 'permanent-key', + 'secret-id-part', + 'publication-time', + 'protocol-versions', + 'signature', +) + + +def _parse_file(descriptor_file, validate = False, **kwargs): + """ + Iterates over the hidden service descriptors in a file. + + :param file descriptor_file: file with descriptor content + :param bool validate: checks the validity of the descriptor's content if + **True**, skips these checks otherwise + :param dict kwargs: additional arguments for the descriptor constructor + + :returns: iterator for :class:`~stem.descriptor.hidden_service_descriptor.HiddenServiceDescriptor` + instances in the file + + :raises: + * **ValueError** if the contents is malformed and validate is **True** + * **IOError** if the file can't be read + """ + + while True: + descriptor_content = _read_until_keywords('signature', descriptor_file) + + # we've reached the 'signature', now include the pgp style block + block_end_prefix = PGP_BLOCK_END.split(' ', 1)[0] + descriptor_content += _read_until_keywords(block_end_prefix, descriptor_file, True) + + if descriptor_content: + if descriptor_content[0].startswith(b'@type'): + descriptor_content = descriptor_content[1:] + + yield HiddenServiceDescriptor(bytes.join(b'', descriptor_content), validate, **kwargs) + else: + break # done parsing file + + +# TODO: For the 'version' and 'protocol-versions' lines should they be ints +# instead? Presently the spec says it's a 'version-number' which is an +# undefined type. + +def _parse_protocol_versions_line(descriptor, entries): + value = _value('protocol-versions', entries) + descriptor.protocol_versions = value.split(',') + +_parse_rendezvous_service_descriptor_line = _parse_simple_line('rendezvous-service-descriptor', 'descriptor_id') +_parse_version_line = _parse_simple_line('version', 'version') +_parse_permanent_key_line = _parse_key_block('permanent-key', 'permanent_key', 'RSA PUBLIC KEY') +_parse_secret_id_part_line = _parse_simple_line('secret-id-part', 'secret_id_part') +_parse_publication_time_line = _parse_timestamp_line('publication-time', 'published') +_parse_introduction_points_line = _parse_key_block('introduction-points', 'introduction_points_blob', 'MESSAGE') +_parse_signature_line = _parse_key_block('signature', 'signature', 'SIGNATURE') + + +class HiddenServiceDescriptor(Descriptor): + """ + Hidden service descriptor. + + :var str descriptor_id: **\*** identifier for this descriptor, this is a base32 hash of several fields + :var str version: **\*** hidden service descriptor version + :var str permanent_key: **\*** long term key of the hidden service + :var str secret_id_part: **\*** hash of the time period, cookie, and replica + values so our descriptor_id can be validated + :var datetime published: **\*** time in UTC when this descriptor was made + :var list protocol_versions: **\*** versions that are supported when establishing a connection + :var str introduction_points_blob: **\*** raw introduction points blob, if + the hidden service uses cookie authentication this is encrypted + :var str signature: signature of the descriptor content + + **\*** attribute is either required when we're parsed with validation or has + a default value, others are left as **None** if undefined + """ + + ATTRIBUTES = { + 'descriptor_id': (None, _parse_rendezvous_service_descriptor_line), + 'version': (None, _parse_version_line), + 'permanent_key': (None, _parse_permanent_key_line), + 'secret_id_part': (None, _parse_secret_id_part_line), + 'published': (None, _parse_publication_time_line), + 'protocol_versions': ([], _parse_protocol_versions_line), + 'introduction_points_blob': (None, _parse_introduction_points_line), + 'signature': (None, _parse_signature_line), + } + + PARSER_FOR_LINE = { + 'rendezvous-service-descriptor': _parse_rendezvous_service_descriptor_line, + 'version': _parse_version_line, + 'permanent-key': _parse_permanent_key_line, + 'secret-id-part': _parse_secret_id_part_line, + 'publication-time': _parse_publication_time_line, + 'protocol-versions': _parse_protocol_versions_line, + 'introduction-points': _parse_introduction_points_line, + 'signature': _parse_signature_line, + } + + def __init__(self, raw_contents, validate = False): + super(HiddenServiceDescriptor, self).__init__(raw_contents, lazy_load = not validate) + entries = _get_descriptor_components(raw_contents, validate) + + if validate: + for keyword in REQUIRED_FIELDS: + if keyword not in entries: + raise ValueError("Hidden service descriptor must have a '%s' entry" % keyword) + elif keyword in entries and len(entries[keyword]) > 1: + raise ValueError("The '%s' entry can only appear once in a hidden service descriptor" % keyword) + + if 'rendezvous-service-descriptor' != list(entries.keys())[0]: + raise ValueError("Hidden service descriptor must start with a 'rendezvous-service-descriptor' entry") + elif 'signature' != list(entries.keys())[-1]: + raise ValueError("Hidden service descriptor must end with a 'signature' entry") + + self._parse(entries, validate) + else: + self._entries = entries diff --git a/test/settings.cfg b/test/settings.cfg index c0c8342..21f5eb2 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -172,6 +172,7 @@ test.unit_tests |test.unit.descriptor.networkstatus.document_v2.TestNetworkStatusDocument |test.unit.descriptor.networkstatus.document_v3.TestNetworkStatusDocument |test.unit.descriptor.networkstatus.bridge_document.TestBridgeNetworkStatusDocument +|test.unit.descriptor.hidden_service_descriptor.TestHiddenServiceDescriptor |test.unit.exit_policy.rule.TestExitPolicyRule |test.unit.exit_policy.policy.TestExitPolicy |test.unit.version.TestVersion diff --git a/test/unit/descriptor/data/hidden_service_duckduckgo b/test/unit/descriptor/data/hidden_service_duckduckgo new file mode 100644 index 0000000..f3f4868 --- /dev/null +++ b/test/unit/descriptor/data/hidden_service_duckduckgo @@ -0,0 +1,60 @@ +@type hidden-service-descriptor 1.0 +rendezvous-service-descriptor y3olqqblqw2gbh6phimfuiroechjjafa +version 2 +permanent-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAJ/SzzgrXPxTlFrKVhXh3buCWv2QfcNgncUpDpKouLn3AtPH5Ocys0jE +aZSKdvaiQ62md2gOwj4x61cFNdi05tdQjS+2thHKEm/KsB9BGLSLBNJYY356bupg +I5gQozM65ENelfxYlysBjJ52xSDBd8C4f/p9umdzaaaCmzXG/nhzAgMBAAE= +-----END RSA PUBLIC KEY----- +secret-id-part e24kgecavwsznj7gpbktqsiwgvngsf4e +publication-time 2015-02-23 20:00:00 +protocol-versions 2,3 +introduction-points +-----BEGIN MESSAGE----- +aW50cm9kdWN0aW9uLXBvaW50IGl3a2k3N3h0YnZwNnF2ZWRmcndkem5jeHMzY2th +eWV1CmlwLWFkZHJlc3MgMTc4LjYyLjIyMi4xMjkKb25pb24tcG9ydCA0NDMKb25p +b24ta2V5Ci0tLS0tQkVHSU4gUlNBIFBVQkxJQyBLRVktLS0tLQpNSUdKQW9HQkFL +OTRCRVlJSFo0S2RFa2V5UGhiTENwUlc1RVNnKzJXUFFock00eXVLWUd1cTh3Rldn +dW1aWVI5CmsvV0EvL0ZZWE1CejBiQitja3Vacy9ZdTluSytITHpwR2FwVjBjbHN0 +NEdVTWNCSW5VQ3pDY3BqSlRRc1FEZ20KMy9ZM2NxaDBXNTVnT0NGaG9tUTQvMVdP +WWc3WUNqazRYWUhKRTIwT2RHMkxsNXpvdEs2ZkFnTUJBQUU9Ci0tLS0tRU5EIFJT +QSBQVUJMSUMgS0VZLS0tLS0Kc2VydmljZS1rZXkKLS0tLS1CRUdJTiBSU0EgUFVC +TElDIEtFWS0tLS0tCk1JR0pBb0dCQUpYbUpiOGxTeWRNTXFDZ0NnZmd2bEIyRTVy +cGQ1N2t6L0FxZzcvZDFIS2MzK2w1UW9Vdkh5dXkKWnNBbHlrYThFdTUzNGhsNDFv +cUVLcEFLWWNNbjFUTTB2cEpFR05WT2MrMDVCSW54STloOWYwTWcwMVBEMHRZdQpH +Y0xIWWdCemNyZkVtS3dNdE04V0VtY01KZDduMnVmZmFBdko4NDZXdWJiZVY3TVcx +WWVoQWdNQkFBRT0KLS0tLS1FTkQgUlNBIFBVQkxJQyBLRVktLS0tLQppbnRyb2R1 +Y3Rpb24tcG9pbnQgZW00Z2prNmVpaXVhbGhtbHlpaWZyemM3bGJ0cnNiaXAKaXAt +YWRkcmVzcyA0Ni40LjE3NC41Mgpvbmlvbi1wb3J0IDQ0Mwpvbmlvbi1rZXkKLS0t +LS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tCk1JR0pBb0dCQUxCbWhkRjV3SHhI +cnBMU21qQVpvdHR4MjIwKzk5NUZkTU9PdFpOalJ3MURCU3ByVVpacXR4V2EKUDhU +S3BIS3p3R0pLQ1ZZSUlqN2xvaGJ2OVQ5dXJtbGZURTA1VVJHZW5ab2lmT0ZOejNZ +d01KVFhTY1FFQkoxMAo5aVdOTERUc2tMekRLQ0FiR2hibi9NS3dPZllHQmhOVGxq +ZHlUbU5ZNUVDUmJSempldjl2QWdNQkFBRT0KLS0tLS1FTkQgUlNBIFBVQkxJQyBL +RVktLS0tLQpzZXJ2aWNlLWtleQotLS0tLUJFR0lOIFJTQSBQVUJMSUMgS0VZLS0t +LS0KTUlHSkFvR0JBTXhNSG9BbXJiVU1zeGlJQ3AzaVRQWWdobjBZdWVLSHgyMTl3 +dThPL1E1MVF5Y1ZWTHBYMjdkMQpoSlhrUEIzM1hRQlhzQlM3U3hzU3NTQ1EzR0V1 +clFKN0d1QkxwWUlSL3Zxc2FrRS9sOHdjMkNKQzVXVWh5RkZrCisxVFdJVUk1dHhu +WEx5V0NSY0tEVXJqcWRvc0RhRG9zZ0hGZzIzTW54K3hYY2FRL2ZyQi9BZ01CQUFF +PQotLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tCmludHJvZHVjdGlvbi1wb2lu +dCBqcWhmbDM2NHgzdXBlNmxxbnhpem9sZXdsZnJzdzJ6eQppcC1hZGRyZXNzIDYy +LjIxMC44Mi4xNjkKb25pb24tcG9ydCA0NDMKb25pb24ta2V5Ci0tLS0tQkVHSU4g +UlNBIFBVQkxJQyBLRVktLS0tLQpNSUdKQW9HQkFQVWtxeGdmWWR3MFBtL2c2TWJo +bVZzR0tsdWppZm1raGRmb0VldXpnbyt3bkVzR3Z3VWVienJ6CmZaSlJ0MGNhWEZo +bkNHZ1FEMklnbWFyVWFVdlAyNGZYby80bVl6TGNQZUk3Z1puZXVBUUpZdm05OFl2 +OXZPSGwKTmFNL1d2RGtDc0ozR1ZOSjFIM3dMUFFSSTN2N0tiTnVjOXRDT1lsL3Iw +OU9oVmFXa3phakFnTUJBQUU9Ci0tLS0tRU5EIFJTQSBQVUJMSUMgS0VZLS0tLS0K +c2VydmljZS1rZXkKLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tCk1JR0pB +b0dCQUxieDhMZXFSb1Avcjl3OWhqd0Q0MVlVbTdQbzY5N3hSdHl0RjBNY3lMQ1M3 +R1JpVVluamk3S1kKZmVwWGR2Ti9KbDVxUUtISUJiNjAya3VPVGwwcE44UStZZUZV +U0lJRGNtUEJMcEJEaEgzUHZyUU1jR1ZhaU9XSAo4dzBITVpDeGd3QWNDQzUxdzVW +d2l1bXhFSk5CVmNac094MG16TjFDbG95KzkwcTBsRlhMQWdNQkFBRT0KLS0tLS1F +TkQgUlNBIFBVQkxJQyBLRVktLS0tLQoK +-----END MESSAGE----- +signature +-----BEGIN SIGNATURE----- +VKMmsDIUUFOrpqvcQroIZjDZTKxqNs88a4M9Te8cR/ZvS7H2nffv6iQs0tom5X4D +4Dy4iZiy+pwYxdHfaOxmdpgMCRvgPb34MExWr5YemH0QuGtnlp5Wxr8GYaAQVuZX +cZjQLW0juUYCbgIGdxVEBnlEt2rgBSM9+1oR7EAfV1U= +-----END SIGNATURE----- diff --git a/test/unit/descriptor/data/hidden_service_facebook b/test/unit/descriptor/data/hidden_service_facebook new file mode 100644 index 0000000..705c776 --- /dev/null +++ b/test/unit/descriptor/data/hidden_service_facebook @@ -0,0 +1,60 @@ +@type hidden-service-descriptor 1.0 +rendezvous-service-descriptor utjk4arxqg6s6zzo7n6cjnq6ot34udhr +version 2 +permanent-key +-----BEGIN RSA PUBLIC KEY----- +MIGKAoGBAL7zXRnwSycHlKSqK+B8PFvy7RhkQ+OytYtxBwOLzhD82oY7zbpjSHY4 +BZ+hsnceXVjB+f1mXGjvLY6pnYxuufV4wsMsk7a58aJOqUvZFFI2vXhJtnLICxoZ +AEHWzajz4ULagahB1Vi62loQE84OEcuFBekTvnHca1ZTxwp16aZtAgQvoiLZ +-----END RSA PUBLIC KEY----- +secret-id-part 6355jaerje3bqozopwq2qmpf4iviizdn +publication-time 2014-10-31 23:00:00 +protocol-versions 2,3 +introduction-points +-----BEGIN MESSAGE----- +aW50cm9kdWN0aW9uLXBvaW50IHJ6Y3V3am5jbDdpYXpkbm41NXV5cmEybWVlamxz +eXVoCmlwLWFkZHJlc3MgMTkzLjExLjExNC40Nwpvbmlvbi1wb3J0IDkwMDQKb25p +b24ta2V5Ci0tLS0tQkVHSU4gUlNBIFBVQkxJQyBLRVktLS0tLQpNSUdKQW9HQkFM +dndQc012TkhqMTY3YlJyLzlLbi9iVGtPUTN0VC9qOFdSOWV5NTlZU2NBVEQ4TXVz +OEV6ZWpNCjc0RjdhTGR1VjZuRk15djlFYmhEbVNIZDZRMmhwNkYwb2FGODB0MHMv +bThXYmVWTUF1aTRvVWRSU1ZRb0drY20KTDJXTlViNy84UWYwb2hFKzZ1K1pTL0FO +U2NEc0FDT2hwNkliMWxHWjhaZGxWZFJzeWxtckFnTUJBQUU9Ci0tLS0tRU5EIFJT +QSBQVUJMSUMgS0VZLS0tLS0Kc2VydmljZS1rZXkKLS0tLS1CRUdJTiBSU0EgUFVC +TElDIEtFWS0tLS0tCk1JR0pBb0dCQU1YVUJRMVd0cXJSZHRkaTdhODkvWU9EQUJ6 +d2U2L0JSS3dEcGFBUENyMC9CZlppVnFadXFFMzcKYmlxbW1pODBPVm5uQzd6eis3 +cDZYQ1QycitEemxtSmFCOGdzTjZCZlU2ek45Wklwd3ptcm1XTy94SGNoZ1BkOQpK +SUNrSUxHWEJlQnorNmtnSTZiZDcrbFZEYmxJYXNRNHkrUjhMWWxTeCtRMk9mK1gy +eDFwQWdNQkFBRT0KLS0tLS1FTkQgUlNBIFBVQkxJQyBLRVktLS0tLQppbnRyb2R1 +Y3Rpb24tcG9pbnQgcXE0YXZmdnZhcHljMnhyNmNpYmpuaXFoNTdpY2FkMnEKaXAt +YWRkcmVzcyAxODguNDAuODMuMjA4Cm9uaW9uLXBvcnQgNDQ0Cm9uaW9uLWtleQot +LS0tLUJFR0lOIFJTQSBQVUJMSUMgS0VZLS0tLS0KTUlHSkFvR0JBTWZySUJ1TlVF +MGZxa3ZySDlEZzJkNkM5SFUvNWR2TGxQL0NGVGFFYlVhMURvUHhyQm5iQ1BEcQp5 +bHhwNGs3M04yVDM3aXJrN3FacmVZQ3B6aUVPQ3paRU4xdGlkM0t1c0tFczgzL0FQ +aDFreTVRRmpkRmpBU0NxCmovRDl0VXNYYkVaa1FEN203SUlmWU40SG52KzROM0dV +WjVqNkhFMWJvYmxsbURyNkd2U0xBZ01CQUFFPQotLS0tLUVORCBSU0EgUFVCTElD +IEtFWS0tLS0tCnNlcnZpY2Uta2V5Ci0tLS0tQkVHSU4gUlNBIFBVQkxJQyBLRVkt +LS0tLQpNSUdKQW9HQkFNRXMyc2tRLzJsVjRFUGpRTDFrZk0zZjExUzRNZUdPcDFC +aWgrcm1ESGhic09yWDRoZk1GV1NuCjY1dTlZUE9zY1hldUlHbFp4YmFyUUY2Nzkz +bGMrbk9KbjVyQ016a0pYNWt0OHdobkZoVlFXN0ZoejUxRmdMNFkKNUFKKzdkaW9l +WTNKdjVNVE1rMUF4Qk1hbkJsR2YxazZrU05kZWxtMCs2bWtZVmVOcVJzckFnTUJB +QUU9Ci0tLS0tRU5EIFJTQSBQVUJMSUMgS0VZLS0tLS0KaW50cm9kdWN0aW9uLXBv +aW50IGI1ZHNpY29oZ2Fqb3Vhc3VxZ3lyNWtwNXIzdzdpaWYyCmlwLWFkZHJlc3Mg +MTg1LjUuNTMuNzIKb25pb24tcG9ydCA5MDAxCm9uaW9uLWtleQotLS0tLUJFR0lO +IFJTQSBQVUJMSUMgS0VZLS0tLS0KTUlHSkFvR0JBTWVUc0VqS1ZMeGsyL3o4bnRG +T2xWU09ZTWFjWTZBeFJTTXdQdkxoT2FsajVkNkRraitnSVA5agpsWDFoekRpZ0VP +b2FBV3FwWjRjRENLa3p6UGdzWEc3ZXVnSmFxTGQydUliZmh4WEdISTJDNWtWQWZZ +VnVyL1N3CnBXNFRRbmhKUytvbG1BWG84b1hvT2JNWGt1YURUdmlLSXZLa3lQWCs0 +dGJ4a1RxWWt5ZURBZ01CQUFFPQotLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0t +CnNlcnZpY2Uta2V5Ci0tLS0tQkVHSU4gUlNBIFBVQkxJQyBLRVktLS0tLQpNSUdK +QW9HQkFPM05SSGMwTld5MFo5aFQrZEUxZ3hKRnFsb0dhdmdLYzZJZ0E1UU5ORDNO +RHJJakdBYmNsRVh4Ck1tY1BBR2hhc1VlNnJ2aFN0VlRkZEVvN1V0SElCM3F2YXNL +d0NPem82Z2dLSHZtVVRFOWRNTk9LSEcyLy9xTi8KVkducGdJUkRoeDRwbE9DOXV6 +NGFKU0RZdlhiSXVtZmtxdGJzb1BacUZMYUFqdGFWR015VkFnTUJBQUU9Ci0tLS0t +RU5EIFJTQSBQVUJMSUMgS0VZLS0tLS0KCg== +-----END MESSAGE----- +signature +-----BEGIN SIGNATURE----- +tFCNLEOFeqeDZOruKt4SBGvJ2Y2lLo29XUzd09FlIalnwb71TKcKSWi1JVB0vDqF +ftGlP4M+nZGh6YGxdEriV0ikFbcW+F4XOpWR5NEPV5cCoQAMd1a8mqivGjM/lQwe +41p+e4XCex2Brsl3o+pSyHy/U+p+xdQkmNPqrHpXVK8= +-----END SIGNATURE----- diff --git a/test/unit/descriptor/hidden_service_descriptor.py b/test/unit/descriptor/hidden_service_descriptor.py new file mode 100644 index 0000000..579c549 --- /dev/null +++ b/test/unit/descriptor/hidden_service_descriptor.py @@ -0,0 +1,114 @@ +""" +Unit tests for stem.descriptor.hidden_service_descriptor. +""" + +import datetime +import unittest + +import stem.descriptor + +from test.unit.descriptor import get_resource + +EXPECTED_DDG_PERMANENT_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAJ/SzzgrXPxTlFrKVhXh3buCWv2QfcNgncUpDpKouLn3AtPH5Ocys0jE +aZSKdvaiQ62md2gOwj4x61cFNdi05tdQjS+2thHKEm/KsB9BGLSLBNJYY356bupg +I5gQozM65ENelfxYlysBjJ52xSDBd8C4f/p9umdzaaaCmzXG/nhzAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECTED_DDG_INTRODUCTION_POINTS_BLOB = """\ +-----BEGIN MESSAGE----- +aW50cm9kdWN0aW9uLXBvaW50IGl3a2k3N3h0YnZwNnF2ZWRmcndkem5jeHMzY2th +eWV1CmlwLWFkZHJlc3MgMTc4LjYyLjIyMi4xMjkKb25pb24tcG9ydCA0NDMKb25p +b24ta2V5Ci0tLS0tQkVHSU4gUlNBIFBVQkxJQyBLRVktLS0tLQpNSUdKQW9HQkFL +OTRCRVlJSFo0S2RFa2V5UGhiTENwUlc1RVNnKzJXUFFock00eXVLWUd1cTh3Rldn +dW1aWVI5CmsvV0EvL0ZZWE1CejBiQitja3Vacy9ZdTluSytITHpwR2FwVjBjbHN0 +NEdVTWNCSW5VQ3pDY3BqSlRRc1FEZ20KMy9ZM2NxaDBXNTVnT0NGaG9tUTQvMVdP +WWc3WUNqazRYWUhKRTIwT2RHMkxsNXpvdEs2ZkFnTUJBQUU9Ci0tLS0tRU5EIFJT +QSBQVUJMSUMgS0VZLS0tLS0Kc2VydmljZS1rZXkKLS0tLS1CRUdJTiBSU0EgUFVC +TElDIEtFWS0tLS0tCk1JR0pBb0dCQUpYbUpiOGxTeWRNTXFDZ0NnZmd2bEIyRTVy +cGQ1N2t6L0FxZzcvZDFIS2MzK2w1UW9Vdkh5dXkKWnNBbHlrYThFdTUzNGhsNDFv +cUVLcEFLWWNNbjFUTTB2cEpFR05WT2MrMDVCSW54STloOWYwTWcwMVBEMHRZdQpH +Y0xIWWdCemNyZkVtS3dNdE04V0VtY01KZDduMnVmZmFBdko4NDZXdWJiZVY3TVcx +WWVoQWdNQkFBRT0KLS0tLS1FTkQgUlNBIFBVQkxJQyBLRVktLS0tLQppbnRyb2R1 +Y3Rpb24tcG9pbnQgZW00Z2prNmVpaXVhbGhtbHlpaWZyemM3bGJ0cnNiaXAKaXAt +YWRkcmVzcyA0Ni40LjE3NC41Mgpvbmlvbi1wb3J0IDQ0Mwpvbmlvbi1rZXkKLS0t +LS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tCk1JR0pBb0dCQUxCbWhkRjV3SHhI +cnBMU21qQVpvdHR4MjIwKzk5NUZkTU9PdFpOalJ3MURCU3ByVVpacXR4V2EKUDhU +S3BIS3p3R0pLQ1ZZSUlqN2xvaGJ2OVQ5dXJtbGZURTA1VVJHZW5ab2lmT0ZOejNZ +d01KVFhTY1FFQkoxMAo5aVdOTERUc2tMekRLQ0FiR2hibi9NS3dPZllHQmhOVGxq +ZHlUbU5ZNUVDUmJSempldjl2QWdNQkFBRT0KLS0tLS1FTkQgUlNBIFBVQkxJQyBL +RVktLS0tLQpzZXJ2aWNlLWtleQotLS0tLUJFR0lOIFJTQSBQVUJMSUMgS0VZLS0t +LS0KTUlHSkFvR0JBTXhNSG9BbXJiVU1zeGlJQ3AzaVRQWWdobjBZdWVLSHgyMTl3 +dThPL1E1MVF5Y1ZWTHBYMjdkMQpoSlhrUEIzM1hRQlhzQlM3U3hzU3NTQ1EzR0V1 +clFKN0d1QkxwWUlSL3Zxc2FrRS9sOHdjMkNKQzVXVWh5RkZrCisxVFdJVUk1dHhu +WEx5V0NSY0tEVXJqcWRvc0RhRG9zZ0hGZzIzTW54K3hYY2FRL2ZyQi9BZ01CQUFF +PQotLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tCmludHJvZHVjdGlvbi1wb2lu +dCBqcWhmbDM2NHgzdXBlNmxxbnhpem9sZXdsZnJzdzJ6eQppcC1hZGRyZXNzIDYy +LjIxMC44Mi4xNjkKb25pb24tcG9ydCA0NDMKb25pb24ta2V5Ci0tLS0tQkVHSU4g +UlNBIFBVQkxJQyBLRVktLS0tLQpNSUdKQW9HQkFQVWtxeGdmWWR3MFBtL2c2TWJo +bVZzR0tsdWppZm1raGRmb0VldXpnbyt3bkVzR3Z3VWVienJ6CmZaSlJ0MGNhWEZo +bkNHZ1FEMklnbWFyVWFVdlAyNGZYby80bVl6TGNQZUk3Z1puZXVBUUpZdm05OFl2 +OXZPSGwKTmFNL1d2RGtDc0ozR1ZOSjFIM3dMUFFSSTN2N0tiTnVjOXRDT1lsL3Iw +OU9oVmFXa3phakFnTUJBQUU9Ci0tLS0tRU5EIFJTQSBQVUJMSUMgS0VZLS0tLS0K +c2VydmljZS1rZXkKLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tCk1JR0pB +b0dCQUxieDhMZXFSb1Avcjl3OWhqd0Q0MVlVbTdQbzY5N3hSdHl0RjBNY3lMQ1M3 +R1JpVVluamk3S1kKZmVwWGR2Ti9KbDVxUUtISUJiNjAya3VPVGwwcE44UStZZUZV +U0lJRGNtUEJMcEJEaEgzUHZyUU1jR1ZhaU9XSAo4dzBITVpDeGd3QWNDQzUxdzVW +d2l1bXhFSk5CVmNac094MG16TjFDbG95KzkwcTBsRlhMQWdNQkFBRT0KLS0tLS1F +TkQgUlNBIFBVQkxJQyBLRVktLS0tLQoK +-----END MESSAGE-----\ +""" + +EXPECTED_DDG_SIGNATURE = """\ +-----BEGIN SIGNATURE----- +VKMmsDIUUFOrpqvcQroIZjDZTKxqNs88a4M9Te8cR/ZvS7H2nffv6iQs0tom5X4D +4Dy4iZiy+pwYxdHfaOxmdpgMCRvgPb34MExWr5YemH0QuGtnlp5Wxr8GYaAQVuZX +cZjQLW0juUYCbgIGdxVEBnlEt2rgBSM9+1oR7EAfV1U= +-----END SIGNATURE-----\ +""" + + +class TestHiddenServiceDescriptor(unittest.TestCase): + def test_for_duckduckgo_with_validation(self): + """ + Parse duckduckgo's descriptor. + """ + + descriptor_file = open(get_resource('hidden_service_duckduckgo'), 'rb') + desc = next(stem.descriptor.parse_file(descriptor_file, 'hidden-service-descriptor 1.0', validate = True)) + self._assert_matches_duckduckgo(desc) + + def test_for_duckduckgo_without_validation(self): + """ + Parse duckduckgo's descriptor + """ + + descriptor_file = open(get_resource('hidden_service_duckduckgo'), 'rb') + desc = next(stem.descriptor.parse_file(descriptor_file, 'hidden-service-descriptor 1.0', validate = False)) + self._assert_matches_duckduckgo(desc) + + def test_for_facebook(self): + """ + Parse facebook's descriptor. + """ + + descriptor_file = open(get_resource('hidden_service_facebook'), 'rb') + + desc = next(stem.descriptor.parse_file(descriptor_file, 'hidden-service-descriptor 1.0', validate = True)) + self.assertEqual('utjk4arxqg6s6zzo7n6cjnq6ot34udhr', desc.descriptor_id) + self.assertEqual('2', desc.version) + self.assertEqual('6355jaerje3bqozopwq2qmpf4iviizdn', desc.secret_id_part) + self.assertEqual(datetime.datetime(2014, 10, 31, 23, 0, 0), desc.published) + self.assertEqual(['2', '3'], desc.protocol_versions) + + def _assert_matches_duckduckgo(self, desc): + self.assertEqual('y3olqqblqw2gbh6phimfuiroechjjafa', desc.descriptor_id) + self.assertEqual('2', desc.version) + self.assertEqual(EXPECTED_DDG_PERMANENT_KEY, desc.permanent_key) + self.assertEqual('e24kgecavwsznj7gpbktqsiwgvngsf4e', desc.secret_id_part) + self.assertEqual(datetime.datetime(2015, 2, 23, 20, 0, 0), desc.published) + self.assertEqual(['2', '3'], desc.protocol_versions) + self.assertEqual(EXPECTED_DDG_INTRODUCTION_POINTS_BLOB, desc.introduction_points_blob) + self.assertEqual(EXPECTED_DDG_SIGNATURE, desc.signature)

1 0

[stem/master] Support hidden service introduction-points
by atagar＠torproject.org 01 Mar '15

01 Mar '15

commit ca21828a88d56fbe7dfa7d4b78ec4f99e0bc8343 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Feb 23 19:32:53 2015 -0800 Support hidden service introduction-points This is a very unusual attribute in that it's a base64 encoded blob of two subdocuments: the first optional authentication data, followed by a list of subdocuments. Parsing the authentication data when reading the field, but leaving the second pass to a method since it may optionally be encrypted. --- stem/descriptor/hidden_service_descriptor.py | 140 ++++++++++++++++++++- stem/util/connection.py | 4 +- test/unit/descriptor/hidden_service_descriptor.py | 130 ++++++++++++++++++- 3 files changed, 266 insertions(+), 8 deletions(-) diff --git a/stem/descriptor/hidden_service_descriptor.py b/stem/descriptor/hidden_service_descriptor.py index c915f0e..d626144 100644 --- a/stem/descriptor/hidden_service_descriptor.py +++ b/stem/descriptor/hidden_service_descriptor.py @@ -23,6 +23,14 @@ the HSDir flag. # # https://collector.torproject.org/formats.html +import base64 +import collections +import io + +import stem.util.connection + +from stem import str_type + from stem.descriptor import ( PGP_BLOCK_END, Descriptor, @@ -34,6 +42,12 @@ from stem.descriptor import ( _parse_key_block, ) +try: + # added in python 3.2 + from functools import lru_cache +except ImportError: + from stem.util.lru_cache import lru_cache + REQUIRED_FIELDS = ( 'rendezvous-service-descriptor', 'version', @@ -44,6 +58,23 @@ REQUIRED_FIELDS = ( 'signature', ) +INTRODUCTION_POINTS_ATTR = { + 'identifier': None, + 'address': None, + 'port': None, + 'onion_key': None, + 'service_key': None, + 'intro_authentication': [], +} + +IntroductionPoint = collections.namedtuple('IntroductionPoints', INTRODUCTION_POINTS_ATTR.keys()) + + +class DecryptionFailure(Exception): + """ + Failure to decrypt the hidden service descriptor's introduction-points. + """ + def _parse_file(descriptor_file, validate = False, **kwargs): """ @@ -86,12 +117,37 @@ def _parse_protocol_versions_line(descriptor, entries): value = _value('protocol-versions', entries) descriptor.protocol_versions = value.split(',') + +def _parse_introduction_points_line(descriptor, entries): + _, block_type, block_contents = entries['introduction-points'][0] + + if not block_contents or block_type != 'MESSAGE': + raise ValueError("'introduction-points' should be followed by a MESSAGE block, but was a %s" % block_type) + + descriptor.introduction_points_encoded = block_contents + + blob = ''.join(block_contents.split('\n')[1:-1]) + decoded_field = base64.b64decode(stem.util.str_tools._to_bytes(blob)) + + auth_types = [] + + while decoded_field.startswith('service-authentication ') and '\n' in decoded_field: + auth_line, decoded_field = decoded_field.split('\n', 1) + auth_line_comp = auth_line.split(' ') + + if len(auth_line_comp) < 3: + raise ValueError("Within introduction-points we expected 'service-authentication [auth_type] [auth_data]', but had '%s'" % auth_line) + + auth_types.append((auth_line_comp[1], auth_line_comp[2])) + + descriptor.introduction_points_auth = auth_types + descriptor.introduction_points_content = decoded_field + _parse_rendezvous_service_descriptor_line = _parse_simple_line('rendezvous-service-descriptor', 'descriptor_id') _parse_version_line = _parse_simple_line('version', 'version') _parse_permanent_key_line = _parse_key_block('permanent-key', 'permanent_key', 'RSA PUBLIC KEY') _parse_secret_id_part_line = _parse_simple_line('secret-id-part', 'secret_id_part') _parse_publication_time_line = _parse_timestamp_line('publication-time', 'published') -_parse_introduction_points_line = _parse_key_block('introduction-points', 'introduction_points_blob', 'MESSAGE') _parse_signature_line = _parse_key_block('signature', 'signature', 'SIGNATURE') @@ -106,8 +162,12 @@ class HiddenServiceDescriptor(Descriptor): values so our descriptor_id can be validated :var datetime published: **\*** time in UTC when this descriptor was made :var list protocol_versions: **\*** versions that are supported when establishing a connection - :var str introduction_points_blob: **\*** raw introduction points blob, if - the hidden service uses cookie authentication this is encrypted + :var str introduction_points_encoded: **\*** raw introduction points blob + :var list introduction_points_auth: **\*** tuples of the form + (auth_method, auth_data) for our introduction_points_content + :var str introduction_points_content: **\*** decoded introduction-points + content without authentication data, if using cookie authentication this is + encrypted :var str signature: signature of the descriptor content **\*** attribute is either required when we're parsed with validation or has @@ -121,7 +181,7 @@ class HiddenServiceDescriptor(Descriptor): 'secret_id_part': (None, _parse_secret_id_part_line), 'published': (None, _parse_publication_time_line), 'protocol_versions': ([], _parse_protocol_versions_line), - 'introduction_points_blob': (None, _parse_introduction_points_line), + 'introduction_points_encoded': (None, _parse_introduction_points_line), 'signature': (None, _parse_signature_line), } @@ -155,3 +215,75 @@ class HiddenServiceDescriptor(Descriptor): self._parse(entries, validate) else: self._entries = entries + + @lru_cache() + def introduction_points(self): + """ + Provided this service's introduction points. This provides a list of + IntroductionPoint instances, which have the following attributes... + + * identifier (str): hash of this introduction point's identity key + * address (str): address of this introduction point + * port (int): port where this introduction point is listening + * onion_key (str): public key for communicating with this introduction point + * service_key (str): public key for communicating with this hidden service + * intro_authentication (list): tuples of the form (auth_type, auth_data) + for establishing a connection + + :returns: **list** of IntroductionPoints instances + + :raises: + * **ValueError** if the our introduction-points is malformed + * **DecryptionFailure** if unable to decrypt this field + """ + + # TODO: Support fields encrypted with a desriptor-cookie. Need sample data + # to implement this. + + if not self.introduction_points_content.startswith('introduction-point '): + raise DecryptionFailure('introduction-point content is encrypted') + + introduction_points = [] + content_io = io.StringIO(str_type(self.introduction_points_content)) + + while True: + content = ''.join(_read_until_keywords('introduction-point', content_io, ignore_first = True)) + + if not content: + break # reached the end + + attr = dict(INTRODUCTION_POINTS_ATTR) + entries = _get_descriptor_components(content, False) + + # TODO: most fields can only appear once, we should check for that + + for keyword, values in list(entries.items()): + value, block_type, block_contents = values[0] + + if keyword == 'introduction-point': + attr['identifier'] = value + elif keyword == 'ip-address': + # TODO: need clarification about if this IPv4, IPv6, or both + attr['address'] = value + elif keyword == 'onion-port': + if not stem.util.connection.is_valid_port(value): + raise ValueError("'%s' is an invalid port" % value) + + attr['port'] = int(value) + elif keyword == 'onion-key': + attr['onion_key'] = block_contents + elif keyword == 'service-key': + attr['service_key'] = block_contents + elif keyword == 'intro-authentication': + auth_entries = [] + + for auth_value, _, _ in values: + if ' ' not in auth_value: + raise ValueError("We expected 'intro-authentication [auth_type] [auth_data]', but had '%s'" % auth_value) + + auth_type, auth_data = auth_value.split(' ')[:2] + auth_entries.append((auth_type, auth_data)) + + introduction_points.append(IntroductionPoint(**attr)) + + return introduction_points diff --git a/stem/util/connection.py b/stem/util/connection.py index fa2a3d5..402a45a 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -140,8 +140,8 @@ RESOLVER_FILTER = { def get_connections(resolver, process_pid = None, process_name = None): """ - Retrieves a list of the current connections for a given process. The provides - a list of Connection instances, which have five attributes... + Retrieves a list of the current connections for a given process. This + provides a list of Connection instances, which have five attributes... * local_address (str) * local_port (int) diff --git a/test/unit/descriptor/hidden_service_descriptor.py b/test/unit/descriptor/hidden_service_descriptor.py index 579c549..4853a28 100644 --- a/test/unit/descriptor/hidden_service_descriptor.py +++ b/test/unit/descriptor/hidden_service_descriptor.py @@ -17,7 +17,7 @@ I5gQozM65ENelfxYlysBjJ52xSDBd8C4f/p9umdzaaaCmzXG/nhzAgMBAAE= -----END RSA PUBLIC KEY-----\ """ -EXPECTED_DDG_INTRODUCTION_POINTS_BLOB = """\ +EXPECTED_DDG_INTRODUCTION_POINTS_ENCODED = """\ -----BEGIN MESSAGE----- aW50cm9kdWN0aW9uLXBvaW50IGl3a2k3N3h0YnZwNnF2ZWRmcndkem5jeHMzY2th eWV1CmlwLWFkZHJlc3MgMTc4LjYyLjIyMi4xMjkKb25pb24tcG9ydCA0NDMKb25p @@ -61,6 +61,55 @@ TkQgUlNBIFBVQkxJQyBLRVktLS0tLQoK -----END MESSAGE-----\ """ +EXPECTED_DDG_INTRODUCTION_POINTS_CONTENT = """\ +introduction-point iwki77xtbvp6qvedfrwdzncxs3ckayeu +ip-address 178.62.222.129 +onion-port 443 +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAK94BEYIHZ4KdEkeyPhbLCpRW5ESg+2WPQhrM4yuKYGuq8wFWgumZYR9 +k/WA//FYXMBz0bB+ckuZs/Yu9nK+HLzpGapV0clst4GUMcBInUCzCcpjJTQsQDgm +3/Y3cqh0W55gOCFhomQ4/1WOYg7YCjk4XYHJE20OdG2Ll5zotK6fAgMBAAE= +-----END RSA PUBLIC KEY----- +service-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAJXmJb8lSydMMqCgCgfgvlB2E5rpd57kz/Aqg7/d1HKc3+l5QoUvHyuy +ZsAlyka8Eu534hl41oqEKpAKYcMn1TM0vpJEGNVOc+05BInxI9h9f0Mg01PD0tYu +GcLHYgBzcrfEmKwMtM8WEmcMJd7n2uffaAvJ846WubbeV7MW1YehAgMBAAE= +-----END RSA PUBLIC KEY----- +introduction-point em4gjk6eiiualhmlyiifrzc7lbtrsbip +ip-address 46.4.174.52 +onion-port 443 +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALBmhdF5wHxHrpLSmjAZottx220+995FdMOOtZNjRw1DBSprUZZqtxWa +P8TKpHKzwGJKCVYIIj7lohbv9T9urmlfTE05URGenZoifOFNz3YwMJTXScQEBJ10 +9iWNLDTskLzDKCAbGhbn/MKwOfYGBhNTljdyTmNY5ECRbRzjev9vAgMBAAE= +-----END RSA PUBLIC KEY----- +service-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAMxMHoAmrbUMsxiICp3iTPYghn0YueKHx219wu8O/Q51QycVVLpX27d1 +hJXkPB33XQBXsBS7SxsSsSCQ3GEurQJ7GuBLpYIR/vqsakE/l8wc2CJC5WUhyFFk ++1TWIUI5txnXLyWCRcKDUrjqdosDaDosgHFg23Mnx+xXcaQ/frB/AgMBAAE= +-----END RSA PUBLIC KEY----- +introduction-point jqhfl364x3upe6lqnxizolewlfrsw2zy +ip-address 62.210.82.169 +onion-port 443 +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAPUkqxgfYdw0Pm/g6MbhmVsGKlujifmkhdfoEeuzgo+wnEsGvwUebzrz +fZJRt0caXFhnCGgQD2IgmarUaUvP24fXo/4mYzLcPeI7gZneuAQJYvm98Yv9vOHl +NaM/WvDkCsJ3GVNJ1H3wLPQRI3v7KbNuc9tCOYl/r09OhVaWkzajAgMBAAE= +-----END RSA PUBLIC KEY----- +service-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALbx8LeqRoP/r9w9hjwD41YUm7Po697xRtytF0McyLCS7GRiUYnji7KY +fepXdvN/Jl5qQKHIBb602kuOTl0pN8Q+YeFUSIIDcmPBLpBDhH3PvrQMcGVaiOWH +8w0HMZCxgwAcCC51w5VwiumxEJNBVcZsOx0mzN1Cloy+90q0lFXLAgMBAAE= +-----END RSA PUBLIC KEY----- + +""" + EXPECTED_DDG_SIGNATURE = """\ -----BEGIN SIGNATURE----- VKMmsDIUUFOrpqvcQroIZjDZTKxqNs88a4M9Te8cR/ZvS7H2nffv6iQs0tom5X4D @@ -69,6 +118,54 @@ cZjQLW0juUYCbgIGdxVEBnlEt2rgBSM9+1oR7EAfV1U= -----END SIGNATURE-----\ """ +EXPECT_POINT_1_ONION_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAK94BEYIHZ4KdEkeyPhbLCpRW5ESg+2WPQhrM4yuKYGuq8wFWgumZYR9 +k/WA//FYXMBz0bB+ckuZs/Yu9nK+HLzpGapV0clst4GUMcBInUCzCcpjJTQsQDgm +3/Y3cqh0W55gOCFhomQ4/1WOYg7YCjk4XYHJE20OdG2Ll5zotK6fAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECT_POINT_1_SERVICE_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAJXmJb8lSydMMqCgCgfgvlB2E5rpd57kz/Aqg7/d1HKc3+l5QoUvHyuy +ZsAlyka8Eu534hl41oqEKpAKYcMn1TM0vpJEGNVOc+05BInxI9h9f0Mg01PD0tYu +GcLHYgBzcrfEmKwMtM8WEmcMJd7n2uffaAvJ846WubbeV7MW1YehAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECT_POINT_2_ONION_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALBmhdF5wHxHrpLSmjAZottx220+995FdMOOtZNjRw1DBSprUZZqtxWa +P8TKpHKzwGJKCVYIIj7lohbv9T9urmlfTE05URGenZoifOFNz3YwMJTXScQEBJ10 +9iWNLDTskLzDKCAbGhbn/MKwOfYGBhNTljdyTmNY5ECRbRzjev9vAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECT_POINT_2_SERVICE_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAMxMHoAmrbUMsxiICp3iTPYghn0YueKHx219wu8O/Q51QycVVLpX27d1 +hJXkPB33XQBXsBS7SxsSsSCQ3GEurQJ7GuBLpYIR/vqsakE/l8wc2CJC5WUhyFFk ++1TWIUI5txnXLyWCRcKDUrjqdosDaDosgHFg23Mnx+xXcaQ/frB/AgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECT_POINT_3_ONION_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAPUkqxgfYdw0Pm/g6MbhmVsGKlujifmkhdfoEeuzgo+wnEsGvwUebzrz +fZJRt0caXFhnCGgQD2IgmarUaUvP24fXo/4mYzLcPeI7gZneuAQJYvm98Yv9vOHl +NaM/WvDkCsJ3GVNJ1H3wLPQRI3v7KbNuc9tCOYl/r09OhVaWkzajAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECT_POINT_3_SERVICE_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALbx8LeqRoP/r9w9hjwD41YUm7Po697xRtytF0McyLCS7GRiUYnji7KY +fepXdvN/Jl5qQKHIBb602kuOTl0pN8Q+YeFUSIIDcmPBLpBDhH3PvrQMcGVaiOWH +8w0HMZCxgwAcCC51w5VwiumxEJNBVcZsOx0mzN1Cloy+90q0lFXLAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + class TestHiddenServiceDescriptor(unittest.TestCase): def test_for_duckduckgo_with_validation(self): @@ -110,5 +207,34 @@ class TestHiddenServiceDescriptor(unittest.TestCase): self.assertEqual('e24kgecavwsznj7gpbktqsiwgvngsf4e', desc.secret_id_part) self.assertEqual(datetime.datetime(2015, 2, 23, 20, 0, 0), desc.published) self.assertEqual(['2', '3'], desc.protocol_versions) - self.assertEqual(EXPECTED_DDG_INTRODUCTION_POINTS_BLOB, desc.introduction_points_blob) + self.assertEqual(EXPECTED_DDG_INTRODUCTION_POINTS_ENCODED, desc.introduction_points_encoded) + self.assertEqual([], desc.introduction_points_auth) + self.assertEqual(EXPECTED_DDG_INTRODUCTION_POINTS_CONTENT, desc.introduction_points_content) self.assertEqual(EXPECTED_DDG_SIGNATURE, desc.signature) + + introduction_points = desc.introduction_points() + self.assertEqual(3, len(introduction_points)) + + point = introduction_points[0] + self.assertEqual('iwki77xtbvp6qvedfrwdzncxs3ckayeu', point.identifier) + self.assertEqual('178.62.222.129', point.address) + self.assertEqual(443, point.port) + self.assertEqual(EXPECT_POINT_1_ONION_KEY, point.onion_key) + self.assertEqual(EXPECT_POINT_1_SERVICE_KEY, point.service_key) + self.assertEqual([], point.intro_authentication) + + point = introduction_points[1] + self.assertEqual('em4gjk6eiiualhmlyiifrzc7lbtrsbip', point.identifier) + self.assertEqual('46.4.174.52', point.address) + self.assertEqual(443, point.port) + self.assertEqual(EXPECT_POINT_2_ONION_KEY, point.onion_key) + self.assertEqual(EXPECT_POINT_2_SERVICE_KEY, point.service_key) + self.assertEqual([], point.intro_authentication) + + point = introduction_points[2] + self.assertEqual('jqhfl364x3upe6lqnxizolewlfrsw2zy', point.identifier) + self.assertEqual('62.210.82.169', point.address) + self.assertEqual(443, point.port) + self.assertEqual(EXPECT_POINT_3_ONION_KEY, point.onion_key) + self.assertEqual(EXPECT_POINT_3_SERVICE_KEY, point.service_key) + self.assertEqual([], point.intro_authentication)

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits March 2015