commit ca21828a88d56fbe7dfa7d4b78ec4f99e0bc8343 Author: Damian Johnson atagar@torproject.org Date: Mon Feb 23 19:32:53 2015 -0800

Support hidden service introduction-points

This is a very unusual attribute in that it's a base64 encoded blob of two subdocuments: the first optional authentication data, followed by a list of subdocuments.

Parsing the authentication data when reading the field, but leaving the second pass to a method since it may optionally be encrypted. --- stem/descriptor/hidden_service_descriptor.py | 140 ++++++++++++++++++++- stem/util/connection.py | 4 +- test/unit/descriptor/hidden_service_descriptor.py | 130 ++++++++++++++++++- 3 files changed, 266 insertions(+), 8 deletions(-)

diff --git a/stem/descriptor/hidden_service_descriptor.py b/stem/descriptor/hidden_service_descriptor.py index c915f0e..d626144 100644 --- a/stem/descriptor/hidden_service_descriptor.py +++ b/stem/descriptor/hidden_service_descriptor.py @@ -23,6 +23,14 @@ the HSDir flag. # # https://collector.torproject.org/formats.html

+import base64 +import collections +import io + +import stem.util.connection + +from stem import str_type + from stem.descriptor import ( PGP_BLOCK_END, Descriptor, @@ -34,6 +42,12 @@ from stem.descriptor import ( _parse_key_block, )

+try: + # added in python 3.2 + from functools import lru_cache +except ImportError: + from stem.util.lru_cache import lru_cache + REQUIRED_FIELDS = ( 'rendezvous-service-descriptor', 'version', @@ -44,6 +58,23 @@ REQUIRED_FIELDS = ( 'signature', )

+INTRODUCTION_POINTS_ATTR = { + 'identifier': None, + 'address': None, + 'port': None, + 'onion_key': None, + 'service_key': None, + 'intro_authentication': [], +} + +IntroductionPoint = collections.namedtuple('IntroductionPoints', INTRODUCTION_POINTS_ATTR.keys()) + + +class DecryptionFailure(Exception): + """ + Failure to decrypt the hidden service descriptor's introduction-points. + """ +

def _parse_file(descriptor_file, validate = False, **kwargs): """ @@ -86,12 +117,37 @@ def _parse_protocol_versions_line(descriptor, entries): value = _value('protocol-versions', entries) descriptor.protocol_versions = value.split(',')

+ +def _parse_introduction_points_line(descriptor, entries): + _, block_type, block_contents = entries['introduction-points'][0] + + if not block_contents or block_type != 'MESSAGE': + raise ValueError("'introduction-points' should be followed by a MESSAGE block, but was a %s" % block_type) + + descriptor.introduction_points_encoded = block_contents + + blob = ''.join(block_contents.split('\n')[1:-1]) + decoded_field = base64.b64decode(stem.util.str_tools._to_bytes(blob)) + + auth_types = [] + + while decoded_field.startswith('service-authentication ') and '\n' in decoded_field: + auth_line, decoded_field = decoded_field.split('\n', 1) + auth_line_comp = auth_line.split(' ') + + if len(auth_line_comp) < 3: + raise ValueError("Within introduction-points we expected 'service-authentication [auth_type] [auth_data]', but had '%s'" % auth_line) + + auth_types.append((auth_line_comp[1], auth_line_comp[2])) + + descriptor.introduction_points_auth = auth_types + descriptor.introduction_points_content = decoded_field + _parse_rendezvous_service_descriptor_line = _parse_simple_line('rendezvous-service-descriptor', 'descriptor_id') _parse_version_line = _parse_simple_line('version', 'version') _parse_permanent_key_line = _parse_key_block('permanent-key', 'permanent_key', 'RSA PUBLIC KEY') _parse_secret_id_part_line = _parse_simple_line('secret-id-part', 'secret_id_part') _parse_publication_time_line = _parse_timestamp_line('publication-time', 'published') -_parse_introduction_points_line = _parse_key_block('introduction-points', 'introduction_points_blob', 'MESSAGE') _parse_signature_line = _parse_key_block('signature', 'signature', 'SIGNATURE')

@@ -106,8 +162,12 @@ class HiddenServiceDescriptor(Descriptor): values so our descriptor_id can be validated :var datetime published: ***** time in UTC when this descriptor was made :var list protocol_versions: ***** versions that are supported when establishing a connection - :var str introduction_points_blob: ***** raw introduction points blob, if - the hidden service uses cookie authentication this is encrypted + :var str introduction_points_encoded: ***** raw introduction points blob + :var list introduction_points_auth: ***** tuples of the form + (auth_method, auth_data) for our introduction_points_content + :var str introduction_points_content: ***** decoded introduction-points + content without authentication data, if using cookie authentication this is + encrypted :var str signature: signature of the descriptor content

***** attribute is either required when we're parsed with validation or has @@ -121,7 +181,7 @@ class HiddenServiceDescriptor(Descriptor): 'secret_id_part': (None, _parse_secret_id_part_line), 'published': (None, _parse_publication_time_line), 'protocol_versions': ([], _parse_protocol_versions_line), - 'introduction_points_blob': (None, _parse_introduction_points_line), + 'introduction_points_encoded': (None, _parse_introduction_points_line), 'signature': (None, _parse_signature_line), }

@@ -155,3 +215,75 @@ class HiddenServiceDescriptor(Descriptor): self._parse(entries, validate) else: self._entries = entries + + @lru_cache() + def introduction_points(self): + """ + Provided this service's introduction points. This provides a list of + IntroductionPoint instances, which have the following attributes... + + * identifier (str): hash of this introduction point's identity key + * address (str): address of this introduction point + * port (int): port where this introduction point is listening + * onion_key (str): public key for communicating with this introduction point + * service_key (str): public key for communicating with this hidden service + * intro_authentication (list): tuples of the form (auth_type, auth_data) + for establishing a connection + + :returns: **list** of IntroductionPoints instances + + :raises: + * **ValueError** if the our introduction-points is malformed + * **DecryptionFailure** if unable to decrypt this field + """ + + # TODO: Support fields encrypted with a desriptor-cookie. Need sample data + # to implement this. + + if not self.introduction_points_content.startswith('introduction-point '): + raise DecryptionFailure('introduction-point content is encrypted') + + introduction_points = [] + content_io = io.StringIO(str_type(self.introduction_points_content)) + + while True: + content = ''.join(_read_until_keywords('introduction-point', content_io, ignore_first = True)) + + if not content: + break # reached the end + + attr = dict(INTRODUCTION_POINTS_ATTR) + entries = _get_descriptor_components(content, False) + + # TODO: most fields can only appear once, we should check for that + + for keyword, values in list(entries.items()): + value, block_type, block_contents = values[0] + + if keyword == 'introduction-point': + attr['identifier'] = value + elif keyword == 'ip-address': + # TODO: need clarification about if this IPv4, IPv6, or both + attr['address'] = value + elif keyword == 'onion-port': + if not stem.util.connection.is_valid_port(value): + raise ValueError("'%s' is an invalid port" % value) + + attr['port'] = int(value) + elif keyword == 'onion-key': + attr['onion_key'] = block_contents + elif keyword == 'service-key': + attr['service_key'] = block_contents + elif keyword == 'intro-authentication': + auth_entries = [] + + for auth_value, _, _ in values: + if ' ' not in auth_value: + raise ValueError("We expected 'intro-authentication [auth_type] [auth_data]', but had '%s'" % auth_value) + + auth_type, auth_data = auth_value.split(' ')[:2] + auth_entries.append((auth_type, auth_data)) + + introduction_points.append(IntroductionPoint(**attr)) + + return introduction_points diff --git a/stem/util/connection.py b/stem/util/connection.py index fa2a3d5..402a45a 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -140,8 +140,8 @@ RESOLVER_FILTER = {

def get_connections(resolver, process_pid = None, process_name = None): """ - Retrieves a list of the current connections for a given process. The provides - a list of Connection instances, which have five attributes... + Retrieves a list of the current connections for a given process. This + provides a list of Connection instances, which have five attributes...

* local_address (str) * local_port (int) diff --git a/test/unit/descriptor/hidden_service_descriptor.py b/test/unit/descriptor/hidden_service_descriptor.py index 579c549..4853a28 100644 --- a/test/unit/descriptor/hidden_service_descriptor.py +++ b/test/unit/descriptor/hidden_service_descriptor.py @@ -17,7 +17,7 @@ I5gQozM65ENelfxYlysBjJ52xSDBd8C4f/p9umdzaaaCmzXG/nhzAgMBAAE= -----END RSA PUBLIC KEY-----\ """

-EXPECTED_DDG_INTRODUCTION_POINTS_BLOB = """\ +EXPECTED_DDG_INTRODUCTION_POINTS_ENCODED = """\ -----BEGIN MESSAGE----- aW50cm9kdWN0aW9uLXBvaW50IGl3a2k3N3h0YnZwNnF2ZWRmcndkem5jeHMzY2th eWV1CmlwLWFkZHJlc3MgMTc4LjYyLjIyMi4xMjkKb25pb24tcG9ydCA0NDMKb25p @@ -61,6 +61,55 @@ TkQgUlNBIFBVQkxJQyBLRVktLS0tLQoK -----END MESSAGE-----\ """

+EXPECTED_DDG_INTRODUCTION_POINTS_CONTENT = """\ +introduction-point iwki77xtbvp6qvedfrwdzncxs3ckayeu +ip-address 178.62.222.129 +onion-port 443 +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAK94BEYIHZ4KdEkeyPhbLCpRW5ESg+2WPQhrM4yuKYGuq8wFWgumZYR9 +k/WA//FYXMBz0bB+ckuZs/Yu9nK+HLzpGapV0clst4GUMcBInUCzCcpjJTQsQDgm +3/Y3cqh0W55gOCFhomQ4/1WOYg7YCjk4XYHJE20OdG2Ll5zotK6fAgMBAAE= +-----END RSA PUBLIC KEY----- +service-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAJXmJb8lSydMMqCgCgfgvlB2E5rpd57kz/Aqg7/d1HKc3+l5QoUvHyuy +ZsAlyka8Eu534hl41oqEKpAKYcMn1TM0vpJEGNVOc+05BInxI9h9f0Mg01PD0tYu +GcLHYgBzcrfEmKwMtM8WEmcMJd7n2uffaAvJ846WubbeV7MW1YehAgMBAAE= +-----END RSA PUBLIC KEY----- +introduction-point em4gjk6eiiualhmlyiifrzc7lbtrsbip +ip-address 46.4.174.52 +onion-port 443 +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALBmhdF5wHxHrpLSmjAZottx220+995FdMOOtZNjRw1DBSprUZZqtxWa +P8TKpHKzwGJKCVYIIj7lohbv9T9urmlfTE05URGenZoifOFNz3YwMJTXScQEBJ10 +9iWNLDTskLzDKCAbGhbn/MKwOfYGBhNTljdyTmNY5ECRbRzjev9vAgMBAAE= +-----END RSA PUBLIC KEY----- +service-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAMxMHoAmrbUMsxiICp3iTPYghn0YueKHx219wu8O/Q51QycVVLpX27d1 +hJXkPB33XQBXsBS7SxsSsSCQ3GEurQJ7GuBLpYIR/vqsakE/l8wc2CJC5WUhyFFk ++1TWIUI5txnXLyWCRcKDUrjqdosDaDosgHFg23Mnx+xXcaQ/frB/AgMBAAE= +-----END RSA PUBLIC KEY----- +introduction-point jqhfl364x3upe6lqnxizolewlfrsw2zy +ip-address 62.210.82.169 +onion-port 443 +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAPUkqxgfYdw0Pm/g6MbhmVsGKlujifmkhdfoEeuzgo+wnEsGvwUebzrz +fZJRt0caXFhnCGgQD2IgmarUaUvP24fXo/4mYzLcPeI7gZneuAQJYvm98Yv9vOHl +NaM/WvDkCsJ3GVNJ1H3wLPQRI3v7KbNuc9tCOYl/r09OhVaWkzajAgMBAAE= +-----END RSA PUBLIC KEY----- +service-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALbx8LeqRoP/r9w9hjwD41YUm7Po697xRtytF0McyLCS7GRiUYnji7KY +fepXdvN/Jl5qQKHIBb602kuOTl0pN8Q+YeFUSIIDcmPBLpBDhH3PvrQMcGVaiOWH +8w0HMZCxgwAcCC51w5VwiumxEJNBVcZsOx0mzN1Cloy+90q0lFXLAgMBAAE= +-----END RSA PUBLIC KEY----- + +""" + EXPECTED_DDG_SIGNATURE = """\ -----BEGIN SIGNATURE----- VKMmsDIUUFOrpqvcQroIZjDZTKxqNs88a4M9Te8cR/ZvS7H2nffv6iQs0tom5X4D @@ -69,6 +118,54 @@ cZjQLW0juUYCbgIGdxVEBnlEt2rgBSM9+1oR7EAfV1U= -----END SIGNATURE-----\ """

+EXPECT_POINT_1_ONION_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAK94BEYIHZ4KdEkeyPhbLCpRW5ESg+2WPQhrM4yuKYGuq8wFWgumZYR9 +k/WA//FYXMBz0bB+ckuZs/Yu9nK+HLzpGapV0clst4GUMcBInUCzCcpjJTQsQDgm +3/Y3cqh0W55gOCFhomQ4/1WOYg7YCjk4XYHJE20OdG2Ll5zotK6fAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECT_POINT_1_SERVICE_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAJXmJb8lSydMMqCgCgfgvlB2E5rpd57kz/Aqg7/d1HKc3+l5QoUvHyuy +ZsAlyka8Eu534hl41oqEKpAKYcMn1TM0vpJEGNVOc+05BInxI9h9f0Mg01PD0tYu +GcLHYgBzcrfEmKwMtM8WEmcMJd7n2uffaAvJ846WubbeV7MW1YehAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECT_POINT_2_ONION_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALBmhdF5wHxHrpLSmjAZottx220+995FdMOOtZNjRw1DBSprUZZqtxWa +P8TKpHKzwGJKCVYIIj7lohbv9T9urmlfTE05URGenZoifOFNz3YwMJTXScQEBJ10 +9iWNLDTskLzDKCAbGhbn/MKwOfYGBhNTljdyTmNY5ECRbRzjev9vAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECT_POINT_2_SERVICE_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAMxMHoAmrbUMsxiICp3iTPYghn0YueKHx219wu8O/Q51QycVVLpX27d1 +hJXkPB33XQBXsBS7SxsSsSCQ3GEurQJ7GuBLpYIR/vqsakE/l8wc2CJC5WUhyFFk ++1TWIUI5txnXLyWCRcKDUrjqdosDaDosgHFg23Mnx+xXcaQ/frB/AgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECT_POINT_3_ONION_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAPUkqxgfYdw0Pm/g6MbhmVsGKlujifmkhdfoEeuzgo+wnEsGvwUebzrz +fZJRt0caXFhnCGgQD2IgmarUaUvP24fXo/4mYzLcPeI7gZneuAQJYvm98Yv9vOHl +NaM/WvDkCsJ3GVNJ1H3wLPQRI3v7KbNuc9tCOYl/r09OhVaWkzajAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" + +EXPECT_POINT_3_SERVICE_KEY = """\ +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALbx8LeqRoP/r9w9hjwD41YUm7Po697xRtytF0McyLCS7GRiUYnji7KY +fepXdvN/Jl5qQKHIBb602kuOTl0pN8Q+YeFUSIIDcmPBLpBDhH3PvrQMcGVaiOWH +8w0HMZCxgwAcCC51w5VwiumxEJNBVcZsOx0mzN1Cloy+90q0lFXLAgMBAAE= +-----END RSA PUBLIC KEY-----\ +""" +

class TestHiddenServiceDescriptor(unittest.TestCase): def test_for_duckduckgo_with_validation(self): @@ -110,5 +207,34 @@ class TestHiddenServiceDescriptor(unittest.TestCase): self.assertEqual('e24kgecavwsznj7gpbktqsiwgvngsf4e', desc.secret_id_part) self.assertEqual(datetime.datetime(2015, 2, 23, 20, 0, 0), desc.published) self.assertEqual(['2', '3'], desc.protocol_versions) - self.assertEqual(EXPECTED_DDG_INTRODUCTION_POINTS_BLOB, desc.introduction_points_blob) + self.assertEqual(EXPECTED_DDG_INTRODUCTION_POINTS_ENCODED, desc.introduction_points_encoded) + self.assertEqual([], desc.introduction_points_auth) + self.assertEqual(EXPECTED_DDG_INTRODUCTION_POINTS_CONTENT, desc.introduction_points_content) self.assertEqual(EXPECTED_DDG_SIGNATURE, desc.signature) + + introduction_points = desc.introduction_points() + self.assertEqual(3, len(introduction_points)) + + point = introduction_points[0] + self.assertEqual('iwki77xtbvp6qvedfrwdzncxs3ckayeu', point.identifier) + self.assertEqual('178.62.222.129', point.address) + self.assertEqual(443, point.port) + self.assertEqual(EXPECT_POINT_1_ONION_KEY, point.onion_key) + self.assertEqual(EXPECT_POINT_1_SERVICE_KEY, point.service_key) + self.assertEqual([], point.intro_authentication) + + point = introduction_points[1] + self.assertEqual('em4gjk6eiiualhmlyiifrzc7lbtrsbip', point.identifier) + self.assertEqual('46.4.174.52', point.address) + self.assertEqual(443, point.port) + self.assertEqual(EXPECT_POINT_2_ONION_KEY, point.onion_key) + self.assertEqual(EXPECT_POINT_2_SERVICE_KEY, point.service_key) + self.assertEqual([], point.intro_authentication) + + point = introduction_points[2] + self.assertEqual('jqhfl364x3upe6lqnxizolewlfrsw2zy', point.identifier) + self.assertEqual('62.210.82.169', point.address) + self.assertEqual(443, point.port) + self.assertEqual(EXPECT_POINT_3_ONION_KEY, point.onion_key) + self.assertEqual(EXPECT_POINT_3_SERVICE_KEY, point.service_key) + self.assertEqual([], point.intro_authentication)