commit 762f0ad396dd752f0f807871e0715c142cdbf475 Author: Damian Johnson <atagar@torproject.org> Date: Fri Jul 5 19:18:02 2013 -0700 Class for issuing remote descriptor queries Adding a Query class which does most of the heavy lifting for downloading descriptors. --- stem/control.py | 4 +- stem/descriptor/__init__.py | 1 + stem/descriptor/remote.py | 174 ++++++++++++++++++++++++++++- stem/descriptor/server_descriptor.py | 3 + test/settings.cfg | 1 + test/unit/descriptor/remote.py | 126 +++++++++++++++++++++ test/unit/descriptor/server_descriptor.py | 8 +- 7 files changed, 308 insertions(+), 9 deletions(-) diff --git a/stem/control.py b/stem/control.py index 8a8f410..0f589e1 100644 --- a/stem/control.py +++ b/stem/control.py @@ -1255,8 +1255,8 @@ class Controller(BaseController): :raises: * :class:`stem.ControllerError` if unable to query the descriptor - * **ValueError** if **relay** doesn't conform with the patter for being a - fingerprint or nickname + * **ValueError** if **relay** doesn't conform with the pattern for being + a fingerprint or nickname An exception is only raised if we weren't provided a default response. """ diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index dc982f4..a527329 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -137,6 +137,7 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen :returns: iterator for :class:`~stem.descriptor.__init__.Descriptor` instances in the file :raises: + * **ValueError** if the contents is malformed and validate is True * **TypeError** if we can't match the contents of the file to a descriptor type * **IOError** if unable to read from the descriptor_file """ diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index ea8dbb3..1f28bbd 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -4,8 +4,9 @@ """ Utilities for retrieving descriptors from directory authorities and mirrors. This is mostly done through the -:class:`~stem.descriptor.remote.DescriptorDownloader` class, which issues the -requests and provides back parsed content. For example... +:class:`~stem.descriptor.remote.DescriptorDownloader` class, which issues +:class:`~stem.descriptor.remote.Query` to get descriptor content. For +example... :: @@ -14,13 +15,174 @@ requests and provides back parsed content. For example... use_mirrors = True, ) + query = downloader.get_server_descriptors() + + print "Exit Relays:" + try: - for desc in downloader.get_server_descriptors(): + for desc in query.run(): if desc.exit_policy.is_exiting_allowed(): - print "%s (%s)" % (desc.nickname, desc.fingerprint) - except IOError, exc: - print "Unable to query the server descriptors: %s" % exc + print " %s (%s)" % (desc.nickname, desc.fingerprint) + + print + print "Query took %0.2f seconds" % query.runtime + except Exception as exc: + print "Unable to query the server descriptors: %s" % query.error + +If you don't care about errors then you can also simply iterate over the query +itself... + +:: + + for desc in downloader.get_server_descriptors(): + if desc.exit_policy.is_exiting_allowed(): + print " %s (%s)" % (desc.nickname, desc.fingerprint) """ +import sys +import threading +import time +import urllib2 + +import stem.descriptor + +# Tor directory authorities as of commit f631b73 (7/4/13): +# https://gitweb.torproject.org/tor.git/blob/f631b73:/src/or/config.c#l816 + +DIRECTORY_AUTHORITIES = { + 'moria1': ('128.31.0.39', 9131), + 'tor26': ('86.59.21.38', 80), + 'dizum': ('194.109.206.212', 80), + 'Tonga': ('82.94.251.203', 80), + 'turtles': ('76.73.17.194', 9030), + 'gabelmoo': ('212.112.245.170', 80), + 'dannenberg': ('193.23.244.244', 80), + 'urras': ('208.83.223.34', 443), + 'maatuska': ('171.25.193.9', 443), + 'Faravahar': ('154.35.32.5', 80), +} + + +class Query(object): + """ + Asynchronous request for descriptor content from a directory authority or + mirror. The caller can block on the response by either calling + :func:~stem.descriptor.remote.run: or iterating over our descriptor content. + + :var str address: address of the authority or mirror we're querying + :var int port: directory port we're querying + :var str resource: resource being fetched, such as '/tor/status-vote/current/consensus.z' + + :var Exception error: exception if a problem occured + :var bool is_done: flag that indicates if our request has finished + :var str descriptor_type: type of descriptors being fetched, see :func:`~stem.descriptor.__init__.parse_file` + + :var float start_time: unix timestamp when we first started running + :var float timeout: duration before we'll time out our request + :var float runtime: time our query took, this is **None** if it's not yet finished + """ + + def __init__(self, address, port, resource, descriptor_type, timeout = None, start = True): + self.address = address + self.port = port + self.resource = resource + + self.error = None + self.is_done = False + self.descriptor_type = descriptor_type + + self.start_time = None + self.timeout = timeout + self.runtime = None + + self._downloader_thread = None + self._downloader_thread_lock = threading.RLock() + + self._results = None # descriptor iterator + + if start: + self.start() + + def get_url(self): + """ + Provides the url being queried. + + :returns: **str** for the url being queried by this request + """ + + return "http://%s:%i/%s" % (self.address, self.port, self.resource.lstrip('/')) + + def start(self): + """ + Starts downloading the scriptors if we haven't started already. + """ + + with self._downloader_thread_lock: + if self._downloader_thread is None: + self._downloader_thread = threading.Thread(target = self._download_descriptors, name="Descriptor Query") + self._downloader_thread.setDaemon(True) + self._downloader_thread.start() + + def run(self, suppress = False): + """ + Blocks until our request is complete then provides the descriptors. If we + haven't yet started our request then this does so. + + :param bool suppress: avoids raising exceptions if **True** + + :returns: iterator for the requested :class:`~stem.descriptor.__init__.Descriptor` instances + + :raises: + Using the iterator can fail with the following if **suppress** is + **False**... + + * **ValueError** if the descriptor contents is malformed + * **socket.timeout** if our request timed out + * **urllib2.URLError** for most request failures + + Note that the urllib2 module may fail with other exception types, in + which case we'll pass it along. + """ + + with self._downloader_thread_lock: + self.start() + self._downloader_thread.join() + + if self.error: + if not suppress: + raise self.error + else: + if self._results is None: + if not suppress: + raise ValueError('BUG: _download_descriptors() finished without either results or an error') + + return + + try: + for desc in self._results: + yield desc + except ValueError as exc: + self.error = exc + + if not suppress: + raise self.error + + def __iter__(self): + for desc in self.run(True): + yield desc + + def _download_descriptors(self): + try: + self.start_time = time.time() + response = urllib2.urlopen(self.get_url(), timeout = self.timeout) + self.runtime = time.time() - self.start_time + + self._results = stem.descriptor.parse_file(response, self.descriptor_type) + except: + self.error = sys.exc_info()[1] + finally: + self.is_done = True + + class DescriptorDownloader(object): pass diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 4af4976..d23f41e 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -136,6 +136,9 @@ def _parse_file(descriptor_file, is_bridge = False, validate = True): else: yield RelayDescriptor(descriptor_text, validate, annotations) else: + if validate and annotations: + raise ValueError('Content conform to being a server descriptor:\n%s' % '\n'.join(annotations)) + break # done parsing descriptors diff --git a/test/settings.cfg b/test/settings.cfg index d5c0997..68b120d 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -156,6 +156,7 @@ test.unit_tests |test.unit.util.tor_tools.TestTorTools |test.unit.descriptor.export.TestExport |test.unit.descriptor.reader.TestDescriptorReader +|test.unit.descriptor.remote.TestDescriptorDownloader |test.unit.descriptor.server_descriptor.TestServerDescriptor |test.unit.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor |test.unit.descriptor.microdescriptor.TestMicrodescriptor diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py new file mode 100644 index 0000000..9e8b5cc --- /dev/null +++ b/test/unit/descriptor/remote.py @@ -0,0 +1,126 @@ +""" +Unit tests for stem.descriptor.remote. +""" + +import io +import socket +import unittest + +import stem.descriptor.remote + +from mock import patch + +# Output from requesting moria1's descriptor from itself... +# % curl http://128.31.0.39:9131/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE... + +TEST_DESCRIPTOR = b"""\ +router moria1 128.31.0.34 9101 0 9131 +platform Tor 0.2.5.0-alpha-dev on Linux +protocols Link 1 2 Circuit 1 +published 2013-07-05 23:48:52 +fingerprint 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31 +uptime 1818933 +bandwidth 512000 62914560 1307929 +extra-info-digest 17D0142F6EBCDF60160EB1794FA6C9717D581F8C +caches-extra-info +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALzd4bhz1usB7wpoaAvP+BBOnNIk7mByAKV6zvyQ0p1M09oEmxPMc3qD +AAm276oJNf0eq6KWC6YprzPWFsXEIdXSqA6RWXCII1JG/jOoy6nt478BkB8TS9I9 +1MJW27ppRaqnLiTmBmM+qzrsgJGwf+onAgUKKH2GxlVgahqz8x6xAgMBAAE= +-----END RSA PUBLIC KEY----- +signing-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBALtJ9uD7cD7iHjqNA3AgsX9prES5QN+yFQyr2uOkxzhvunnaf6SNhzWW +bkfylnMrRm/qCz/czcjZO6N6EKHcXmypehvP566B7gAQ9vDsb+l7VZVWgXvzNc2s +tl3P7qpC08rgyJh1GqmtQTCesIDqkEyWxwToympCt09ZQRq+fIttAgMBAAE= +-----END RSA PUBLIC KEY----- +hidden-service-dir +contact 1024D/28988BF5 arma mit edu +ntor-onion-key 9ZVjNkf/iLEnD685SpC5kcDytQ7u5ViiI9JOftdbE0k= +reject *:* +router-signature +-----BEGIN SIGNATURE----- +Y8Tj2e7mPbFJbguulkPEBVYzyO57p4btpWEXvRMD6vxIh/eyn25pehg5dUVBtZlL +iO3EUE0AEYah2W9gdz8t+i3Dtr0zgqLS841GC/TyDKCm+MKmN8d098qnwK0NGF9q +01NZPuSqXM1b6hnl2espFzL7XL8XEGRU+aeg+f/ukw4= +-----END SIGNATURE----- +""" + + +class TestDescriptorDownloader(unittest.TestCase): + @patch('urllib2.urlopen') + def test_query_download(self, urlopen_mock): + """ + Check Query functionality when we successfully download a descriptor. + """ + + urlopen_mock.return_value = io.BytesIO(TEST_DESCRIPTOR) + + query = stem.descriptor.remote.Query( + '128.31.0.39', + 9131, + '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31', + 'server-descriptor 1.0', + ) + + expeced_url = 'http://128.31.0.39:9131/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE...' + self.assertEqual(expeced_url, query.get_url()) + + descriptors = list(query) + self.assertEqual(1, len(descriptors)) + desc = descriptors[0] + + self.assertEqual('moria1', desc.nickname) + self.assertEqual('128.31.0.34', desc.address) + self.assertEqual('9695DFC35FFEB861329B9F1AB04C46397020CE31', desc.fingerprint) + self.assertEqual(TEST_DESCRIPTOR, desc.get_bytes()) + + urlopen_mock.assert_called_once_with(expeced_url, timeout = None) + + @patch('urllib2.urlopen') + def test_query_with_malformed_content(self, urlopen_mock): + """ + Query with malformed descriptor content. + """ + + descriptor_content = b'some malformed stuff' + urlopen_mock.return_value = io.BytesIO(descriptor_content) + + query = stem.descriptor.remote.Query( + '128.31.0.39', + 9131, + '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31', + 'server-descriptor 1.0', + ) + + # checking via the iterator + + expected_error_msg = 'Content conform to being a server descriptor:\nsome malformed stuff' + + descriptors = list(query) + self.assertEqual(0, len(descriptors)) + self.assertEqual(ValueError, type(query.error)) + self.assertEqual(expected_error_msg, str(query.error)) + + # check via the run() method + + self.assertRaises(ValueError, list, query.run()) + + @patch('urllib2.urlopen') + def test_query_with_timeout(self, urlopen_mock): + urlopen_mock.side_effect = socket.timeout('connection timed out') + + query = stem.descriptor.remote.Query( + '128.31.0.39', + 9131, + '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31', + 'server-descriptor 1.0', + 5, + ) + + self.assertRaises(socket.timeout, list, query.run()) + urlopen_mock.assert_called_once_with( + 'http://128.31.0.39:9131/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE...', + timeout = 5, + ) diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index c679588..50f27f6 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -213,10 +213,16 @@ class TestServerDescriptor(unittest.TestCase): desc_text = b"@pepperjack very tasty\n@mushrooms not so much\n" desc_text += get_relay_server_descriptor(content = True) - desc_text += b"\ntrailing text that should be ignored, ho hum" + desc_text += b"\ntrailing text that should be invalid, ho hum" # running _parse_file should provide an iterator with a single descriptor desc_iter = stem.descriptor.server_descriptor._parse_file(io.BytesIO(desc_text)) + self.assertRaises(ValueError, list, desc_iter) + + desc_text = b"@pepperjack very tasty\n@mushrooms not so much\n" + desc_text += get_relay_server_descriptor(content = True) + desc_iter = stem.descriptor.server_descriptor._parse_file(io.BytesIO(desc_text)) + desc_entries = list(desc_iter) self.assertEquals(1, len(desc_entries)) desc = desc_entries[0]