July 2013 - tor-commits - lists.torproject.org

[stem/master] Workaround for screwy RuntimeError related to mock causing imports
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 762b7239e4866ba1b98212259e9751723e0865df Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jul 6 10:54:58 2013 -0700 Workaround for screwy RuntimeError related to mock causing imports The following isn't used by this directly, but we're still importing it due to a screwy aspect of how mock works. If patched() results in an import that we haven't done before then we can fail with... ====================================================================== ERROR: test_query_download ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/mock.py", line 1201, in patched return func(*args, **keywargs) ... File "/home/atagar/Desktop/stem/stem/descriptor/__init__.py", line 156, in parse_file import stem.descriptor.networkstatus RuntimeError: cannot unmarshal code objects in restricted execution mode I've only seen this arise when running this test directly (without the rest of the unit tests, some of which import networkstatus on their own). --- test/unit/descriptor/remote.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 9e8b5cc..079937b 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -10,6 +10,13 @@ import stem.descriptor.remote from mock import patch +# The following isn't used by this directly, but we're still importing it due +# to a screwy aspect of how mock works. If patched() results in an import that +# we haven't done before then we can fail with a RuntimeError. In practice this +# just arises if we run this unit test on its own. + +import stem.descriptor.networkstatus + # Output from requesting moria1's descriptor from itself... # % curl http://128.31.0.39:9131/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020C…

1 0

[stem/master] Initial pass at the DescriptorDownloader
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 0b1c13553f842088fd5f5ae21600d28e25e6b633 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jul 15 08:43:41 2013 -0700 Initial pass at the DescriptorDownloader Implementation of the DescriptorDownloader's _query() helper method which will be the basis of its other methods. On reflection I should move retry fruntionality and probably caching down to the Query class. --- stem/descriptor/remote.py | 60 +++++++++++++++++++++++++++++++++++++++- test/unit/descriptor/remote.py | 2 +- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 3db72ea..464e7cd 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -40,6 +40,7 @@ itself... """ import io +import random import sys import threading import time @@ -194,4 +195,61 @@ class Query(object): class DescriptorDownloader(object): - pass + """ + Configurable class through which descriptors can be downloaded. This provides + caching, retries, and other capabilities to make downloading descriptors easy + and efficient. + + Queries can be made in either a blocking or non-blocking fashion. If + non-blocking then retries cannot be performed (since we do not know at the + time of the request if it succeeded or failed). + + For more advanced use cases you can use the + :class:`~stem.descriptor.remote.Query` class directly. + + :var bool block: blocks until requests have been concluded if **True**, + otherwise provides the query as soon as its been issued + :var int retries: number of times to attempt the request if it fails + :var float timeout: duration before we'll time out our request, no timeout is + applied if **None** + :var bool start_when_requested: issues requests when our methods are called + if **True**, otherwise provides non-running + :class:`~stem.descriptor.remote.Query` instances + :var bool fall_back_to_authority: when retrying request issues the last + request to a directory authority if **True** + """ + + def __init__(self, block = True, retries = 2, timeout = None, start_when_requested = True, fall_back_to_authority = True): + self.block = block + self.retries = retries + self.timeout = timeout + self.start_when_requested = start_when_requested + self.fall_back_to_authority = fall_back_to_authority + self._directories = DIRECTORY_AUTHORITIES.values() + + def _query(self, resource, descriptor_type, retries): + """ + Issues a request for the given resource. + """ + + if self.fall_back_to_authority and retries == 0: + address, dirport = random.choice(DIRECTORY_AUTHORITIES.values()) + else: + address, dirport = random.choice(self._directories) + + query = Query( + address, + dirport, + resource, + descriptor_type, + timeout = self.timeout, + start = self.start_when_requested, + ) + + if self.block: + query.run(True) + + if query.error and retries > 0: + return self.query(resource, descriptor_type, retries - 1) + + return query diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 079937b..ab2276e 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -81,7 +81,7 @@ class TestDescriptorDownloader(unittest.TestCase): self.assertEqual('moria1', desc.nickname) self.assertEqual('128.31.0.34', desc.address) self.assertEqual('9695DFC35FFEB861329B9F1AB04C46397020CE31', desc.fingerprint) - self.assertEqual(TEST_DESCRIPTOR, desc.get_bytes()) + self.assertEqual(TEST_DESCRIPTOR.strip(), desc.get_bytes()) urlopen_mock.assert_called_once_with(expeced_url, timeout = None)

1 0

[stem/master] Dropping Tonga from remote descriptor fetching
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 61f00819dc3f18b17af0d8d9d3f4f90791364e39 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 14 11:07:40 2013 -0700 Dropping Tonga from remote descriptor fetching Karsten reports that we shouldn't use authorities that lack 'v3ident'. At present that's just Tonga. --- stem/descriptor/remote.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 1f28bbd..ed86eda 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -46,14 +46,15 @@ import urllib2 import stem.descriptor -# Tor directory authorities as of commit f631b73 (7/4/13): +# Tor directory authorities as of commit f631b73 (7/4/13). This should only +# include authorities with 'v3ident': +# # https://gitweb.torproject.org/tor.git/blob/f631b73:/src/or/config.c#l816 DIRECTORY_AUTHORITIES = { 'moria1': ('128.31.0.39', 9131), 'tor26': ('86.59.21.38', 80), 'dizum': ('194.109.206.212', 80), - 'Tonga': ('82.94.251.203', 80), 'turtles': ('76.73.17.194', 9030), 'gabelmoo': ('212.112.245.170', 80), 'dannenberg': ('193.23.244.244', 80),

1 0

[stem/master] Implementing get_server_descriptors() method
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit ac694d3a98ce1e56c674893f8006efe785eb41c9 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 16 09:16:06 2013 -0700 Implementing get_server_descriptors() method Implementing our first downloader method and added an ONLINE integ test. Initially I did a get_microdescriptors() method but turns out that tor didn't implement what we need... https://trac.torproject.org/9271 --- stem/descriptor/remote.py | 47 +++++++++++++++++++++++++++++++-------- test/integ/descriptor/remote.py | 30 +++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 88e3258..a4b21d4 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -80,16 +80,20 @@ class Query(object): :var list endpoints: (address, dirport) tuples of the authority or mirror we're querying, this uses authorities if undefined - :var int retries: number of times to attempt the request if it fails + :var int retries: number of times to attempt the request if downloading it + fails :var bool fall_back_to_authority: when retrying request issues the last request to a directory authority if **True** :var Exception error: exception if a problem occured :var bool is_done: flag that indicates if our request has finished + :var str download_url: last url used to download the descriptor, this is + unset until we've actually made a download attempt :var float start_time: unix timestamp when we first started running :var float timeout: duration before we'll time out our request - :var float runtime: time our query took, this is **None** if it's not yet finished + :var float runtime: time our query took, this is **None** if it's not yet + finished """ def __init__(self, resource, descriptor_type, endpoints = None, retries = 2, fall_back_to_authority = True, timeout = None, start = True): @@ -102,6 +106,7 @@ class Query(object): self.error = None self.is_done = False + self.download_url = None self.start_time = None self.timeout = timeout @@ -183,6 +188,8 @@ class Query(object): for desc in self._results: yield desc except ValueError as exc: + # encountered a parsing error + self.error = exc if not suppress: @@ -195,10 +202,10 @@ class Query(object): def _download_descriptors(self, retries): try: use_authority = retries == 0 and self.fall_back_to_authority - resource_url = self.pick_url(use_authority) + self.download_url = self.pick_url(use_authority) self.start_time = time.time() - response = urllib2.urlopen(resource_url, timeout = self.timeout) + response = urllib2.urlopen(self.download_url, timeout = self.timeout) self.runtime = time.time() - self.start_time # This sucks. We need to read the full response into memory before @@ -209,15 +216,15 @@ class Query(object): response = io.BytesIO(response.read().strip()) self._results = stem.descriptor.parse_file(response, self.descriptor_type) - log.trace("Descriptors retrieved from '%s' in %0.2fs" % (resource_url, self.runtime)) + log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime)) except: exc = sys.exc_info()[1] if retries > 0: - log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (resource_url, retries, exc)) + log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (self.download_url, retries, exc)) return self._download_descriptors(retries - 1) else: - log.debug("Unable to download descriptors from '%s': %s" % (resource_url, exc)) + log.debug("Unable to download descriptors from '%s': %s" % (self.download_url, exc)) self.error = exc finally: self.is_done = True @@ -242,14 +249,36 @@ class DescriptorDownloader(object): request to a directory authority if **True** """ - def __init__(self, retries = 2, timeout = None, start_when_requested = True, fall_back_to_authority = True): + def __init__(self, retries = 2, fall_back_to_authority = True, timeout = None, start_when_requested = True): self.retries = retries self.timeout = timeout self.start_when_requested = start_when_requested self.fall_back_to_authority = fall_back_to_authority self._endpoints = DIRECTORY_AUTHORITIES.values() - def _query(self, resource, descriptor_type, retries): + def get_server_descriptors(self, fingerprints = None): + """ + Provides the server descriptors with the given fingerprints. If no + fingerprints are provided then this returns all descriptors in the present + consensus. + + :param str,list fingerprints: fingerprint or list of fingerprints to be + retrieved, gets all descriptors if **None** + + :returns: :class:`~stem.descriptor.remote.Query` for the server descriptors + """ + + resource = '/tor/server/all' + + if fingerprints: + if isinstance(fingerprints, str): + resource = '/tor/server/fp/%s' % fingerprints + else: + resource = '/tor/server/fp/%s' % '+'.join(fingerprints) + + return self._query(resource, 'server-descriptor 1.0') + + def _query(self, resource, descriptor_type): """ Issues a request for the given resource. """ diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index 8509556..6e1f426 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -44,3 +44,33 @@ class TestDescriptorReader(unittest.TestCase): self.assertEqual(1, len(descriptors)) self.assertEqual('moria1', descriptors[0].nickname) + def test_get_server_descriptors(self): + """ + Exercises the downloader's get_server_descriptors() method. + """ + + downloader = stem.descriptor.remote.DescriptorDownloader() + + # Fetch a single descriptor and a batch. I'd love to also exercise + # retrieving all descriptors, but that adds roughly a minute to the runtime + # of this test and adds an appreciable load to directory authorities. + + single_query = downloader.get_server_descriptors('9695DFC35FFEB861329B9F1AB04C46397020CE31') + + multiple_query = downloader.get_server_descriptors([ + '9695DFC35FFEB861329B9F1AB04C46397020CE31', + '847B1F850344D7876491A54892F904934E4EB85D', + ]) + + # Explicitly running the queries so they'll provide a useful error if + # unsuccessful. + + single_query.run() + multiple_query.run() + + single_query_results = list(single_query) + self.assertEqual(1, len(single_query_results)) + self.assertEqual('moria1', single_query_results[0].nickname) + + self.assertEqual(2, len(list(multiple_query))) +

1 0

[stem/master] Performing integ test queries in parallel
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit d87f35c7c0f99d52a530cdf12d03ef6abd9cadb0 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 14 14:06:12 2013 -0700 Performing integ test queries in parallel What good is designing an asynchronous descriptor fetching API if you poll things serially? Trivial tweak to reduce this test's runtime from 2.4s to 0.4s. --- test/integ/descriptor/remote.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index 7ac6757..50d6d28 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -25,15 +25,18 @@ class TestDescriptorReader(unittest.TestCase): if test.runner.require_online(self): return + queries = [] + for authority, (address, dirport) in stem.descriptor.remote.DIRECTORY_AUTHORITIES.items(): - query = stem.descriptor.remote.Query( + queries.append(stem.descriptor.remote.Query( address, dirport, '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31', 'server-descriptor 1.0', 30, - ) + )) + for query in queries: try: descriptors = list(query.run()) except Exception, exc:

1 0

[stem/master] Moving retry functionality to Query class
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 7d04653d908f0c62e197bafa27a5cd94634cbb53 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jul 15 09:16:54 2013 -0700 Moving retry functionality to Query class It's far better if the Query class handles retries rather than the DescriptorDownloader. Advantages include... * The DescriptorDownloader no longer has a reason to support blocking queries. There's no advantage to them since the Query knows if/when the request fails (and hence can retry it on our behalf). * The Query class is easier to test. The more functionality we can push down into it the simpler the DescriptorDownloader will be. * More advanced use cases will be using Query instances rather than DescriptorDownloader. By having retry functionality there it'll be easily available to them. --- stem/descriptor/remote.py | 88 ++++++++++++++++++++++----------------- test/integ/descriptor/remote.py | 5 +-- test/unit/descriptor/remote.py | 17 ++++---- 3 files changed, 59 insertions(+), 51 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 464e7cd..88e3258 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -48,6 +48,8 @@ import urllib2 import stem.descriptor +from stem.util import log + # Tor directory authorities as of commit f631b73 (7/4/13). This should only # include authorities with 'v3ident': # @@ -72,27 +74,34 @@ class Query(object): mirror. The caller can block on the response by either calling :func:~stem.descriptor.remote.run: or iterating over our descriptor content. - :var str address: address of the authority or mirror we're querying - :var int port: directory port we're querying :var str resource: resource being fetched, such as '/tor/status-vote/current/consensus.z' + :var str descriptor_type: type of descriptors being fetched, see + :func:`~stem.descriptor.__init__.parse_file` + + :var list endpoints: (address, dirport) tuples of the authority or mirror + we're querying, this uses authorities if undefined + :var int retries: number of times to attempt the request if it fails + :var bool fall_back_to_authority: when retrying request issues the last + request to a directory authority if **True** :var Exception error: exception if a problem occured :var bool is_done: flag that indicates if our request has finished - :var str descriptor_type: type of descriptors being fetched, see :func:`~stem.descriptor.__init__.parse_file` :var float start_time: unix timestamp when we first started running :var float timeout: duration before we'll time out our request :var float runtime: time our query took, this is **None** if it's not yet finished """ - def __init__(self, address, port, resource, descriptor_type, timeout = None, start = True): - self.address = address - self.port = port + def __init__(self, resource, descriptor_type, endpoints = None, retries = 2, fall_back_to_authority = True, timeout = None, start = True): self.resource = resource + self.descriptor_type = descriptor_type + + self.endpoints = endpoints if endpoints else [] + self.retries = retries + self.fall_back_to_authority = fall_back_to_authority self.error = None self.is_done = False - self.descriptor_type = descriptor_type self.start_time = None self.timeout = timeout @@ -106,14 +115,23 @@ class Query(object): if start: self.start() - def get_url(self): + def pick_url(self, use_authority = False): """ - Provides the url being queried. + Provides a url that can be queried. If we have multiple endpoints then one + will be picked randomly. + + :param bool use_authority: ignores our endpoints and uses a directory + authority instead :returns: **str** for the url being queried by this request """ - return "http://%s:%i/%s" % (self.address, self.port, self.resource.lstrip('/')) + if use_authority or not self.endpoints: + address, dirport = random.choice(DIRECTORY_AUTHORITIES.values()) + else: + address, dirport = random.choice(self.endpoints) + + return "http://%s:%i/%s" % (address, dirport, self.resource.lstrip('/')) def start(self): """ @@ -122,7 +140,7 @@ class Query(object): with self._downloader_thread_lock: if self._downloader_thread is None: - self._downloader_thread = threading.Thread(target = self._download_descriptors, name="Descriptor Query") + self._downloader_thread = threading.Thread(target = self._download_descriptors, name="Descriptor Query", args = (self.retries,)) self._downloader_thread.setDaemon(True) self._downloader_thread.start() @@ -174,10 +192,13 @@ class Query(object): for desc in self.run(True): yield desc - def _download_descriptors(self): + def _download_descriptors(self, retries): try: + use_authority = retries == 0 and self.fall_back_to_authority + resource_url = self.pick_url(use_authority) + self.start_time = time.time() - response = urllib2.urlopen(self.get_url(), timeout = self.timeout) + response = urllib2.urlopen(resource_url, timeout = self.timeout) self.runtime = time.time() - self.start_time # This sucks. We need to read the full response into memory before @@ -188,8 +209,16 @@ class Query(object): response = io.BytesIO(response.read().strip()) self._results = stem.descriptor.parse_file(response, self.descriptor_type) + log.trace("Descriptors retrieved from '%s' in %0.2fs" % (resource_url, self.runtime)) except: - self.error = sys.exc_info()[1] + exc = sys.exc_info()[1] + + if retries > 0: + log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (resource_url, retries, exc)) + return self._download_descriptors(retries - 1) + else: + log.debug("Unable to download descriptors from '%s': %s" % (resource_url, exc)) + self.error = exc finally: self.is_done = True @@ -200,15 +229,9 @@ class DescriptorDownloader(object): caching, retries, and other capabilities to make downloading descriptors easy and efficient. - Queries can be made in either a blocking or non-blocking fashion. If - non-blocking then retries cannot be performed (since we do not know at the - time of the request if it succeeded or failed). - For more advanced use cases you can use the :class:`~stem.descriptor.remote.Query` class directly. - :var bool block: blocks until requests have been concluded if **True**, - otherwise provides the query as soon as its been issued :var int retries: number of times to attempt the request if it fails :var float timeout: duration before we'll time out our request, no timeout is applied if **None** @@ -219,37 +242,24 @@ class DescriptorDownloader(object): request to a directory authority if **True** """ - def __init__(self, block = True, retries = 2, timeout = None, start_when_requested = True, fall_back_to_authority = True): - self.block = block + def __init__(self, retries = 2, timeout = None, start_when_requested = True, fall_back_to_authority = True): self.retries = retries self.timeout = timeout self.start_when_requested = start_when_requested self.fall_back_to_authority = fall_back_to_authority - self._directories = DIRECTORY_AUTHORITIES.values() + self._endpoints = DIRECTORY_AUTHORITIES.values() def _query(self, resource, descriptor_type, retries): """ Issues a request for the given resource. """ - if self.fall_back_to_authority and retries == 0: - address, dirport = random.choice(DIRECTORY_AUTHORITIES.values()) - else: - address, dirport = random.choice(self._directories) - - query = Query( - address, - dirport, + return Query( resource, descriptor_type, + endpoints = self._endpoints, + retries = self.retries, + fall_back_to_authority = self.fall_back_to_authority, timeout = self.timeout, start = self.start_when_requested, ) - - if self.block: - query.run(True) - - if query.error and retries > 0: - return self.query(resource, descriptor_type, retries - 1) - - return query diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index 50d6d28..8509556 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -29,11 +29,10 @@ class TestDescriptorReader(unittest.TestCase): for authority, (address, dirport) in stem.descriptor.remote.DIRECTORY_AUTHORITIES.items(): queries.append(stem.descriptor.remote.Query( - address, - dirport, '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31', 'server-descriptor 1.0', - 30, + endpoints = [(address, dirport)], + timeout = 30, )) for query in queries: diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index ab2276e..3aadaac 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -65,14 +65,13 @@ class TestDescriptorDownloader(unittest.TestCase): urlopen_mock.return_value = io.BytesIO(TEST_DESCRIPTOR) query = stem.descriptor.remote.Query( - '128.31.0.39', - 9131, '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31', 'server-descriptor 1.0', + endpoints = [('128.31.0.39', 9131)], ) expeced_url = 'http://128.31.0.39:9131/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020C…' - self.assertEqual(expeced_url, query.get_url()) + self.assertEqual(expeced_url, query.pick_url()) descriptors = list(query) self.assertEqual(1, len(descriptors)) @@ -95,10 +94,9 @@ class TestDescriptorDownloader(unittest.TestCase): urlopen_mock.return_value = io.BytesIO(descriptor_content) query = stem.descriptor.remote.Query( - '128.31.0.39', - 9131, '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31', 'server-descriptor 1.0', + endpoints = [('128.31.0.39', 9131)], ) # checking via the iterator @@ -119,15 +117,16 @@ class TestDescriptorDownloader(unittest.TestCase): urlopen_mock.side_effect = socket.timeout('connection timed out') query = stem.descriptor.remote.Query( - '128.31.0.39', - 9131, '/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31', 'server-descriptor 1.0', - 5, + endpoints = [('128.31.0.39', 9131)], + fall_back_to_authority = False, + timeout = 5, ) self.assertRaises(socket.timeout, list, query.run()) - urlopen_mock.assert_called_once_with( + urlopen_mock.assert_called_with( 'http://128.31.0.39:9131/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020C…', timeout = 5, ) + self.assertEqual(3, urlopen_mock.call_count)

1 0

[stem/master] Rejecting requests for more than 96 descriptors
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 871a957f23097807f67e502fd6e2c9a9b9a6c456 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 16 09:28:18 2013 -0700 Rejecting requests for more than 96 descriptors In our tor-dev@ discussion Karsten mentioned that we could request at most 96 descriptors at a time when polling by their fingerprints... https://lists.torproject.org/pipermail/tor-dev/2013-June/005005.html I've emailed him to clarify where this limitation comes from (the url length or something within tor?), but in the meantime adding a check for this to our get_sever_descriptors() method. --- stem/descriptor/remote.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index a4b21d4..3021f63 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -50,6 +50,11 @@ import stem.descriptor from stem.util import log +# Tor has a limit on the number of descriptors we can fetch explicitly by their +# fingerprint. + +MAX_BATCH_SIZE = 96 + # Tor directory authorities as of commit f631b73 (7/4/13). This should only # include authorities with 'v3ident': # @@ -266,6 +271,8 @@ class DescriptorDownloader(object): retrieved, gets all descriptors if **None** :returns: :class:`~stem.descriptor.remote.Query` for the server descriptors + + :raises: **ValueError** if we request descriptors by fingerprints and there are more than 96 of them (this is a limitation of tor). """ resource = '/tor/server/all' @@ -274,6 +281,9 @@ class DescriptorDownloader(object): if isinstance(fingerprints, str): resource = '/tor/server/fp/%s' % fingerprints else: + if len(fingerprints) > MAX_BATCH_SIZE: + raise ValueError('We can request at most %i descritors at a time when retrieving by fingerprints' % MAX_BATCH_SIZE) + resource = '/tor/server/fp/%s' % '+'.join(fingerprints) return self._query(resource, 'server-descriptor 1.0')

1 0

[stem/master] Implementing get_extrainfo_descriptors() method
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit a014b9b967e8b825a9a4c99514ce2098ab6ba614 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jul 17 09:58:35 2013 -0700 Implementing get_extrainfo_descriptors() method Very similar to the get_server_descriptors() counterpart. --- stem/descriptor/remote.py | 33 +++++++++++++++++++++++++++++++-- test/integ/descriptor/remote.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 3021f63..a18d991 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -272,7 +272,8 @@ class DescriptorDownloader(object): :returns: :class:`~stem.descriptor.remote.Query` for the server descriptors - :raises: **ValueError** if we request descriptors by fingerprints and there are more than 96 of them (this is a limitation of tor). + :raises: **ValueError** if we request more than 96 descriptors by their + fingerprints (this is due to a limit on the url length by squid proxies). """ resource = '/tor/server/all' @@ -282,12 +283,40 @@ class DescriptorDownloader(object): resource = '/tor/server/fp/%s' % fingerprints else: if len(fingerprints) > MAX_BATCH_SIZE: - raise ValueError('We can request at most %i descritors at a time when retrieving by fingerprints' % MAX_BATCH_SIZE) + raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) resource = '/tor/server/fp/%s' % '+'.join(fingerprints) return self._query(resource, 'server-descriptor 1.0') + def get_extrainfo_descriptors(self, fingerprints = None): + """ + Provides the extrainfo descriptors with the given fingerprints. If no + fingerprints are provided then this returns all descriptors in the present + consensus. + + :param str,list fingerprints: fingerprint or list of fingerprints to be + retrieved, gets all descriptors if **None** + + :returns: :class:`~stem.descriptor.remote.Query` for the extrainfo descriptors + + :raises: **ValueError** if we request more than 96 descriptors by their + fingerprints (this is due to a limit on the url length by squid proxies). + """ + + resource = '/tor/extra/all' + + if fingerprints: + if isinstance(fingerprints, str): + resource = '/tor/extra/fp/%s' % fingerprints + else: + if len(fingerprints) > MAX_BATCH_SIZE: + raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) + + resource = '/tor/extra/fp/%s' % '+'.join(fingerprints) + + return self._query(resource, 'extra-info 1.0') + def _query(self, resource, descriptor_type): """ Issues a request for the given resource. diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index 6e1f426..62348ab 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -4,6 +4,8 @@ Integration tests for stem.descriptor.remote. import unittest +import stem.descriptor.server_descriptor +import stem.descriptor.extrainfo_descriptor import stem.descriptor.remote import test.runner @@ -71,6 +73,32 @@ class TestDescriptorReader(unittest.TestCase): single_query_results = list(single_query) self.assertEqual(1, len(single_query_results)) self.assertEqual('moria1', single_query_results[0].nickname) + self.assertTrue(isinstance(single_query_results[0], stem.descriptor.stem.descriptor.server_descriptor.ServerDescriptor)) self.assertEqual(2, len(list(multiple_query))) + def test_get_extrainfo_descriptors(self): + """ + Exercises the downloader's get_extrainfo_descriptors() method. + """ + + downloader = stem.descriptor.remote.DescriptorDownloader() + + single_query = downloader.get_extrainfo_descriptors('9695DFC35FFEB861329B9F1AB04C46397020CE31') + + multiple_query = downloader.get_extrainfo_descriptors([ + '9695DFC35FFEB861329B9F1AB04C46397020CE31', + '847B1F850344D7876491A54892F904934E4EB85D', + ]) + + single_query.run() + multiple_query.run() + + single_query_results = list(single_query) + self.assertEqual(1, len(single_query_results)) + self.assertEqual('moria1', single_query_results[0].nickname) + self.assertTrue(isinstance(single_query_results[0], stem.descriptor.extrainfo_descriptor.ExtraInfoDescriptor)) + + self.assertEqual(2, len(list(multiple_query))) + +

1 0

[stem/master] Adding a broken get_microdescriptors() method
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 749c8d37fd54ea6e41dfca04495c35b47a755f6a Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jul 20 09:52:07 2013 -0700 Adding a broken get_microdescriptors() method Huh. Not really sure why this isn't working. Adding a get_microdescriptors() for how the spec seems to indicate that the method works but tor gives an empty reply. % curl http://217.13.197.5:9030/tor/micro/d/6dCl6ab8CLo0LeMjxi/MZgVJiZgWN8WKTesWPB… Checking with karsten on irc t see if he knows what's up, and moving on to another part. --- stem/descriptor/remote.py | 40 +++++++++++++++++++++++++++++++++------ test/integ/descriptor/remote.py | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 9df09f4..e0ca612 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -52,9 +52,10 @@ from stem import Flag from stem.util import log # Tor has a limit on the number of descriptors we can fetch explicitly by their -# fingerprint. +# fingerprint or hashes due to the url lenght of squid proxies. -MAX_BATCH_SIZE = 96 +MAX_DESCRIPTOR_BATCH_SIZE = 96 +MAX_MICRODESCRIPTOR_BATCH_SIZE = 92 # Tor directory authorities as of commit f631b73 (7/4/13). This should only # include authorities with 'v3ident': @@ -318,8 +319,8 @@ class DescriptorDownloader(object): fingerprints = [fingerprints] if fingerprints: - if len(fingerprints) > MAX_BATCH_SIZE: - raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) + if len(fingerprints) > MAX_DESCRIPTOR_BATCH_SIZE: + raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_DESCRIPTOR_BATCH_SIZE) resource = '/tor/server/fp/%s' % '+'.join(fingerprints) @@ -346,13 +347,40 @@ class DescriptorDownloader(object): fingerprints = [fingerprints] if fingerprints: - if len(fingerprints) > MAX_BATCH_SIZE: - raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) + if len(fingerprints) > MAX_DESCRIPTOR_BATCH_SIZE: + raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_DESCRIPTOR_BATCH_SIZE) resource = '/tor/extra/fp/%s' % '+'.join(fingerprints) return self._query(resource, 'extra-info 1.0') + def get_microdescriptors(self, hashes): + """ + Provides the microdescriptors with the given hashes. To get these see the + 'microdescriptor_hashes' attribute of + :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV3`. Note + that these are only provided via a microdescriptor consensus (such as + 'cached-microdesc-consensus' in your data directory). + + :param str,list hashes: microdescriptor hash or list of hashes to be + retrieved + + :returns: :class:`~stem.descriptor.remote.Query` for the microdescriptors + + :raises: **ValueError** if we request more than 92 microdescriptors by their + hashes (this is due to a limit on the url length by squid proxies). + """ + + if isinstance(hashes, str): + hashes = [hashes] + + if len(hashes) > MAX_MICRODESCRIPTOR_BATCH_SIZE: + raise ValueError("Unable to request more than %i microdescriptors at a time by their hashes" % MAX_MICRODESCRIPTOR_BATCH_SIZE) + + resource = '/tor/micro/d/%s' % '-'.join(hashes) + + return self._query(resource, 'microdescriptor 1.0') + def get_consensus(self, document_handler = stem.descriptor.DocumentHandler.ENTRIES, authority_v3ident = None): """ Provides the present router status entries. diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index a705486..e9d4e8a 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -5,6 +5,7 @@ Integration tests for stem.descriptor.remote. import unittest import stem.descriptor.extrainfo_descriptor +import stem.descriptor.microdescriptor import stem.descriptor.remote import stem.descriptor.router_status_entry import stem.descriptor.server_descriptor @@ -128,6 +129,40 @@ class TestDescriptorReader(unittest.TestCase): self.assertEqual(2, len(list(multiple_query))) + def test_get_microdescriptors(self): + """ + Exercises the downloader's get_microdescriptors() method. + """ + + # TODO: method needs to be fixed - not quite sure what's going wrong... + + test.runner.skip(self, '(test presently broken)') + return + + if test.runner.require_online(self): + return + elif test.runner.only_run_once(self, "test_get_microdescriptors"): + return + + downloader = stem.descriptor.remote.DescriptorDownloader() + + single_query = downloader.get_microdescriptors('6dCl6ab8CLo0LeMjxi/MZgVJiZgWN8WKTesWPBMtyTo') + + multiple_query = downloader.get_microdescriptors([ + '6dCl6ab8CLo0LeMjxi/MZgVJiZgWN8WKTesWPBMtyTo', # moria1 + 'oXBV80OwMACBJpqNeZrYSXF18l9EJCi4/mB8UOl9sME', # tor26 + ]) + + single_query.run() + multiple_query.run() + + single_query_results = list(single_query) + self.assertEqual(1, len(single_query_results)) + self.assertEqual('moria1', single_query_results[0].digest) + self.assertTrue(isinstance(single_query_results[0], stem.descriptor.microdescriptor.Microdescriptor)) + + self.assertEqual(2, len(list(multiple_query))) + def test_get_consensus(self): """ Exercises the downloader's get_consensus() method.

1 0

[stem/master] Implementing a use_directory_mirrors() method
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 4005a88becb68858ec4742455ea4d80e1ff9f827 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 19 10:12:39 2013 -0700 Implementing a use_directory_mirrors() method Method so we can balance load against directory mirrors rather than hammering the authorities. We can either request this during construction (which fails silently) or call this method explicitly (to get the exception). --- stem/descriptor/remote.py | 37 ++++++++++++++++++++++++++++++++++++- test/integ/descriptor/remote.py | 14 ++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 66e6d85..9df09f4 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -48,6 +48,7 @@ import urllib2 import stem.descriptor +from stem import Flag from stem.util import log # Tor has a limit on the number of descriptors we can fetch explicitly by their @@ -250,6 +251,9 @@ class DescriptorDownloader(object): :var int retries: number of times to attempt the request if it fails :var float timeout: duration before we'll time out our request, no timeout is applied if **None** + :var bool use_mirrors: downloads the present consensus and uses the directory + mirrors to fetch future requests, this fails silently if the consensus + cannot be downloaded :var bool start_when_requested: issues requests when our methods are called if **True**, otherwise provides non-running :class:`~stem.descriptor.remote.Query` instances @@ -257,13 +261,42 @@ class DescriptorDownloader(object): request to a directory authority if **True** """ - def __init__(self, retries = 2, fall_back_to_authority = True, timeout = None, start_when_requested = True): + def __init__(self, retries = 2, use_mirrors = False, fall_back_to_authority = True, timeout = None, start_when_requested = True): self.retries = retries self.timeout = timeout self.start_when_requested = start_when_requested self.fall_back_to_authority = fall_back_to_authority self._endpoints = DIRECTORY_AUTHORITIES.values() + if use_mirrors: + try: + start_time = time.time() + self.use_directory_mirrors() + log.debug("Retrieve directory mirrors (took %0.2fs)" % (time.time() - start_time)) + except Exception, exc: + log.debug("Unable to retrieve directory mirrors: %s" % exc) + + def use_directory_mirrors(self): + """ + Downloads the present consensus and configures ourselves to use directory + mirrors, in addition to authorities. + + :raises: **Exception** if unable to determine the directory mirrors + """ + + new_endpoints = set(DIRECTORY_AUTHORITIES.values()) + + query = self.get_consensus() + query.run() # running explicitly so we'll raise errors + + for desc in query: + if Flag.V2DIR in desc.flags: + new_endpoints.add((desc.address, desc.dir_port)) + + # we need our endpoints to be a list rather than set for random.choice() + + self._endpoints = list(new_endpoints) + def get_server_descriptors(self, fingerprints = None): """ Provides the server descriptors with the given fingerprints. If no @@ -351,6 +384,8 @@ class DescriptorDownloader(object): Issues a request for the given resource. """ + log.trace("Retrieving descriptors (resource: %s, type: %s)" % (resource, descriptor_type)) + return Query( resource, descriptor_type, diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index 9273b61..a705486 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -49,6 +49,20 @@ class TestDescriptorReader(unittest.TestCase): self.assertEqual(1, len(descriptors)) self.assertEqual('moria1', descriptors[0].nickname) + def test_use_directory_mirrors(self): + """ + Checks that we can fetch and use a list of directory mirrors. + """ + + if test.runner.require_online(self): + return + elif test.runner.only_run_once(self, "test_use_directory_mirrors"): + return + + downloader = stem.descriptor.remote.DescriptorDownloader() + downloader.use_directory_mirrors() + self.assertTrue(len(downloader._endpoints) > 50) + def test_get_server_descriptors(self): """ Exercises the downloader's get_server_descriptors() method.

1 0