tor-commits July 2013

tor-commits@lists.torproject.org

19 participants
1044 discussions

[stem/master] Support for compressed descriptors
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 90ec648ed075245f6ed2bae42dd4c6968e15fab1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 21 15:57:47 2013 -0700 Support for compressed descriptors Ummm... well, that was easy. When I was initially looking into tor's directory responses I had trouble getting valid compressed results. Kostas suspected that the responses might have been truncated. Oh well, maybe a wget thing. In either case adding compression support to our module was trivial. --- stem/descriptor/remote.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 8ec6f1d..777f769 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -74,6 +74,7 @@ import sys import threading import time import urllib2 +import zlib import stem.descriptor @@ -335,15 +336,17 @@ class Query(object): self.download_url = self._pick_url(use_authority) self.start_time = time.time() - response = urllib2.urlopen(self.download_url, timeout = self.timeout) + response = urllib2.urlopen(self.download_url, timeout = self.timeout).read() - # This sucks. We need to read the full response into memory before - # processing the content. This is because urllib2 returns a 'file like' - # object that lacks tell() or seek(). Hence we need to read it into our - # own buffer that does support these. + if self.download_url.endswith('.z'): + response = zlib.decompress(response) - response = io.BytesIO(response.read().strip()) - self._results = stem.descriptor.parse_file(response, self.descriptor_type, validate = self.validate, document_handler = self.document_handler) + self._results = stem.descriptor.parse_file( + io.BytesIO(response.strip()), + self.descriptor_type, + validate = self.validate, + document_handler = self.document_handler, + ) self.runtime = time.time() - self.start_time log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime)) @@ -434,7 +437,7 @@ class DescriptorDownloader(object): fingerprints (this is due to a limit on the url length by squid proxies). """ - resource = '/tor/server/all' + resource = '/tor/server/all.z' if isinstance(fingerprints, str): fingerprints = [fingerprints] @@ -443,7 +446,7 @@ class DescriptorDownloader(object): if len(fingerprints) > MAX_DESCRIPTOR_BATCH_SIZE: raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_DESCRIPTOR_BATCH_SIZE) - resource = '/tor/server/fp/%s' % '+'.join(fingerprints) + resource = '/tor/server/fp/%s.z' % '+'.join(fingerprints) return self._query(resource) @@ -462,7 +465,7 @@ class DescriptorDownloader(object): fingerprints (this is due to a limit on the url length by squid proxies). """ - resource = '/tor/extra/all' + resource = '/tor/extra/all.z' if isinstance(fingerprints, str): fingerprints = [fingerprints] @@ -471,7 +474,7 @@ class DescriptorDownloader(object): if len(fingerprints) > MAX_DESCRIPTOR_BATCH_SIZE: raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_DESCRIPTOR_BATCH_SIZE) - resource = '/tor/extra/fp/%s' % '+'.join(fingerprints) + resource = '/tor/extra/fp/%s.z' % '+'.join(fingerprints) return self._query(resource) @@ -498,7 +501,7 @@ class DescriptorDownloader(object): if len(hashes) > MAX_MICRODESCRIPTOR_BATCH_SIZE: raise ValueError("Unable to request more than %i microdescriptors at a time by their hashes" % MAX_MICRODESCRIPTOR_BATCH_SIZE) - return self._query('/tor/micro/d/%s' % '-'.join(hashes)) + return self._query('/tor/micro/d/%s.z' % '-'.join(hashes)) def get_consensus(self, document_handler = stem.descriptor.DocumentHandler.ENTRIES, authority_v3ident = None): """ @@ -520,7 +523,7 @@ class DescriptorDownloader(object): if authority_v3ident: resource += '/%s' % authority_v3ident - return self._query(resource, document_handler = document_handler) + return self._query(resource + '.z', document_handler = document_handler) def _query(self, resource, descriptor_type = None, document_handler = stem.descriptor.DocumentHandler.ENTRIES): """

1 0

[stem/master] Expanding remote descriptor pydocs
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 60ad230d88b383a259c531649fdba78d0acd6493 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 21 09:03:39 2013 -0700 Expanding remote descriptor pydocs Adding documentation and examples for the remote descriptor API. This includes some minor API tweaks and a missing import statement. --- stem/descriptor/__init__.py | 1 + stem/descriptor/remote.py | 243 ++++++++++++++++++++++++++++----------- test/integ/descriptor/remote.py | 5 +- test/settings.cfg | 2 +- test/unit/descriptor/remote.py | 2 +- 5 files changed, 181 insertions(+), 72 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 82f846d..f1fdee4 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -154,6 +154,7 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen import stem.descriptor.server_descriptor import stem.descriptor.extrainfo_descriptor import stem.descriptor.networkstatus + import stem.descriptor.microdescriptor # The tor descriptor specifications do not provide a reliable method for # identifying a descriptor file's type and version so we need to guess diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 83416c8..8ec6f1d 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -2,14 +2,16 @@ # See LICENSE for licensing information """ -Utilities for retrieving descriptors from directory authorities and mirrors. -This is mostly done through the +Module for remotely retrieving descriptors from directory authorities and +mirrors. This is most easily done through the :class:`~stem.descriptor.remote.DescriptorDownloader` class, which issues -:class:`~stem.descriptor.remote.Query` to get descriptor content. For -example... +:class:`~stem.descriptor.remote.Query` instances to get you the descriptor +content. For example... :: + from stem.descriptor.remote import DescriptorDownloader + downloader = DescriptorDownloader( cache = '/tmp/descriptor_cache', use_mirrors = True, @@ -27,7 +29,7 @@ example... print print "Query took %0.2f seconds" % query.runtime except Exception as exc: - print "Unable to query the server descriptors: %s" % query.error + print "Unable to retrieve the server descriptors: %s" % exc If you don't care about errors then you can also simply iterate over the query itself... @@ -37,6 +39,33 @@ itself... for desc in downloader.get_server_descriptors(): if desc.exit_policy.is_exiting_allowed(): print " %s (%s)" % (desc.nickname, desc.fingerprint) + +:: + + Query - Asynchronous request to download tor descriptors + |- start - issues the query if it isn't already running + +- run - blocks until the request is finished and provides the results + + DescriptorDownloader - Configurable class for issuing queries + |- use_directory_mirrors - use directory mirrors to download future descriptors + |- get_server_descriptors - provides present :class:`~stem.descriptor.stem.descriptor.server_descriptor.ServerDescriptor` + |- get_extrainfo_descriptors - provides present :class:`~stem.descriptor.extrainfo_descriptor.ExtraInfoDescriptor` + |- get_microdescriptors - provides present :class:`~stem.descriptor.microdescriptor.Microdescriptor` + +- get_consensus - provides present :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV3` + +.. data:: MAX_DESCRIPTOR_BATCH_SIZE + + Maximum number of server or extrainfo descriptors that can requested at a + time by their fingerprints. + +.. data:: MAX_MICRODESCRIPTOR_BATCH_SIZE + + Maximum number of microdescriptors that can requested at a time by their + hashes. + +.. data:: DIRECTORY_AUTHORITIES + + Mapping of directory authority nicknames to their (address, dirport) tuple. """ import io @@ -51,8 +80,8 @@ import stem.descriptor from stem import Flag from stem.util import log -# Tor has a limit on the number of descriptors we can fetch explicitly by their -# fingerprint or hashes due to the url lenght of squid proxies. +# Tor has a limited number of descriptors we can fetch explicitly by their +# fingerprint or hashes due to a limit on the url length by squid proxies. MAX_DESCRIPTOR_BATCH_SIZE = 96 MAX_MICRODESCRIPTOR_BATCH_SIZE = 92 @@ -75,15 +104,87 @@ DIRECTORY_AUTHORITIES = { } +def _guess_descriptor_type(resource): + # Attempts to determine the descriptor type based on the resource url. This + # raises a ValueError if the resource isn't recognized. + + if resource.startswith('/tor/server/'): + return 'server-descriptor 1.0' + elif resource.startswith('/tor/extra/'): + return 'extra-info 1.0' + elif resource.startswith('/tor/micro/'): + return 'microdescriptor 1.0' + elif resource.startswith('/tor/status-vote/'): + return 'network-status-consensus-3 1.0' + else: + raise ValueError("Unable to determine the descriptor type for '%s'" % resource) + + class Query(object): """ Asynchronous request for descriptor content from a directory authority or - mirror. The caller can block on the response by either calling - :func:~stem.descriptor.remote.run: or iterating over our descriptor content. + mirror. These can either be made through the + :class:`~stem.descriptor.remote.DescriptorDownloader` or directly for more + advanced usage. + + To block on the response and get results either call + :func:`~stem.descriptor.remote.Query.run` or iterate over the Query. The + :func:`~stem.descriptor.remote.run` method pass along any errors that + arise... + + :: + + from stem.descriptor.remote import Query + + query = Query( + '/tor/server/all.z', + 'server-descriptor 1.0', + timeout = 30, + ) + + print "Current relays:" + + try: + for desc in query.run(): + print desc.fingerprint + except Exception as exc: + print "Unable to retrieve the server descriptors: %s" % exc + + ... while iterating fails silently... + + :: - :var str resource: resource being fetched, such as '/tor/status-vote/current/consensus.z' - :var str descriptor_type: type of descriptors being fetched, see - :func:`~stem.descriptor.__init__.parse_file` + print "Current relays:" + + for desc in Query('/tor/server/all.z', 'server-descriptor 1.0'): + print desc.fingerprint + + In either case exceptions are available via our 'error' attribute. + + Tor provides quite a few different descriptor resources via its directory + protocol (see section 4.2 and later of the `dir-spec + <https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt>`_). + Commonly useful ones include... + + ======== =========== + Resource Description + ======== =========== + /tor/server/all.z all present server descriptors + /tor/server/fp/<fp1>+<fp2>+<fp3>.z server descriptors with the given fingerprints + /tor/extra/all.z all present extrainfo descriptors + /tor/extra/fp/<fp1>+<fp2>+<fp3>.z extrainfo descriptors with the given fingerprints + /tor/micro/d/<hash1>-<hash2>.z microdescriptors with the given hashes + /tor/status-vote/current/consensus.z present consensus + ======== =========== + + The '.z' suffix can be excluded to get a plaintext rather than compressed + response. Compression is handled transparently, so this shouldn't matter to + the caller. + + :var str resource: resource being fetched, such as '/tor/server/all.z' + :var str descriptor_type: type of descriptors being fetched (for options see + :func:`~stem.descriptor.__init__.parse_file`), this is guessed from the + resource if **None** :var list endpoints: (address, dirport) tuples of the authority or mirror we're querying, this uses authorities if undefined @@ -108,9 +209,16 @@ class Query(object): which to parse the :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` """ - def __init__(self, resource, descriptor_type, endpoints = None, retries = 2, fall_back_to_authority = True, timeout = None, start = True, validate = True, document_handler = stem.descriptor.DocumentHandler.ENTRIES): + def __init__(self, resource, descriptor_type = None, endpoints = None, retries = 2, fall_back_to_authority = True, timeout = None, start = True, validate = True, document_handler = stem.descriptor.DocumentHandler.ENTRIES): + if not resource.startswith('/'): + raise ValueError("Resources should start with a '/': %s" % resource) + self.resource = resource - self.descriptor_type = descriptor_type + + if descriptor_type: + self.descriptor_type = descriptor_type + else: + self.descriptor_type = _guess_descriptor_type(resource) self.endpoints = endpoints if endpoints else [] self.retries = retries @@ -135,24 +243,6 @@ class Query(object): if start: self.start() - def pick_url(self, use_authority = False): - """ - Provides a url that can be queried. If we have multiple endpoints then one - will be picked randomly. - - :param bool use_authority: ignores our endpoints and uses a directory - authority instead - - :returns: **str** for the url being queried by this request - """ - - if use_authority or not self.endpoints: - address, dirport = random.choice(DIRECTORY_AUTHORITIES.values()) - else: - address, dirport = random.choice(self.endpoints) - - return "http://%s:%i/%s" % (address, dirport, self.resource.lstrip('/')) - def start(self): """ Starts downloading the scriptors if we haven't started already. @@ -160,7 +250,12 @@ class Query(object): with self._downloader_thread_lock: if self._downloader_thread is None: - self._downloader_thread = threading.Thread(target = self._download_descriptors, name="Descriptor Query", args = (self.retries,)) + self._downloader_thread = threading.Thread( + name = "Descriptor Query", + target = self._download_descriptors, + args = (self.retries,) + ) + self._downloader_thread.setDaemon(True) self._downloader_thread.start() @@ -190,38 +285,57 @@ class Query(object): self._downloader_thread.join() if self.error: - if not suppress: - raise self.error + if suppress: + return + + raise self.error else: if self._results is None: - if not suppress: - raise ValueError('BUG: _download_descriptors() finished without either results or an error') + if suppress: + return - return + raise ValueError('BUG: _download_descriptors() finished without either results or an error') try: for desc in self._results: yield desc except ValueError as exc: - # encountered a parsing error + self.error = exc # encountered a parsing error - self.error = exc + if suppress: + return - if not suppress: - raise self.error + raise self.error def __iter__(self): for desc in self.run(True): yield desc + def _pick_url(self, use_authority = False): + """ + Provides a url that can be queried. If we have multiple endpoints then one + will be picked randomly. + + :param bool use_authority: ignores our endpoints and uses a directory + authority instead + + :returns: **str** for the url being queried by this request + """ + + if use_authority or not self.endpoints: + address, dirport = random.choice(DIRECTORY_AUTHORITIES.values()) + else: + address, dirport = random.choice(self.endpoints) + + return "http://%s:%i/%s" % (address, dirport, self.resource.lstrip('/')) + def _download_descriptors(self, retries): try: use_authority = retries == 0 and self.fall_back_to_authority - self.download_url = self.pick_url(use_authority) + self.download_url = self._pick_url(use_authority) self.start_time = time.time() response = urllib2.urlopen(self.download_url, timeout = self.timeout) - self.runtime = time.time() - self.start_time # This sucks. We need to read the full response into memory before # processing the content. This is because urllib2 returns a 'file like' @@ -229,8 +343,9 @@ class Query(object): # own buffer that does support these. response = io.BytesIO(response.read().strip()) - self._results = stem.descriptor.parse_file(response, self.descriptor_type, validate = self.validate, document_handler = self.document_handler) + + self.runtime = time.time() - self.start_time log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime)) except: exc = sys.exc_info()[1] @@ -254,26 +369,27 @@ class DescriptorDownloader(object): For more advanced use cases you can use the :class:`~stem.descriptor.remote.Query` class directly. + :param bool use_mirrors: downloads the present consensus and uses the directory + mirrors to fetch future requests, this fails silently if the consensus + cannot be downloaded + :var int retries: number of times to attempt the request if it fails + :var bool fall_back_to_authority: when retrying request issues the last + request to a directory authority if **True** :var float timeout: duration before we'll time out our request, no timeout is applied if **None** - :var bool use_mirrors: downloads the present consensus and uses the directory - mirrors to fetch future requests, this fails silently if the consensus - cannot be downloaded :var bool start_when_requested: issues requests when our methods are called if **True**, otherwise provides non-running :class:`~stem.descriptor.remote.Query` instances :var bool validate: checks the validity of the descriptor's content if **True**, skips these checks otherwise - :var bool fall_back_to_authority: when retrying request issues the last - request to a directory authority if **True** """ - def __init__(self, retries = 2, use_mirrors = False, fall_back_to_authority = True, timeout = None, start_when_requested = True, validate = True): + def __init__(self, use_mirrors = False, retries = 2, fall_back_to_authority = True, timeout = None, start_when_requested = True, validate = True): self.retries = retries + self.fall_back_to_authority = fall_back_to_authority self.timeout = timeout self.start_when_requested = start_when_requested - self.fall_back_to_authority = fall_back_to_authority self.validate = validate self._endpoints = DIRECTORY_AUTHORITIES.values() @@ -282,7 +398,7 @@ class DescriptorDownloader(object): start_time = time.time() self.use_directory_mirrors() log.debug("Retrieve directory mirrors (took %0.2fs)" % (time.time() - start_time)) - except Exception, exc: + except Exception as exc: log.debug("Unable to retrieve directory mirrors: %s" % exc) def use_directory_mirrors(self): @@ -295,10 +411,7 @@ class DescriptorDownloader(object): new_endpoints = set(DIRECTORY_AUTHORITIES.values()) - query = self.get_consensus() - query.run() # running explicitly so we'll raise errors - - for desc in query: + for desc in self.get_consensus().run(): if Flag.V2DIR in desc.flags: new_endpoints.add((desc.address, desc.dir_port)) @@ -332,7 +445,7 @@ class DescriptorDownloader(object): resource = '/tor/server/fp/%s' % '+'.join(fingerprints) - return self._query(resource, 'server-descriptor 1.0') + return self._query(resource) def get_extrainfo_descriptors(self, fingerprints = None): """ @@ -360,7 +473,7 @@ class DescriptorDownloader(object): resource = '/tor/extra/fp/%s' % '+'.join(fingerprints) - return self._query(resource, 'extra-info 1.0') + return self._query(resource) def get_microdescriptors(self, hashes): """ @@ -385,9 +498,7 @@ class DescriptorDownloader(object): if len(hashes) > MAX_MICRODESCRIPTOR_BATCH_SIZE: raise ValueError("Unable to request more than %i microdescriptors at a time by their hashes" % MAX_MICRODESCRIPTOR_BATCH_SIZE) - resource = '/tor/micro/d/%s' % '-'.join(hashes) - - return self._query(resource, 'microdescriptor 1.0') + return self._query('/tor/micro/d/%s' % '-'.join(hashes)) def get_consensus(self, document_handler = stem.descriptor.DocumentHandler.ENTRIES, authority_v3ident = None): """ @@ -409,13 +520,9 @@ class DescriptorDownloader(object): if authority_v3ident: resource += '/%s' % authority_v3ident - return self._query( - resource, - 'network-status-consensus-3 1.0', - document_handler = document_handler, - ) + return self._query(resource, document_handler = document_handler) - def _query(self, resource, descriptor_type, document_handler = stem.descriptor.DocumentHandler.ENTRIES): + def _query(self, resource, descriptor_type = None, document_handler = stem.descriptor.DocumentHandler.ENTRIES): """ Issues a request for the given resource. """ diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index e9d4e8a..7c45118 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -15,7 +15,8 @@ import test.runner import stem.descriptor.networkstatus -class TestDescriptorReader(unittest.TestCase): + +class TestDescriptorDownloader(unittest.TestCase): def test_using_authorities(self): """ Fetches a descriptor from each of the directory authorities. This is @@ -44,7 +45,7 @@ class TestDescriptorReader(unittest.TestCase): for query in queries: try: descriptors = list(query.run()) - except Exception, exc: + except Exception as exc: self.fail("Unable to use %s (%s:%s, %s): %s" % (authority, address, dirport, type(exc), exc)) self.assertEqual(1, len(descriptors)) diff --git a/test/settings.cfg b/test/settings.cfg index d0fd5b0..80aaf9a 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -187,7 +187,7 @@ test.integ_tests |test.integ.util.proc.TestProc |test.integ.util.system.TestSystem |test.integ.descriptor.reader.TestDescriptorReader -|test.integ.descriptor.remote.TestDescriptorReader +|test.integ.descriptor.remote.TestDescriptorDownloader |test.integ.descriptor.server_descriptor.TestServerDescriptor |test.integ.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor |test.integ.descriptor.microdescriptor.TestMicrodescriptor diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index 3aadaac..fb2e3f0 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -71,7 +71,7 @@ class TestDescriptorDownloader(unittest.TestCase): ) expeced_url = 'http://128.31.0.39:9131/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020C…' - self.assertEqual(expeced_url, query.pick_url()) + self.assertEqual(expeced_url, query._pick_url()) descriptors = list(query) self.assertEqual(1, len(descriptors))

1 0

[stem/master] Making descriptor validation optional
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 25951d4f12a16a1fa6e42ea6b7479959d636139b Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jul 20 10:00:04 2013 -0700 Making descriptor validation optional Adding a 'validate' argument that can be passed thorough when parsing descriptors. --- stem/descriptor/remote.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index e0ca612..83416c8 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -84,8 +84,6 @@ class Query(object): :var str resource: resource being fetched, such as '/tor/status-vote/current/consensus.z' :var str descriptor_type: type of descriptors being fetched, see :func:`~stem.descriptor.__init__.parse_file` - :param stem.descriptor.__init__.DocumentHandler document_handler: method in - which to parse the :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` :var list endpoints: (address, dirport) tuples of the authority or mirror we're querying, this uses authorities if undefined @@ -103,12 +101,16 @@ class Query(object): :var float timeout: duration before we'll time out our request :var float runtime: time our query took, this is **None** if it's not yet finished + + :var bool validate: checks the validity of the descriptor's content if + **True**, skips these checks otherwise + :var stem.descriptor.__init__.DocumentHandler document_handler: method in + which to parse the :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` """ - def __init__(self, resource, descriptor_type, endpoints = None, retries = 2, fall_back_to_authority = True, timeout = None, start = True, document_handler = stem.descriptor.DocumentHandler.ENTRIES): + def __init__(self, resource, descriptor_type, endpoints = None, retries = 2, fall_back_to_authority = True, timeout = None, start = True, validate = True, document_handler = stem.descriptor.DocumentHandler.ENTRIES): self.resource = resource self.descriptor_type = descriptor_type - self.document_handler = document_handler self.endpoints = endpoints if endpoints else [] self.retries = retries @@ -122,6 +124,9 @@ class Query(object): self.timeout = timeout self.runtime = None + self.validate = validate + self.document_handler = document_handler + self._downloader_thread = None self._downloader_thread_lock = threading.RLock() @@ -225,7 +230,7 @@ class Query(object): response = io.BytesIO(response.read().strip()) - self._results = stem.descriptor.parse_file(response, self.descriptor_type, document_handler = self.document_handler) + self._results = stem.descriptor.parse_file(response, self.descriptor_type, validate = self.validate, document_handler = self.document_handler) log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime)) except: exc = sys.exc_info()[1] @@ -258,15 +263,18 @@ class DescriptorDownloader(object): :var bool start_when_requested: issues requests when our methods are called if **True**, otherwise provides non-running :class:`~stem.descriptor.remote.Query` instances + :var bool validate: checks the validity of the descriptor's content if + **True**, skips these checks otherwise :var bool fall_back_to_authority: when retrying request issues the last request to a directory authority if **True** """ - def __init__(self, retries = 2, use_mirrors = False, fall_back_to_authority = True, timeout = None, start_when_requested = True): + def __init__(self, retries = 2, use_mirrors = False, fall_back_to_authority = True, timeout = None, start_when_requested = True, validate = True): self.retries = retries self.timeout = timeout self.start_when_requested = start_when_requested self.fall_back_to_authority = fall_back_to_authority + self.validate = validate self._endpoints = DIRECTORY_AUTHORITIES.values() if use_mirrors: @@ -422,5 +430,6 @@ class DescriptorDownloader(object): fall_back_to_authority = self.fall_back_to_authority, timeout = self.timeout, start = self.start_when_requested, + validate = self.validate, document_handler = document_handler, )

1 0

[stem/master] Simplifying DescriptorDownloader class
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 6f12acc9e54842371f52203550a6ef2f9cbfb835 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 21 16:25:58 2013 -0700 Simplifying DescriptorDownloader class The DescriptorDownloader is a very, very simple class that simply issues Queries on your behalf. Why should it pretend to be anything else? Dropping most of its attributes (and with it a lot of copy-and-pasted pydocs). Also making the query() method public. --- stem/descriptor/remote.py | 62 ++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 37 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 777f769..01130f3 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -51,7 +51,8 @@ itself... |- get_server_descriptors - provides present :class:`~stem.descriptor.stem.descriptor.server_descriptor.ServerDescriptor` |- get_extrainfo_descriptors - provides present :class:`~stem.descriptor.extrainfo_descriptor.ExtraInfoDescriptor` |- get_microdescriptors - provides present :class:`~stem.descriptor.microdescriptor.Microdescriptor` - +- get_consensus - provides present :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV3` + |- get_consensus - provides present :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV3` + +- query - request an arbitrary descriptor resource .. data:: MAX_DESCRIPTOR_BATCH_SIZE @@ -365,35 +366,18 @@ class Query(object): class DescriptorDownloader(object): """ - Configurable class through which descriptors can be downloaded. This provides - caching, retries, and other capabilities to make downloading descriptors easy - and efficient. - - For more advanced use cases you can use the - :class:`~stem.descriptor.remote.Query` class directly. + Configurable class that issues :class:`~stem.descriptor.remote.Query` + instances on your behalf. :param bool use_mirrors: downloads the present consensus and uses the directory mirrors to fetch future requests, this fails silently if the consensus cannot be downloaded - - :var int retries: number of times to attempt the request if it fails - :var bool fall_back_to_authority: when retrying request issues the last - request to a directory authority if **True** - :var float timeout: duration before we'll time out our request, no timeout is - applied if **None** - :var bool start_when_requested: issues requests when our methods are called - if **True**, otherwise provides non-running - :class:`~stem.descriptor.remote.Query` instances - :var bool validate: checks the validity of the descriptor's content if - **True**, skips these checks otherwise + :param default_args: default arguments for the + :class:`~stem.descriptor.remote.Query` constructor """ - def __init__(self, use_mirrors = False, retries = 2, fall_back_to_authority = True, timeout = None, start_when_requested = True, validate = True): - self.retries = retries - self.fall_back_to_authority = fall_back_to_authority - self.timeout = timeout - self.start_when_requested = start_when_requested - self.validate = validate + def __init__(self, use_mirrors = False, **default_args): + self._default_args = default_args self._endpoints = DIRECTORY_AUTHORITIES.values() if use_mirrors: @@ -448,7 +432,7 @@ class DescriptorDownloader(object): resource = '/tor/server/fp/%s.z' % '+'.join(fingerprints) - return self._query(resource) + return self.query(resource) def get_extrainfo_descriptors(self, fingerprints = None): """ @@ -476,7 +460,7 @@ class DescriptorDownloader(object): resource = '/tor/extra/fp/%s.z' % '+'.join(fingerprints) - return self._query(resource) + return self.query(resource) def get_microdescriptors(self, hashes): """ @@ -501,7 +485,7 @@ class DescriptorDownloader(object): if len(hashes) > MAX_MICRODESCRIPTOR_BATCH_SIZE: raise ValueError("Unable to request more than %i microdescriptors at a time by their hashes" % MAX_MICRODESCRIPTOR_BATCH_SIZE) - return self._query('/tor/micro/d/%s.z' % '-'.join(hashes)) + return self.query('/tor/micro/d/%s.z' % '-'.join(hashes)) def get_consensus(self, document_handler = stem.descriptor.DocumentHandler.ENTRIES, authority_v3ident = None): """ @@ -523,23 +507,27 @@ class DescriptorDownloader(object): if authority_v3ident: resource += '/%s' % authority_v3ident - return self._query(resource + '.z', document_handler = document_handler) + return self.query(resource + '.z', document_handler = document_handler) - def _query(self, resource, descriptor_type = None, document_handler = stem.descriptor.DocumentHandler.ENTRIES): + def query(self, resource, **kwargs): """ Issues a request for the given resource. + + :param str resource: resource being fetched, such as '/tor/server/all.z' + :param kwargs: additional arguments for the + :class:`~stem.descriptor.remote.Query` constructor + + :returns: :class:`~stem.descriptor.remote.Query` for the descriptors + + :raises: **ValueError** if resource is clearly invalid or the descriptor + type can't be determined when 'descriptor_type' is **None** """ - log.trace("Retrieving descriptors (resource: %s, type: %s)" % (resource, descriptor_type)) + query_args = dict(self._default_args) + query_args.update(kwargs) return Query( resource, - descriptor_type, endpoints = self._endpoints, - retries = self.retries, - fall_back_to_authority = self.fall_back_to_authority, - timeout = self.timeout, - start = self.start_when_requested, - validate = self.validate, - document_handler = document_handler, + **query_args )

1 0

[stem/master] Adding a query_args parameter to all downloader methods
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit e95e00dd6431001000a79e2bfc009a7f3abbde80 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jul 21 16:31:00 2013 -0700 Adding a query_args parameter to all downloader methods Simple tweak in case callers want to do something dynamic (such as varying the timeout with each request). --- stem/descriptor/remote.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 01130f3..a009078 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -406,7 +406,7 @@ class DescriptorDownloader(object): self._endpoints = list(new_endpoints) - def get_server_descriptors(self, fingerprints = None): + def get_server_descriptors(self, fingerprints = None, **query_args): """ Provides the server descriptors with the given fingerprints. If no fingerprints are provided then this returns all descriptors in the present @@ -414,6 +414,8 @@ class DescriptorDownloader(object): :param str,list fingerprints: fingerprint or list of fingerprints to be retrieved, gets all descriptors if **None** + :param query_args: additional arguments for the + :class:`~stem.descriptor.remote.Query` constructor :returns: :class:`~stem.descriptor.remote.Query` for the server descriptors @@ -432,9 +434,9 @@ class DescriptorDownloader(object): resource = '/tor/server/fp/%s.z' % '+'.join(fingerprints) - return self.query(resource) + return self.query(resource, **query_args) - def get_extrainfo_descriptors(self, fingerprints = None): + def get_extrainfo_descriptors(self, fingerprints = None, **query_args): """ Provides the extrainfo descriptors with the given fingerprints. If no fingerprints are provided then this returns all descriptors in the present @@ -442,6 +444,8 @@ class DescriptorDownloader(object): :param str,list fingerprints: fingerprint or list of fingerprints to be retrieved, gets all descriptors if **None** + :param query_args: additional arguments for the + :class:`~stem.descriptor.remote.Query` constructor :returns: :class:`~stem.descriptor.remote.Query` for the extrainfo descriptors @@ -460,9 +464,9 @@ class DescriptorDownloader(object): resource = '/tor/extra/fp/%s.z' % '+'.join(fingerprints) - return self.query(resource) + return self.query(resource, **query_args) - def get_microdescriptors(self, hashes): + def get_microdescriptors(self, hashes, **query_args): """ Provides the microdescriptors with the given hashes. To get these see the 'microdescriptor_hashes' attribute of @@ -472,6 +476,8 @@ class DescriptorDownloader(object): :param str,list hashes: microdescriptor hash or list of hashes to be retrieved + :param query_args: additional arguments for the + :class:`~stem.descriptor.remote.Query` constructor :returns: :class:`~stem.descriptor.remote.Query` for the microdescriptors @@ -485,9 +491,9 @@ class DescriptorDownloader(object): if len(hashes) > MAX_MICRODESCRIPTOR_BATCH_SIZE: raise ValueError("Unable to request more than %i microdescriptors at a time by their hashes" % MAX_MICRODESCRIPTOR_BATCH_SIZE) - return self.query('/tor/micro/d/%s.z' % '-'.join(hashes)) + return self.query('/tor/micro/d/%s.z' % '-'.join(hashes), **query_args) - def get_consensus(self, document_handler = stem.descriptor.DocumentHandler.ENTRIES, authority_v3ident = None): + def get_consensus(self, document_handler = stem.descriptor.DocumentHandler.ENTRIES, authority_v3ident = None, **query_args): """ Provides the present router status entries. @@ -497,6 +503,8 @@ class DescriptorDownloader(object): to get the consensus, see `'v3ident' in tor's config.c <https://gitweb.torproject.org/tor.git/blob/f631b73:/src/or/config.c#l816>`_ for the values. + :param query_args: additional arguments for the + :class:`~stem.descriptor.remote.Query` constructor :returns: :class:`~stem.descriptor.remote.Query` for the router status entries @@ -507,14 +515,14 @@ class DescriptorDownloader(object): if authority_v3ident: resource += '/%s' % authority_v3ident - return self.query(resource + '.z', document_handler = document_handler) + return self.query(resource + '.z', document_handler = document_handler, **query_args) - def query(self, resource, **kwargs): + def query(self, resource, **query_args): """ Issues a request for the given resource. :param str resource: resource being fetched, such as '/tor/server/all.z' - :param kwargs: additional arguments for the + :param query_args: additional arguments for the :class:`~stem.descriptor.remote.Query` constructor :returns: :class:`~stem.descriptor.remote.Query` for the descriptors @@ -523,11 +531,11 @@ class DescriptorDownloader(object): type can't be determined when 'descriptor_type' is **None** """ - query_args = dict(self._default_args) - query_args.update(kwargs) + args = dict(self._default_args) + args.update(query_args) return Query( resource, endpoints = self._endpoints, - **query_args + **args )

1 0

[stem/master] Implementing a get_consensus() method
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 5514b2cfd7d313f429bfe1bb8a13c975f7c3413f Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jul 18 09:59:30 2013 -0700 Implementing a get_consensus() method Originally this was gonna be a get_network_status(fingerprint) method but evedently we can't request individual router status entries. Understandable since signatures are for the whole document but still a bit of a pity from an API perspective. Oh well. --- stem/descriptor/__init__.py | 2 +- stem/descriptor/remote.py | 60 +++++++++++++++++++++++++++++---------- test/integ/descriptor/remote.py | 32 ++++++++++++++++++++- test/runner.py | 1 + 4 files changed, 78 insertions(+), 17 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index a527329..82f846d 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -132,7 +132,7 @@ def parse_file(descriptor_file, descriptor_type = None, validate = True, documen :param bool validate: checks the validity of the descriptor's content if **True**, skips these checks otherwise :param stem.descriptor.__init__.DocumentHandler document_handler: method in - which to parse :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` + which to parse the :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` :returns: iterator for :class:`~stem.descriptor.__init__.Descriptor` instances in the file diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index a18d991..66e6d85 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -82,6 +82,8 @@ class Query(object): :var str resource: resource being fetched, such as '/tor/status-vote/current/consensus.z' :var str descriptor_type: type of descriptors being fetched, see :func:`~stem.descriptor.__init__.parse_file` + :param stem.descriptor.__init__.DocumentHandler document_handler: method in + which to parse the :class:`~stem.descriptor.networkstatus.NetworkStatusDocument` :var list endpoints: (address, dirport) tuples of the authority or mirror we're querying, this uses authorities if undefined @@ -101,9 +103,10 @@ class Query(object): finished """ - def __init__(self, resource, descriptor_type, endpoints = None, retries = 2, fall_back_to_authority = True, timeout = None, start = True): + def __init__(self, resource, descriptor_type, endpoints = None, retries = 2, fall_back_to_authority = True, timeout = None, start = True, document_handler = stem.descriptor.DocumentHandler.ENTRIES): self.resource = resource self.descriptor_type = descriptor_type + self.document_handler = document_handler self.endpoints = endpoints if endpoints else [] self.retries = retries @@ -220,7 +223,7 @@ class Query(object): response = io.BytesIO(response.read().strip()) - self._results = stem.descriptor.parse_file(response, self.descriptor_type) + self._results = stem.descriptor.parse_file(response, self.descriptor_type, document_handler = self.document_handler) log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime)) except: exc = sys.exc_info()[1] @@ -278,14 +281,14 @@ class DescriptorDownloader(object): resource = '/tor/server/all' + if isinstance(fingerprints, str): + fingerprints = [fingerprints] + if fingerprints: - if isinstance(fingerprints, str): - resource = '/tor/server/fp/%s' % fingerprints - else: - if len(fingerprints) > MAX_BATCH_SIZE: - raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) + if len(fingerprints) > MAX_BATCH_SIZE: + raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) - resource = '/tor/server/fp/%s' % '+'.join(fingerprints) + resource = '/tor/server/fp/%s' % '+'.join(fingerprints) return self._query(resource, 'server-descriptor 1.0') @@ -306,18 +309,44 @@ class DescriptorDownloader(object): resource = '/tor/extra/all' + if isinstance(fingerprints, str): + fingerprints = [fingerprints] + if fingerprints: - if isinstance(fingerprints, str): - resource = '/tor/extra/fp/%s' % fingerprints - else: - if len(fingerprints) > MAX_BATCH_SIZE: - raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) + if len(fingerprints) > MAX_BATCH_SIZE: + raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) - resource = '/tor/extra/fp/%s' % '+'.join(fingerprints) + resource = '/tor/extra/fp/%s' % '+'.join(fingerprints) return self._query(resource, 'extra-info 1.0') - def _query(self, resource, descriptor_type): + def get_consensus(self, document_handler = stem.descriptor.DocumentHandler.ENTRIES, authority_v3ident = None): + """ + Provides the present router status entries. + + :param stem.descriptor.__init__.DocumentHandler document_handler: method in + which to parse the :class:`~stem.descriptor.networkstatus.NetworkStatusDocumentV3` + :param str authority_v3ident: fingerprint of the authority key for which + to get the consensus, see `'v3ident' in tor's config.c + <https://gitweb.torproject.org/tor.git/blob/f631b73:/src/or/config.c#l816>`_ + for the values. + + :returns: :class:`~stem.descriptor.remote.Query` for the router status + entries + """ + + resource = '/tor/status-vote/current/consensus' + + if authority_v3ident: + resource += '/%s' % authority_v3ident + + return self._query( + resource, + 'network-status-consensus-3 1.0', + document_handler = document_handler, + ) + + def _query(self, resource, descriptor_type, document_handler = stem.descriptor.DocumentHandler.ENTRIES): """ Issues a request for the given resource. """ @@ -330,4 +359,5 @@ class DescriptorDownloader(object): fall_back_to_authority = self.fall_back_to_authority, timeout = self.timeout, start = self.start_when_requested, + document_handler = document_handler, ) diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index 62348ab..9273b61 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -4,9 +4,10 @@ Integration tests for stem.descriptor.remote. import unittest -import stem.descriptor.server_descriptor import stem.descriptor.extrainfo_descriptor import stem.descriptor.remote +import stem.descriptor.router_status_entry +import stem.descriptor.server_descriptor import test.runner # Required to prevent unmarshal error when running this test alone. @@ -26,6 +27,8 @@ class TestDescriptorReader(unittest.TestCase): if test.runner.require_online(self): return + elif test.runner.only_run_once(self, "test_using_authorities"): + return queries = [] @@ -51,6 +54,11 @@ class TestDescriptorReader(unittest.TestCase): Exercises the downloader's get_server_descriptors() method. """ + if test.runner.require_online(self): + return + elif test.runner.only_run_once(self, "test_get_server_descriptors"): + return + downloader = stem.descriptor.remote.DescriptorDownloader() # Fetch a single descriptor and a batch. I'd love to also exercise @@ -82,6 +90,11 @@ class TestDescriptorReader(unittest.TestCase): Exercises the downloader's get_extrainfo_descriptors() method. """ + if test.runner.require_online(self): + return + elif test.runner.only_run_once(self, "test_get_extrainfo_descriptors"): + return + downloader = stem.descriptor.remote.DescriptorDownloader() single_query = downloader.get_extrainfo_descriptors('9695DFC35FFEB861329B9F1AB04C46397020CE31') @@ -101,4 +114,21 @@ class TestDescriptorReader(unittest.TestCase): self.assertEqual(2, len(list(multiple_query))) + def test_get_consensus(self): + """ + Exercises the downloader's get_consensus() method. + """ + + if test.runner.require_online(self): + return + elif test.runner.only_run_once(self, "test_get_consensus"): + return + + downloader = stem.descriptor.remote.DescriptorDownloader() + + consensus_query = downloader.get_consensus() + consensus_query.run() + consensus = list(consensus_query) + self.assertTrue(len(consensus) > 50) + self.assertTrue(isinstance(consensus[0], stem.descriptor.router_status_entry.RouterStatusEntryV3)) diff --git a/test/runner.py b/test/runner.py index 9bd62f7..a1eda9e 100644 --- a/test/runner.py +++ b/test/runner.py @@ -16,6 +16,7 @@ about the tor test instance they're running against. require_control - skips the test unless tor provides a controller endpoint require_version - skips the test unless we meet a tor version requirement require_online - skips unless targets allow for online tests + only_run_once - skip the test if it has been ran before exercise_controller - basic sanity check that a controller connection can be used get_runner - Singleton for fetching our runtime context.

1 0

[stem/master] Implementing a use_directory_mirrors() method
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 4005a88becb68858ec4742455ea4d80e1ff9f827 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jul 19 10:12:39 2013 -0700 Implementing a use_directory_mirrors() method Method so we can balance load against directory mirrors rather than hammering the authorities. We can either request this during construction (which fails silently) or call this method explicitly (to get the exception). --- stem/descriptor/remote.py | 37 ++++++++++++++++++++++++++++++++++++- test/integ/descriptor/remote.py | 14 ++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 66e6d85..9df09f4 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -48,6 +48,7 @@ import urllib2 import stem.descriptor +from stem import Flag from stem.util import log # Tor has a limit on the number of descriptors we can fetch explicitly by their @@ -250,6 +251,9 @@ class DescriptorDownloader(object): :var int retries: number of times to attempt the request if it fails :var float timeout: duration before we'll time out our request, no timeout is applied if **None** + :var bool use_mirrors: downloads the present consensus and uses the directory + mirrors to fetch future requests, this fails silently if the consensus + cannot be downloaded :var bool start_when_requested: issues requests when our methods are called if **True**, otherwise provides non-running :class:`~stem.descriptor.remote.Query` instances @@ -257,13 +261,42 @@ class DescriptorDownloader(object): request to a directory authority if **True** """ - def __init__(self, retries = 2, fall_back_to_authority = True, timeout = None, start_when_requested = True): + def __init__(self, retries = 2, use_mirrors = False, fall_back_to_authority = True, timeout = None, start_when_requested = True): self.retries = retries self.timeout = timeout self.start_when_requested = start_when_requested self.fall_back_to_authority = fall_back_to_authority self._endpoints = DIRECTORY_AUTHORITIES.values() + if use_mirrors: + try: + start_time = time.time() + self.use_directory_mirrors() + log.debug("Retrieve directory mirrors (took %0.2fs)" % (time.time() - start_time)) + except Exception, exc: + log.debug("Unable to retrieve directory mirrors: %s" % exc) + + def use_directory_mirrors(self): + """ + Downloads the present consensus and configures ourselves to use directory + mirrors, in addition to authorities. + + :raises: **Exception** if unable to determine the directory mirrors + """ + + new_endpoints = set(DIRECTORY_AUTHORITIES.values()) + + query = self.get_consensus() + query.run() # running explicitly so we'll raise errors + + for desc in query: + if Flag.V2DIR in desc.flags: + new_endpoints.add((desc.address, desc.dir_port)) + + # we need our endpoints to be a list rather than set for random.choice() + + self._endpoints = list(new_endpoints) + def get_server_descriptors(self, fingerprints = None): """ Provides the server descriptors with the given fingerprints. If no @@ -351,6 +384,8 @@ class DescriptorDownloader(object): Issues a request for the given resource. """ + log.trace("Retrieving descriptors (resource: %s, type: %s)" % (resource, descriptor_type)) + return Query( resource, descriptor_type, diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index 9273b61..a705486 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -49,6 +49,20 @@ class TestDescriptorReader(unittest.TestCase): self.assertEqual(1, len(descriptors)) self.assertEqual('moria1', descriptors[0].nickname) + def test_use_directory_mirrors(self): + """ + Checks that we can fetch and use a list of directory mirrors. + """ + + if test.runner.require_online(self): + return + elif test.runner.only_run_once(self, "test_use_directory_mirrors"): + return + + downloader = stem.descriptor.remote.DescriptorDownloader() + downloader.use_directory_mirrors() + self.assertTrue(len(downloader._endpoints) > 50) + def test_get_server_descriptors(self): """ Exercises the downloader's get_server_descriptors() method.

1 0

[stem/master] Adding a broken get_microdescriptors() method
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 749c8d37fd54ea6e41dfca04495c35b47a755f6a Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jul 20 09:52:07 2013 -0700 Adding a broken get_microdescriptors() method Huh. Not really sure why this isn't working. Adding a get_microdescriptors() for how the spec seems to indicate that the method works but tor gives an empty reply. % curl http://217.13.197.5:9030/tor/micro/d/6dCl6ab8CLo0LeMjxi/MZgVJiZgWN8WKTesWPB… Checking with karsten on irc t see if he knows what's up, and moving on to another part. --- stem/descriptor/remote.py | 40 +++++++++++++++++++++++++++++++++------ test/integ/descriptor/remote.py | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 9df09f4..e0ca612 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -52,9 +52,10 @@ from stem import Flag from stem.util import log # Tor has a limit on the number of descriptors we can fetch explicitly by their -# fingerprint. +# fingerprint or hashes due to the url lenght of squid proxies. -MAX_BATCH_SIZE = 96 +MAX_DESCRIPTOR_BATCH_SIZE = 96 +MAX_MICRODESCRIPTOR_BATCH_SIZE = 92 # Tor directory authorities as of commit f631b73 (7/4/13). This should only # include authorities with 'v3ident': @@ -318,8 +319,8 @@ class DescriptorDownloader(object): fingerprints = [fingerprints] if fingerprints: - if len(fingerprints) > MAX_BATCH_SIZE: - raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) + if len(fingerprints) > MAX_DESCRIPTOR_BATCH_SIZE: + raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_DESCRIPTOR_BATCH_SIZE) resource = '/tor/server/fp/%s' % '+'.join(fingerprints) @@ -346,13 +347,40 @@ class DescriptorDownloader(object): fingerprints = [fingerprints] if fingerprints: - if len(fingerprints) > MAX_BATCH_SIZE: - raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) + if len(fingerprints) > MAX_DESCRIPTOR_BATCH_SIZE: + raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_DESCRIPTOR_BATCH_SIZE) resource = '/tor/extra/fp/%s' % '+'.join(fingerprints) return self._query(resource, 'extra-info 1.0') + def get_microdescriptors(self, hashes): + """ + Provides the microdescriptors with the given hashes. To get these see the + 'microdescriptor_hashes' attribute of + :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV3`. Note + that these are only provided via a microdescriptor consensus (such as + 'cached-microdesc-consensus' in your data directory). + + :param str,list hashes: microdescriptor hash or list of hashes to be + retrieved + + :returns: :class:`~stem.descriptor.remote.Query` for the microdescriptors + + :raises: **ValueError** if we request more than 92 microdescriptors by their + hashes (this is due to a limit on the url length by squid proxies). + """ + + if isinstance(hashes, str): + hashes = [hashes] + + if len(hashes) > MAX_MICRODESCRIPTOR_BATCH_SIZE: + raise ValueError("Unable to request more than %i microdescriptors at a time by their hashes" % MAX_MICRODESCRIPTOR_BATCH_SIZE) + + resource = '/tor/micro/d/%s' % '-'.join(hashes) + + return self._query(resource, 'microdescriptor 1.0') + def get_consensus(self, document_handler = stem.descriptor.DocumentHandler.ENTRIES, authority_v3ident = None): """ Provides the present router status entries. diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index a705486..e9d4e8a 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -5,6 +5,7 @@ Integration tests for stem.descriptor.remote. import unittest import stem.descriptor.extrainfo_descriptor +import stem.descriptor.microdescriptor import stem.descriptor.remote import stem.descriptor.router_status_entry import stem.descriptor.server_descriptor @@ -128,6 +129,40 @@ class TestDescriptorReader(unittest.TestCase): self.assertEqual(2, len(list(multiple_query))) + def test_get_microdescriptors(self): + """ + Exercises the downloader's get_microdescriptors() method. + """ + + # TODO: method needs to be fixed - not quite sure what's going wrong... + + test.runner.skip(self, '(test presently broken)') + return + + if test.runner.require_online(self): + return + elif test.runner.only_run_once(self, "test_get_microdescriptors"): + return + + downloader = stem.descriptor.remote.DescriptorDownloader() + + single_query = downloader.get_microdescriptors('6dCl6ab8CLo0LeMjxi/MZgVJiZgWN8WKTesWPBMtyTo') + + multiple_query = downloader.get_microdescriptors([ + '6dCl6ab8CLo0LeMjxi/MZgVJiZgWN8WKTesWPBMtyTo', # moria1 + 'oXBV80OwMACBJpqNeZrYSXF18l9EJCi4/mB8UOl9sME', # tor26 + ]) + + single_query.run() + multiple_query.run() + + single_query_results = list(single_query) + self.assertEqual(1, len(single_query_results)) + self.assertEqual('moria1', single_query_results[0].digest) + self.assertTrue(isinstance(single_query_results[0], stem.descriptor.microdescriptor.Microdescriptor)) + + self.assertEqual(2, len(list(multiple_query))) + def test_get_consensus(self): """ Exercises the downloader's get_consensus() method.

1 0

[stem/master] Implementing get_extrainfo_descriptors() method
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit a014b9b967e8b825a9a4c99514ce2098ab6ba614 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jul 17 09:58:35 2013 -0700 Implementing get_extrainfo_descriptors() method Very similar to the get_server_descriptors() counterpart. --- stem/descriptor/remote.py | 33 +++++++++++++++++++++++++++++++-- test/integ/descriptor/remote.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index 3021f63..a18d991 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -272,7 +272,8 @@ class DescriptorDownloader(object): :returns: :class:`~stem.descriptor.remote.Query` for the server descriptors - :raises: **ValueError** if we request descriptors by fingerprints and there are more than 96 of them (this is a limitation of tor). + :raises: **ValueError** if we request more than 96 descriptors by their + fingerprints (this is due to a limit on the url length by squid proxies). """ resource = '/tor/server/all' @@ -282,12 +283,40 @@ class DescriptorDownloader(object): resource = '/tor/server/fp/%s' % fingerprints else: if len(fingerprints) > MAX_BATCH_SIZE: - raise ValueError('We can request at most %i descritors at a time when retrieving by fingerprints' % MAX_BATCH_SIZE) + raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) resource = '/tor/server/fp/%s' % '+'.join(fingerprints) return self._query(resource, 'server-descriptor 1.0') + def get_extrainfo_descriptors(self, fingerprints = None): + """ + Provides the extrainfo descriptors with the given fingerprints. If no + fingerprints are provided then this returns all descriptors in the present + consensus. + + :param str,list fingerprints: fingerprint or list of fingerprints to be + retrieved, gets all descriptors if **None** + + :returns: :class:`~stem.descriptor.remote.Query` for the extrainfo descriptors + + :raises: **ValueError** if we request more than 96 descriptors by their + fingerprints (this is due to a limit on the url length by squid proxies). + """ + + resource = '/tor/extra/all' + + if fingerprints: + if isinstance(fingerprints, str): + resource = '/tor/extra/fp/%s' % fingerprints + else: + if len(fingerprints) > MAX_BATCH_SIZE: + raise ValueError("Unable to request more than %i descriptors at a time by their fingerprints" % MAX_BATCH_SIZE) + + resource = '/tor/extra/fp/%s' % '+'.join(fingerprints) + + return self._query(resource, 'extra-info 1.0') + def _query(self, resource, descriptor_type): """ Issues a request for the given resource. diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py index 6e1f426..62348ab 100644 --- a/test/integ/descriptor/remote.py +++ b/test/integ/descriptor/remote.py @@ -4,6 +4,8 @@ Integration tests for stem.descriptor.remote. import unittest +import stem.descriptor.server_descriptor +import stem.descriptor.extrainfo_descriptor import stem.descriptor.remote import test.runner @@ -71,6 +73,32 @@ class TestDescriptorReader(unittest.TestCase): single_query_results = list(single_query) self.assertEqual(1, len(single_query_results)) self.assertEqual('moria1', single_query_results[0].nickname) + self.assertTrue(isinstance(single_query_results[0], stem.descriptor.stem.descriptor.server_descriptor.ServerDescriptor)) self.assertEqual(2, len(list(multiple_query))) + def test_get_extrainfo_descriptors(self): + """ + Exercises the downloader's get_extrainfo_descriptors() method. + """ + + downloader = stem.descriptor.remote.DescriptorDownloader() + + single_query = downloader.get_extrainfo_descriptors('9695DFC35FFEB861329B9F1AB04C46397020CE31') + + multiple_query = downloader.get_extrainfo_descriptors([ + '9695DFC35FFEB861329B9F1AB04C46397020CE31', + '847B1F850344D7876491A54892F904934E4EB85D', + ]) + + single_query.run() + multiple_query.run() + + single_query_results = list(single_query) + self.assertEqual(1, len(single_query_results)) + self.assertEqual('moria1', single_query_results[0].nickname) + self.assertTrue(isinstance(single_query_results[0], stem.descriptor.extrainfo_descriptor.ExtraInfoDescriptor)) + + self.assertEqual(2, len(list(multiple_query))) + +

1 0

[stem/master] Rejecting requests for more than 96 descriptors
by atagar＠torproject.org 22 Jul '13

22 Jul '13

commit 871a957f23097807f67e502fd6e2c9a9b9a6c456 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jul 16 09:28:18 2013 -0700 Rejecting requests for more than 96 descriptors In our tor-dev@ discussion Karsten mentioned that we could request at most 96 descriptors at a time when polling by their fingerprints... https://lists.torproject.org/pipermail/tor-dev/2013-June/005005.html I've emailed him to clarify where this limitation comes from (the url length or something within tor?), but in the meantime adding a check for this to our get_sever_descriptors() method. --- stem/descriptor/remote.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index a4b21d4..3021f63 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -50,6 +50,11 @@ import stem.descriptor from stem.util import log +# Tor has a limit on the number of descriptors we can fetch explicitly by their +# fingerprint. + +MAX_BATCH_SIZE = 96 + # Tor directory authorities as of commit f631b73 (7/4/13). This should only # include authorities with 'v3ident': # @@ -266,6 +271,8 @@ class DescriptorDownloader(object): retrieved, gets all descriptors if **None** :returns: :class:`~stem.descriptor.remote.Query` for the server descriptors + + :raises: **ValueError** if we request descriptors by fingerprints and there are more than 96 of them (this is a limitation of tor). """ resource = '/tor/server/all' @@ -274,6 +281,9 @@ class DescriptorDownloader(object): if isinstance(fingerprints, str): resource = '/tor/server/fp/%s' % fingerprints else: + if len(fingerprints) > MAX_BATCH_SIZE: + raise ValueError('We can request at most %i descritors at a time when retrieving by fingerprints' % MAX_BATCH_SIZE) + resource = '/tor/server/fp/%s' % '+'.join(fingerprints) return self._query(resource, 'server-descriptor 1.0')

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits July 2013