[tor-commits] [stem/master] Implementing get_server_descriptors() method

[atagar at torproject.org]

commit ac694d3a98ce1e56c674893f8006efe785eb41c9
Author: Damian Johnson <atagar at torproject.org>
Date:   Tue Jul 16 09:16:06 2013 -0700

    Implementing get_server_descriptors() method
    
    Implementing our first downloader method and added an ONLINE integ test.
    Initially I did a get_microdescriptors() method but turns out that tor didn't
    implement what we need...
    
      https://trac.torproject.org/9271
---
 stem/descriptor/remote.py       |   47 +++++++++++++++++++++++++++++++--------
 test/integ/descriptor/remote.py |   30 +++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py
index 88e3258..a4b21d4 100644
--- a/stem/descriptor/remote.py
+++ b/stem/descriptor/remote.py
@@ -80,16 +80,20 @@ class Query(object):
 
   :var list endpoints: (address, dirport) tuples of the authority or mirror
     we're querying, this uses authorities if undefined
-  :var int retries: number of times to attempt the request if it fails
+  :var int retries: number of times to attempt the request if downloading it
+    fails
   :var bool fall_back_to_authority: when retrying request issues the last
     request to a directory authority if **True**
 
   :var Exception error: exception if a problem occured
   :var bool is_done: flag that indicates if our request has finished
+  :var str download_url: last url used to download the descriptor, this is
+    unset until we've actually made a download attempt
 
   :var float start_time: unix timestamp when we first started running
   :var float timeout: duration before we'll time out our request
-  :var float runtime: time our query took, this is **None** if it's not yet finished
+  :var float runtime: time our query took, this is **None** if it's not yet
+    finished
   """
 
   def __init__(self, resource, descriptor_type, endpoints = None, retries = 2, fall_back_to_authority = True, timeout = None, start = True):
@@ -102,6 +106,7 @@ class Query(object):
 
     self.error = None
     self.is_done = False
+    self.download_url = None
 
     self.start_time = None
     self.timeout = timeout
@@ -183,6 +188,8 @@ class Query(object):
           for desc in self._results:
             yield desc
         except ValueError as exc:
+          # encountered a parsing error
+
           self.error = exc
 
           if not suppress:
@@ -195,10 +202,10 @@ class Query(object):
   def _download_descriptors(self, retries):
     try:
       use_authority = retries == 0 and self.fall_back_to_authority
-      resource_url = self.pick_url(use_authority)
+      self.download_url = self.pick_url(use_authority)
 
       self.start_time = time.time()
-      response = urllib2.urlopen(resource_url, timeout = self.timeout)
+      response = urllib2.urlopen(self.download_url, timeout = self.timeout)
       self.runtime = time.time() - self.start_time
 
       # This sucks. We need to read the full response into memory before
@@ -209,15 +216,15 @@ class Query(object):
       response = io.BytesIO(response.read().strip())
 
       self._results = stem.descriptor.parse_file(response, self.descriptor_type)
-      log.trace("Descriptors retrieved from '%s' in %0.2fs" % (resource_url, self.runtime))
+      log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime))
     except:
       exc = sys.exc_info()[1]
 
       if retries > 0:
-        log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (resource_url, retries, exc))
+        log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (self.download_url, retries, exc))
         return self._download_descriptors(retries - 1)
       else:
-        log.debug("Unable to download descriptors from '%s': %s" % (resource_url, exc))
+        log.debug("Unable to download descriptors from '%s': %s" % (self.download_url, exc))
         self.error = exc
     finally:
       self.is_done = True
@@ -242,14 +249,36 @@ class DescriptorDownloader(object):
     request to a directory authority if **True**
   """
 
-  def __init__(self, retries = 2, timeout = None, start_when_requested = True, fall_back_to_authority = True):
+  def __init__(self, retries = 2, fall_back_to_authority = True, timeout = None, start_when_requested = True):
     self.retries = retries
     self.timeout = timeout
     self.start_when_requested = start_when_requested
     self.fall_back_to_authority = fall_back_to_authority
     self._endpoints = DIRECTORY_AUTHORITIES.values()
 
-  def _query(self, resource, descriptor_type, retries):
+  def get_server_descriptors(self, fingerprints = None):
+    """
+    Provides the server descriptors with the given fingerprints. If no
+    fingerprints are provided then this returns all descriptors in the present
+    consensus.
+
+    :param str,list fingerprints: fingerprint or list of fingerprints to be
+      retrieved, gets all descriptors if **None**
+
+    :returns: :class:`~stem.descriptor.remote.Query` for the server descriptors
+    """
+
+    resource = '/tor/server/all'
+
+    if fingerprints:
+      if isinstance(fingerprints, str):
+        resource = '/tor/server/fp/%s' % fingerprints
+      else:
+        resource = '/tor/server/fp/%s' % '+'.join(fingerprints)
+
+    return self._query(resource, 'server-descriptor 1.0')
+
+  def _query(self, resource, descriptor_type):
     """
     Issues a request for the given resource.
     """
diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py
index 8509556..6e1f426 100644
--- a/test/integ/descriptor/remote.py
+++ b/test/integ/descriptor/remote.py
@@ -44,3 +44,33 @@ class TestDescriptorReader(unittest.TestCase):
       self.assertEqual(1, len(descriptors))
       self.assertEqual('moria1', descriptors[0].nickname)
 
+  def test_get_server_descriptors(self):
+    """
+    Exercises the downloader's get_server_descriptors() method.
+    """
+
+    downloader = stem.descriptor.remote.DescriptorDownloader()
+
+    # Fetch a single descriptor and a batch. I'd love to also exercise
+    # retrieving all descriptors, but that adds roughly a minute to the runtime
+    # of this test and adds an appreciable load to directory authorities.
+
+    single_query = downloader.get_server_descriptors('9695DFC35FFEB861329B9F1AB04C46397020CE31')
+
+    multiple_query = downloader.get_server_descriptors([
+      '9695DFC35FFEB861329B9F1AB04C46397020CE31',
+      '847B1F850344D7876491A54892F904934E4EB85D',
+    ])
+
+    # Explicitly running the queries so they'll provide a useful error if
+    # unsuccessful.
+
+    single_query.run()
+    multiple_query.run()
+
+    single_query_results = list(single_query)
+    self.assertEqual(1, len(single_query_results))
+    self.assertEqual('moria1', single_query_results[0].nickname)
+
+    self.assertEqual(2, len(list(multiple_query)))
+