[tor-commits] [stem/master] Move helper functions to end of remote module

atagar at torproject.org atagar at torproject.org
Tue May 8 20:20:09 UTC 2018


commit df2df79855ff92d3452fa5841f18397895424861
Author: Damian Johnson <atagar at torproject.org>
Date:   Sat May 5 13:46:53 2018 -0700

    Move helper functions to end of remote module
    
    Generally I've kept with the ordering of: functions, classes, helpers. This way
    the publicly accessible stuff is first.
---
 stem/descriptor/remote.py | 304 +++++++++++++++++++++++-----------------------
 1 file changed, 152 insertions(+), 152 deletions(-)

diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py
index 1b6a7ea6..bc23063a 100644
--- a/stem/descriptor/remote.py
+++ b/stem/descriptor/remote.py
@@ -202,158 +202,6 @@ def get_consensus(authority_v3ident = None, microdescriptor = False, **query_arg
   return get_instance().get_consensus(authority_v3ident, microdescriptor, **query_args)
 
 
-def _download_from_orport(endpoint, compression, resource):
-  """
-  Downloads descriptors from the given orport. Payload is just like an http
-  response (headers and all)...
-
-  ::
-
-    HTTP/1.0 200 OK
-    Date: Mon, 23 Apr 2018 18:43:47 GMT
-    Content-Type: text/plain
-    X-Your-Address-Is: 216.161.254.25
-    Content-Encoding: identity
-    Expires: Wed, 25 Apr 2018 18:43:47 GMT
-
-    router dannenberg 193.23.244.244 443 0 80
-    identity-ed25519
-    ... rest of the descriptor content...
-
-  :param stem.ORPort endpoint: endpoint to download from
-  :param list compression: compression methods for the request
-  :param str resource: descriptor resource to download
-
-  :returns: two value tuple of the form (data, reply_headers)
-
-  :raises:
-    * :class:`stem.ProtocolError` if not a valid descriptor response
-    * :class:`stem.SocketError` if unable to establish a connection
-  """
-
-  link_protocols = endpoint.link_protocols if endpoint.link_protocols else [3]
-
-  with stem.client.Relay.connect(endpoint.address, endpoint.port, link_protocols) as relay:
-    with relay.create_circuit() as circ:
-      request = '\r\n'.join((
-        'GET %s HTTP/1.0' % resource,
-        'Accept-Encoding: %s' % ', '.join(compression),
-        'User-Agent: Stem/%s' % stem.__version__,
-      )) + '\r\n\r\n'
-
-      circ.send('RELAY_BEGIN_DIR', stream_id = 1)
-      response = b''.join([cell.data for cell in circ.send('RELAY_DATA', request, stream_id = 1)])
-      first_line, data = response.split(b'\r\n', 1)
-      header_data, data = data.split(b'\r\n\r\n', 1)
-
-      if first_line != b'HTTP/1.0 200 OK':
-        raise stem.ProtocolError("Response should begin with HTTP success, but was '%s'" % first_line)
-
-      headers = {}
-
-      for line in str_tools._to_unicode(header_data).splitlines():
-        if ': ' not in line:
-          raise stem.ProtocolError("'%s' is not a HTTP header:\n\n%s" % line)
-
-        key, value = line.split(': ', 1)
-        headers[key] = value
-
-      return _decompress(data, headers.get('Content-Encoding')), headers
-
-
-def _download_from_dirport(url, compression, timeout):
-  """
-  Downloads descriptors from the given url.
-
-  :param str url: dirport url from which to download from
-  :param list compression: compression methods for the request
-  :param float timeout: duration before we'll time out our request
-
-  :returns: two value tuple of the form (data, reply_headers)
-
-  :raises:
-    * **socket.timeout** if our request timed out
-    * **urllib2.URLError** for most request failures
-  """
-
-  response = urllib.urlopen(
-    urllib.Request(
-      url,
-      headers = {
-        'Accept-Encoding': ', '.join(compression),
-        'User-Agent': 'Stem/%s' % stem.__version__,
-      }
-    ),
-    timeout = timeout,
-  )
-
-  return _decompress(response.read(), response.headers.get('Content-Encoding')), response.headers
-
-
-def _decompress(data, encoding):
-  """
-  Decompresses descriptor data.
-
-  Tor doesn't include compression headers. As such when using gzip we
-  need to include '32' for automatic header detection...
-
-    https://stackoverflow.com/questions/3122145/zlib-error-error-3-while-decompressing-incorrect-header-check/22310760#22310760
-
-  ... and with zstd we need to use the streaming API.
-
-  :param bytes data: data we received
-  :param str encoding: 'Content-Encoding' header of the response
-
-  :raises:
-    * **ValueError** if encoding is unrecognized
-    * **ImportError** if missing the decompression module
-  """
-
-  if encoding == Compression.PLAINTEXT:
-    return data.strip()
-  elif encoding in (Compression.GZIP, 'deflate'):
-    return zlib.decompress(data, zlib.MAX_WBITS | 32).strip()
-  elif encoding == Compression.ZSTD:
-    if not stem.prereq.is_zstd_available():
-      raise ImportError('Decompressing zstd data requires https://pypi.python.org/pypi/zstandard')
-
-    import zstd
-    output_buffer = io.BytesIO()
-
-    with zstd.ZstdDecompressor().write_to(output_buffer) as decompressor:
-      decompressor.write(data)
-
-    return output_buffer.getvalue().strip()
-  elif encoding == Compression.LZMA:
-    if not stem.prereq.is_lzma_available():
-      raise ImportError('Decompressing lzma data requires https://docs.python.org/3/library/lzma.html')
-
-    import lzma
-    return lzma.decompress(data).strip()
-  else:
-    raise ValueError("'%s' isn't a recognized type of encoding" % encoding)
-
-
-def _guess_descriptor_type(resource):
-  # Attempts to determine the descriptor type based on the resource url. This
-  # raises a ValueError if the resource isn't recognized.
-
-  if resource.startswith('/tor/server/'):
-    return 'server-descriptor 1.0'
-  elif resource.startswith('/tor/extra/'):
-    return 'extra-info 1.0'
-  elif resource.startswith('/tor/micro/'):
-    return 'microdescriptor 1.0'
-  elif resource.startswith('/tor/status-vote/current/consensus-microdesc'):
-    return 'network-status-microdesc-consensus-3 1.0'
-  elif resource.startswith('/tor/status-vote/'):
-    return 'network-status-consensus-3 1.0'
-  elif resource.startswith('/tor/keys/'):
-    return 'dir-key-certificate-3 1.0'
-  else:
-    raise ValueError("Unable to determine the descriptor type for '%s'" % resource)
-
-
 class Query(object):
   """
   Asynchronous request for descriptor content from a directory authority or
@@ -962,6 +810,158 @@ class DescriptorDownloader(object):
     return Query(resource, **args)
 
 
+def _download_from_orport(endpoint, compression, resource):
+  """
+  Downloads descriptors from the given orport. Payload is just like an http
+  response (headers and all)...
+
+  ::
+
+    HTTP/1.0 200 OK
+    Date: Mon, 23 Apr 2018 18:43:47 GMT
+    Content-Type: text/plain
+    X-Your-Address-Is: 216.161.254.25
+    Content-Encoding: identity
+    Expires: Wed, 25 Apr 2018 18:43:47 GMT
+
+    router dannenberg 193.23.244.244 443 0 80
+    identity-ed25519
+    ... rest of the descriptor content...
+
+  :param stem.ORPort endpoint: endpoint to download from
+  :param list compression: compression methods for the request
+  :param str resource: descriptor resource to download
+
+  :returns: two value tuple of the form (data, reply_headers)
+
+  :raises:
+    * :class:`stem.ProtocolError` if not a valid descriptor response
+    * :class:`stem.SocketError` if unable to establish a connection
+  """
+
+  link_protocols = endpoint.link_protocols if endpoint.link_protocols else [3]
+
+  with stem.client.Relay.connect(endpoint.address, endpoint.port, link_protocols) as relay:
+    with relay.create_circuit() as circ:
+      request = '\r\n'.join((
+        'GET %s HTTP/1.0' % resource,
+        'Accept-Encoding: %s' % ', '.join(compression),
+        'User-Agent: Stem/%s' % stem.__version__,
+      )) + '\r\n\r\n'
+
+      circ.send('RELAY_BEGIN_DIR', stream_id = 1)
+      response = b''.join([cell.data for cell in circ.send('RELAY_DATA', request, stream_id = 1)])
+      first_line, data = response.split(b'\r\n', 1)
+      header_data, data = data.split(b'\r\n\r\n', 1)
+
+      if first_line != b'HTTP/1.0 200 OK':
+        raise stem.ProtocolError("Response should begin with HTTP success, but was '%s'" % first_line)
+
+      headers = {}
+
+      for line in str_tools._to_unicode(header_data).splitlines():
+        if ': ' not in line:
+          raise stem.ProtocolError("'%s' is not a HTTP header:\n\n%s" % line)
+
+        key, value = line.split(': ', 1)
+        headers[key] = value
+
+      return _decompress(data, headers.get('Content-Encoding')), headers
+
+
+def _download_from_dirport(url, compression, timeout):
+  """
+  Downloads descriptors from the given url.
+
+  :param str url: dirport url from which to download from
+  :param list compression: compression methods for the request
+  :param float timeout: duration before we'll time out our request
+
+  :returns: two value tuple of the form (data, reply_headers)
+
+  :raises:
+    * **socket.timeout** if our request timed out
+    * **urllib2.URLError** for most request failures
+  """
+
+  response = urllib.urlopen(
+    urllib.Request(
+      url,
+      headers = {
+        'Accept-Encoding': ', '.join(compression),
+        'User-Agent': 'Stem/%s' % stem.__version__,
+      }
+    ),
+    timeout = timeout,
+  )
+
+  return _decompress(response.read(), response.headers.get('Content-Encoding')), response.headers
+
+
+def _decompress(data, encoding):
+  """
+  Decompresses descriptor data.
+
+  Tor doesn't include compression headers. As such when using gzip we
+  need to include '32' for automatic header detection...
+
+    https://stackoverflow.com/questions/3122145/zlib-error-error-3-while-decompressing-incorrect-header-check/22310760#22310760
+
+  ... and with zstd we need to use the streaming API.
+
+  :param bytes data: data we received
+  :param str encoding: 'Content-Encoding' header of the response
+
+  :raises:
+    * **ValueError** if encoding is unrecognized
+    * **ImportError** if missing the decompression module
+  """
+
+  if encoding == Compression.PLAINTEXT:
+    return data.strip()
+  elif encoding in (Compression.GZIP, 'deflate'):
+    return zlib.decompress(data, zlib.MAX_WBITS | 32).strip()
+  elif encoding == Compression.ZSTD:
+    if not stem.prereq.is_zstd_available():
+      raise ImportError('Decompressing zstd data requires https://pypi.python.org/pypi/zstandard')
+
+    import zstd
+    output_buffer = io.BytesIO()
+
+    with zstd.ZstdDecompressor().write_to(output_buffer) as decompressor:
+      decompressor.write(data)
+
+    return output_buffer.getvalue().strip()
+  elif encoding == Compression.LZMA:
+    if not stem.prereq.is_lzma_available():
+      raise ImportError('Decompressing lzma data requires https://docs.python.org/3/library/lzma.html')
+
+    import lzma
+    return lzma.decompress(data).strip()
+  else:
+    raise ValueError("'%s' isn't a recognized type of encoding" % encoding)
+
+
+def _guess_descriptor_type(resource):
+  # Attempts to determine the descriptor type based on the resource url. This
+  # raises a ValueError if the resource isn't recognized.
+
+  if resource.startswith('/tor/server/'):
+    return 'server-descriptor 1.0'
+  elif resource.startswith('/tor/extra/'):
+    return 'extra-info 1.0'
+  elif resource.startswith('/tor/micro/'):
+    return 'microdescriptor 1.0'
+  elif resource.startswith('/tor/status-vote/current/consensus-microdesc'):
+    return 'network-status-microdesc-consensus-3 1.0'
+  elif resource.startswith('/tor/status-vote/'):
+    return 'network-status-consensus-3 1.0'
+  elif resource.startswith('/tor/keys/'):
+    return 'dir-key-certificate-3 1.0'
+  else:
+    raise ValueError("Unable to determine the descriptor type for '%s'" % resource)
+
+
 def get_authorities():
   """
   Provides cached Tor directory authority information. The directory





More information about the tor-commits mailing list