[tor-commits] [stem/master] Unit test collector downloading

atagar at torproject.org atagar at torproject.org
Sat Aug 17 20:44:27 UTC 2019


commit 4a319f1794e54fc216d84ac192cc3efc1ce15ec2
Author: Damian Johnson <atagar at torproject.org>
Date:   Sun Aug 11 05:14:49 2019 -0700

    Unit test collector downloading
    
    On reflection there's no need for us to rely exclusively on integ tests for
    download coverage. Those certainly do a great job, but simple to get unit test
    coverage of this as well.
---
 stem/descriptor/collector.py                       |   8 +-
 test/unit/descriptor/collector.py                  |  81 ++++++++++++++++++++-
 test/unit/descriptor/data/__init__.py              |   2 +-
 test/unit/descriptor/data/collector/__init__.py    |   7 ++
 .../data/collector/consensuses-2018-06-cropped.tar | Bin 0 -> 100864 bytes
 .../data/collector/extra-infos-2019-04-cropped.tar | Bin 0 -> 22528 bytes
 .../{collector_index.json => collector/index.json} |   0
 .../{collector_index.py => collector/index.py}     |   0
 .../data/collector/microdescs-2019-05-cropped.tar  | Bin 0 -> 193024 bytes
 .../server-descriptors-2005-12-cropped.tar         | Bin 0 -> 22528 bytes
 10 files changed, 91 insertions(+), 7 deletions(-)

diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index c0c97832..09e94e74 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -435,7 +435,7 @@ class CollecTor(object):
     # TODO: support bridge variants ('bridge-server-descriptor' type)
 
     for f in self.files('server-descriptor', start, end):
-      for desc in f.read(cache_to, timeout = timeout, retries = retries):
+      for desc in f.read(cache_to, 'server-descriptor', timeout = timeout, retries = retries):
         yield desc
 
   def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
@@ -461,7 +461,7 @@ class CollecTor(object):
     # TODO: support bridge variants ('bridge-extra-info' type)
 
     for f in self.files('extra-info', start, end):
-      for desc in f.read(cache_to, timeout = timeout, retries = retries):
+      for desc in f.read(cache_to, 'extra-info', timeout = timeout, retries = retries):
         yield desc
 
   def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
@@ -494,7 +494,7 @@ class CollecTor(object):
     """
 
     for f in self.files('microdescriptor', start, end):
-      for desc in f.read(cache_to, timeout = timeout, retries = retries):
+      for desc in f.read(cache_to, 'microdescriptor', timeout = timeout, retries = retries):
         yield desc
 
   def get_consensus(self, start = None, end = None, cache_to = None, version = 3, microdescriptor = False, timeout = None, retries = 3):
@@ -536,7 +536,7 @@ class CollecTor(object):
     # TODO: document vs router status entries (ie. DocumentType)?
 
     for f in self.files(desc_type, start, end):
-      for desc in f.read(cache_to, timeout = timeout, retries = retries):
+      for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries):
         yield desc
 
   def index(self, compression = 'best'):
diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py
index 77c1c460..f5bed87e 100644
--- a/test/unit/descriptor/collector.py
+++ b/test/unit/descriptor/collector.py
@@ -11,7 +11,7 @@ import stem.prereq
 from stem.descriptor import Compression
 from stem.descriptor.collector import CollecTor, File
 from test.unit.descriptor import get_resource
-from test.unit.descriptor.data.collector_index import EXAMPLE_INDEX
+from test.unit.descriptor.data.collector.index import EXAMPLE_INDEX
 
 try:
   # added in python 3.3
@@ -22,7 +22,7 @@ except ImportError:
 URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen'
 
 
-with open(get_resource('collector_index.json'), 'rb') as index_file:
+with open(get_resource('collector/index.json'), 'rb') as index_file:
   EXAMPLE_INDEX_JSON = index_file.read()
 
 
@@ -194,3 +194,80 @@ class TestCollector(unittest.TestCase):
     self.assertEqual([
       'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz',
     ], [f.path for f in collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1))])
+
+  @patch('stem.util.connection.download')
+  @patch('stem.descriptor.collector.CollecTor.files')
+  def test_reading_server_descriptors(self, files_mock, download_mock):
+    with open(get_resource('collector/server-descriptors-2005-12-cropped.tar'), 'rb') as archive:
+      download_mock.return_value = archive.read()
+
+    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar', 12345, '2016-09-04 09:21')]
+
+    descriptors = list(stem.descriptor.collector.get_server_descriptors())
+    self.assertEqual(5, len(descriptors))
+
+    f = descriptors[0]
+    self.assertEqual('RelayDescriptor', type(f).__name__)
+    self.assertEqual('3E2F63E2356F52318B536A12B6445373808A5D6C', f.fingerprint)
+
+  @patch('stem.util.connection.download')
+  @patch('stem.descriptor.collector.CollecTor.files')
+  def test_reading_extrainfo_descriptors(self, files_mock, download_mock):
+    with open(get_resource('collector/extra-infos-2019-04-cropped.tar'), 'rb') as archive:
+      download_mock.return_value = archive.read()
+
+    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/extra-infos/extra-infos-2019-04.tar', 12345, '2016-09-04 09:21')]
+
+    descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors())
+    self.assertEqual(7, len(descriptors))
+
+    f = descriptors[0]
+    self.assertEqual('RelayExtraInfoDescriptor', type(f).__name__)
+    self.assertEqual('170EF19C0FA0491DFCEA6E1FB0941670B80506E1', f.fingerprint)
+
+  @patch('stem.util.connection.download')
+  @patch('stem.descriptor.collector.CollecTor.files')
+  def test_reading_microdescriptors(self, files_mock, download_mock):
+    with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive:
+      download_mock.return_value = archive.read()
+
+    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/microdescs/microdescs-2019-05.tar', 12345, '2016-09-04 09:21')]
+
+    descriptors = list(stem.descriptor.collector.get_microdescriptors())
+    self.assertEqual(3, len(descriptors))
+
+    f = descriptors[0]
+    self.assertEqual('Microdescriptor', type(f).__name__)
+    self.assertEqual(['ed25519'], list(f.identifiers.keys()))
+
+  @patch('stem.util.connection.download')
+  @patch('stem.descriptor.collector.CollecTor.files')
+  def test_reading_consensus(self, files_mock, download_mock):
+    with open(get_resource('collector/consensuses-2018-06-cropped.tar'), 'rb') as archive:
+      download_mock.return_value = archive.read()
+
+    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/consensuses/consensuses-2018-06.tar', 12345, '2016-09-04 09:21')]
+
+    descriptors = list(stem.descriptor.collector.get_consensus())
+    self.assertEqual(243, len(descriptors))
+
+    f = descriptors[0]
+    self.assertEqual('RouterStatusEntryV3', type(f).__name__)
+    self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint)
+
+    # this archive shouldn't have any v2 or microdescriptor consensus data
+
+    self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(version = 2))))
+    self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(microdescriptor = True))))
+
+    # but the microdescriptor archive *does* have microdescriptor consensuses
+
+    with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive:
+      download_mock.return_value = archive.read()
+
+    descriptors = list(stem.descriptor.collector.get_consensus(microdescriptor = True))
+    self.assertEqual(556, len(descriptors))
+
+    f = descriptors[0]
+    self.assertEqual('RouterStatusEntryMicroV3', type(f).__name__)
+    self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint)
diff --git a/test/unit/descriptor/data/__init__.py b/test/unit/descriptor/data/__init__.py
index 376864fd..8bc640c6 100644
--- a/test/unit/descriptor/data/__init__.py
+++ b/test/unit/descriptor/data/__init__.py
@@ -3,5 +3,5 @@ Test data for test.unit.descriptor tests.
 """
 
 __all__ = [
-  'collector_index',
+  'collector',
 ]
diff --git a/test/unit/descriptor/data/collector/__init__.py b/test/unit/descriptor/data/collector/__init__.py
new file mode 100644
index 00000000..fe90c9fb
--- /dev/null
+++ b/test/unit/descriptor/data/collector/__init__.py
@@ -0,0 +1,7 @@
+"""
+Test data for test.unit.descriptor's collector tests.
+"""
+
+__all__ = [
+  'index',
+]
diff --git a/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar b/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar
new file mode 100644
index 00000000..0728e595
Binary files /dev/null and b/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar differ
diff --git a/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar b/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar
new file mode 100644
index 00000000..21227286
Binary files /dev/null and b/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar differ
diff --git a/test/unit/descriptor/data/collector_index.json b/test/unit/descriptor/data/collector/index.json
similarity index 100%
rename from test/unit/descriptor/data/collector_index.json
rename to test/unit/descriptor/data/collector/index.json
diff --git a/test/unit/descriptor/data/collector_index.py b/test/unit/descriptor/data/collector/index.py
similarity index 100%
rename from test/unit/descriptor/data/collector_index.py
rename to test/unit/descriptor/data/collector/index.py
diff --git a/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar b/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar
new file mode 100644
index 00000000..789774c2
Binary files /dev/null and b/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar differ
diff --git a/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar b/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar
new file mode 100644
index 00000000..4ffe1434
Binary files /dev/null and b/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar differ





More information about the tor-commits mailing list