commit 4a319f1794e54fc216d84ac192cc3efc1ce15ec2 Author: Damian Johnson atagar@torproject.org Date: Sun Aug 11 05:14:49 2019 -0700

Unit test collector downloading

On reflection there's no need for us to rely exclusively on integ tests for download coverage. Those certainly do a great job, but simple to get unit test coverage of this as well. --- stem/descriptor/collector.py | 8 +- test/unit/descriptor/collector.py | 81 ++++++++++++++++++++- test/unit/descriptor/data/__init__.py | 2 +- test/unit/descriptor/data/collector/__init__.py | 7 ++ .../data/collector/consensuses-2018-06-cropped.tar | Bin 0 -> 100864 bytes .../data/collector/extra-infos-2019-04-cropped.tar | Bin 0 -> 22528 bytes .../{collector_index.json => collector/index.json} | 0 .../{collector_index.py => collector/index.py} | 0 .../data/collector/microdescs-2019-05-cropped.tar | Bin 0 -> 193024 bytes .../server-descriptors-2005-12-cropped.tar | Bin 0 -> 22528 bytes 10 files changed, 91 insertions(+), 7 deletions(-)

diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py index c0c97832..09e94e74 100644 --- a/stem/descriptor/collector.py +++ b/stem/descriptor/collector.py @@ -435,7 +435,7 @@ class CollecTor(object): # TODO: support bridge variants ('bridge-server-descriptor' type)

for f in self.files('server-descriptor', start, end): - for desc in f.read(cache_to, timeout = timeout, retries = retries): + for desc in f.read(cache_to, 'server-descriptor', timeout = timeout, retries = retries): yield desc

def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): @@ -461,7 +461,7 @@ class CollecTor(object): # TODO: support bridge variants ('bridge-extra-info' type)

for f in self.files('extra-info', start, end): - for desc in f.read(cache_to, timeout = timeout, retries = retries): + for desc in f.read(cache_to, 'extra-info', timeout = timeout, retries = retries): yield desc

def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3): @@ -494,7 +494,7 @@ class CollecTor(object): """

for f in self.files('microdescriptor', start, end): - for desc in f.read(cache_to, timeout = timeout, retries = retries): + for desc in f.read(cache_to, 'microdescriptor', timeout = timeout, retries = retries): yield desc

def get_consensus(self, start = None, end = None, cache_to = None, version = 3, microdescriptor = False, timeout = None, retries = 3): @@ -536,7 +536,7 @@ class CollecTor(object): # TODO: document vs router status entries (ie. DocumentType)?

for f in self.files(desc_type, start, end): - for desc in f.read(cache_to, timeout = timeout, retries = retries): + for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries): yield desc

def index(self, compression = 'best'): diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py index 77c1c460..f5bed87e 100644 --- a/test/unit/descriptor/collector.py +++ b/test/unit/descriptor/collector.py @@ -11,7 +11,7 @@ import stem.prereq from stem.descriptor import Compression from stem.descriptor.collector import CollecTor, File from test.unit.descriptor import get_resource -from test.unit.descriptor.data.collector_index import EXAMPLE_INDEX +from test.unit.descriptor.data.collector.index import EXAMPLE_INDEX

try: # added in python 3.3 @@ -22,7 +22,7 @@ except ImportError: URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen'

-with open(get_resource('collector_index.json'), 'rb') as index_file: +with open(get_resource('collector/index.json'), 'rb') as index_file: EXAMPLE_INDEX_JSON = index_file.read()

@@ -194,3 +194,80 @@ class TestCollector(unittest.TestCase): self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', ], [f.path for f in collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1))]) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_server_descriptors(self, files_mock, download_mock): + with open(get_resource('collector/server-descriptors-2005-12-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_server_descriptors()) + self.assertEqual(5, len(descriptors)) + + f = descriptors[0] + self.assertEqual('RelayDescriptor', type(f).__name__) + self.assertEqual('3E2F63E2356F52318B536A12B6445373808A5D6C', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_extrainfo_descriptors(self, files_mock, download_mock): + with open(get_resource('collector/extra-infos-2019-04-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/extra-infos/extra-infos-2019-04.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors()) + self.assertEqual(7, len(descriptors)) + + f = descriptors[0] + self.assertEqual('RelayExtraInfoDescriptor', type(f).__name__) + self.assertEqual('170EF19C0FA0491DFCEA6E1FB0941670B80506E1', f.fingerprint) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_microdescriptors(self, files_mock, download_mock): + with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/microdescs/microdescs-2019-05.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_microdescriptors()) + self.assertEqual(3, len(descriptors)) + + f = descriptors[0] + self.assertEqual('Microdescriptor', type(f).__name__) + self.assertEqual(['ed25519'], list(f.identifiers.keys())) + + @patch('stem.util.connection.download') + @patch('stem.descriptor.collector.CollecTor.files') + def test_reading_consensus(self, files_mock, download_mock): + with open(get_resource('collector/consensuses-2018-06-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/consensuses/consensuses-2018-06.tar', 12345, '2016-09-04 09:21')] + + descriptors = list(stem.descriptor.collector.get_consensus()) + self.assertEqual(243, len(descriptors)) + + f = descriptors[0] + self.assertEqual('RouterStatusEntryV3', type(f).__name__) + self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint) + + # this archive shouldn't have any v2 or microdescriptor consensus data + + self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(version = 2)))) + self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(microdescriptor = True)))) + + # but the microdescriptor archive *does* have microdescriptor consensuses + + with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive: + download_mock.return_value = archive.read() + + descriptors = list(stem.descriptor.collector.get_consensus(microdescriptor = True)) + self.assertEqual(556, len(descriptors)) + + f = descriptors[0] + self.assertEqual('RouterStatusEntryMicroV3', type(f).__name__) + self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint) diff --git a/test/unit/descriptor/data/__init__.py b/test/unit/descriptor/data/__init__.py index 376864fd..8bc640c6 100644 --- a/test/unit/descriptor/data/__init__.py +++ b/test/unit/descriptor/data/__init__.py @@ -3,5 +3,5 @@ Test data for test.unit.descriptor tests. """

__all__ = [ - 'collector_index', + 'collector', ] diff --git a/test/unit/descriptor/data/collector/__init__.py b/test/unit/descriptor/data/collector/__init__.py new file mode 100644 index 00000000..fe90c9fb --- /dev/null +++ b/test/unit/descriptor/data/collector/__init__.py @@ -0,0 +1,7 @@ +""" +Test data for test.unit.descriptor's collector tests. +""" + +__all__ = [ + 'index', +] diff --git a/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar b/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar new file mode 100644 index 00000000..0728e595 Binary files /dev/null and b/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar differ diff --git a/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar b/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar new file mode 100644 index 00000000..21227286 Binary files /dev/null and b/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar differ diff --git a/test/unit/descriptor/data/collector_index.json b/test/unit/descriptor/data/collector/index.json similarity index 100% rename from test/unit/descriptor/data/collector_index.json rename to test/unit/descriptor/data/collector/index.json diff --git a/test/unit/descriptor/data/collector_index.py b/test/unit/descriptor/data/collector/index.py similarity index 100% rename from test/unit/descriptor/data/collector_index.py rename to test/unit/descriptor/data/collector/index.py diff --git a/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar b/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar new file mode 100644 index 00000000..789774c2 Binary files /dev/null and b/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar differ diff --git a/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar b/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar new file mode 100644 index 00000000..4ffe1434 Binary files /dev/null and b/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar differ