commit cf2b82b06688b4355a848029592ccdbea7fb3ebd Author: Damian Johnson atagar@torproject.org Date: Sat May 5 14:30:57 2018 -0700

Separate fallback unit tests

Now that directories have their own module we can separate the fallback unit tests into their own module. We should add similar tests for authorities. --- test/settings.cfg | 1 + test/unit/__init__.py | 1 + test/unit/descriptor/remote.py | 176 ------------------------------------ test/unit/directory/__init__.py | 7 ++ test/unit/directory/fallback.py | 194 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 203 insertions(+), 176 deletions(-)

diff --git a/test/settings.cfg b/test/settings.cfg index 60224463..2206c7e7 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -220,6 +220,7 @@ test.unit_tests |test.unit.endpoint.TestEndpoint |test.unit.version.TestVersion |test.unit.manual.TestManual +|test.unit.directory.fallback.TestFallback |test.unit.tutorial.TestTutorial |test.unit.tutorial_examples.TestTutorialExamples |test.unit.response.add_onion.TestAddOnionResponse diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 34fcbb19..1b16bc99 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -10,6 +10,7 @@ __all__ = [ 'connection', 'control', 'descriptor', + 'directory', 'exit_policy', 'socket', 'util', diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py index fd831e1e..478a7143 100644 --- a/test/unit/descriptor/remote.py +++ b/test/unit/descriptor/remote.py @@ -5,13 +5,11 @@ Unit tests for stem.descriptor.remote. import io import re import socket -import tempfile import time import unittest

import stem.descriptor.remote import stem.prereq -import stem.util.conf import stem.util.str_tools

from stem.descriptor.remote import Compression @@ -23,12 +21,6 @@ except ImportError: from httplib import HTTPMessage # python2

try: - # added in python 2.7 - from collections import OrderedDict -except ImportError: - from stem.util.ordereddict import OrderedDict - -try: # added in python 3.3 from unittest.mock import patch, Mock, MagicMock except ImportError: @@ -79,46 +71,6 @@ iO3EUE0AEYah2W9gdz8t+i3Dtr0zgqLS841GC/TyDKCm+MKmN8d098qnwK0NGF9q -----END SIGNATURE----- """

-FALLBACK_DIR_CONTENT = b"""\ -/* type=fallback */ -/* version=2.0.0 */ -/* timestamp=20170526090242 */ -/* ===== */ -/* Whitelist & blacklist excluded 1326 of 1513 candidates. */ -/* Checked IPv4 DirPorts served a consensus within 15.0s. */ -/* -Final Count: 151 (Eligible 187, Target 392 (1963 * 0.20), Max 200) -Excluded: 36 (Same Operator 27, Failed/Skipped Download 9, Excess 0) -Bandwidth Range: 1.3 - 40.0 MByte/s -*/ -/* -Onionoo Source: details Date: 2017-05-16 07:00:00 Version: 4.0 -URL: https:onionoo.torproject.orgdetails?fields=fingerprint%2Cnickname%2Ccontact%2Clast_changed_address_or_port%2Cconsensus_weight%2Cadvertised_bandwidth%2Cor_addresses%2Cdir_address%2Crecommended_version%2Cflags%2Ceffective_family%2Cplatform&flag=V2Dir&type=relay&last_seen_days=-0&first_seen_days=30- -*/ -/* -Onionoo Source: uptime Date: 2017-05-16 07:00:00 Version: 4.0 -URL: https:onionoo.torproject.orguptime?first_seen_days=30-&flag=V2Dir&type=relay&last_seen_days=-0 -*/ -/* ===== */ -"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" -" ipv6=[2a01:4f8:162:51e2::2]:9001" -/* nickname=rueckgrat */ -/* extrainfo=1 */ -/* ===== */ -, -"193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524" -/* nickname= */ -/* extrainfo=0 */ -/* ===== */ -""" - -FALLBACK_ENTRY = b"""\ -"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" -" ipv6=[2a01:4f8:162:51e2::2]:9001" -/* nickname=rueckgrat */ -/* extrainfo=1 */ -""" - HEADER = '\r\n'.join([ 'Date: Fri, 13 Apr 2018 16:35:50 GMT', 'Content-Type: application/octet-stream', @@ -421,131 +373,3 @@ class TestDescriptorDownloader(unittest.TestCase): def test_using_authorities_in_hash(self): # ensure our DirectoryAuthority instances can be used in hashes {stem.descriptor.remote.get_authorities()['moria1']: 'hello'} - - def test_fallback_directories_from_cache(self): - # quick sanity test that we can load cached content - fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache() - self.assertTrue(len(fallback_directories) > 10) - self.assertEqual('5.39.92.199', fallback_directories['0BEA4A88D069753218EAAAD6D22EA87B9A1319D6'].address) - - @patch(URL_OPEN, _dirport_mock(FALLBACK_DIR_CONTENT)) - def test_fallback_directories_from_remote(self): - fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote() - header = OrderedDict((('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'))) - - expected = { - '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.descriptor.remote.FallbackDirectory( - address = '5.9.110.236', - or_port = 9001, - dir_port = 9030, - fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', - nickname = 'rueckgrat', - has_extrainfo = True, - orport_v6 = ('2a01:4f8:162:51e2::2', 9001), - header = header, - ), - '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.descriptor.remote.FallbackDirectory( - address = '193.171.202.146', - or_port = 9001, - dir_port = 9030, - fingerprint = '01A9258A46E97FF8B2CAC7910577862C14F2C524', - nickname = None, - has_extrainfo = False, - orport_v6 = None, - header = header, - ), - } - - self.assertEqual(expected, fallback_directories) - - def test_fallback_persistence(self): - header = OrderedDict((('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'))) - - expected = { - '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.descriptor.remote.FallbackDirectory( - address = '5.9.110.236', - or_port = 9001, - dir_port = 9030, - fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', - nickname = 'rueckgrat', - has_extrainfo = True, - orport_v6 = ('2a01:4f8:162:51e2::2', 9001), - header = header, - ), - '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.descriptor.remote.FallbackDirectory( - address = '193.171.202.146', - or_port = 9001, - dir_port = 9030, - fingerprint = '01A9258A46E97FF8B2CAC7910577862C14F2C524', - nickname = None, - has_extrainfo = False, - orport_v6 = None, - header = header, - ), - } - - excepted_config = { - 'tor_commit': ['abc'], - 'stem_commit': ['def'], - 'header.type': ['fallback'], - 'header.version': ['2.0.0'], - 'header.timestamp': ['20170526090242'], - '01A9258A46E97FF8B2CAC7910577862C14F2C524.address': ['193.171.202.146'], - '01A9258A46E97FF8B2CAC7910577862C14F2C524.or_port': ['9001'], - '01A9258A46E97FF8B2CAC7910577862C14F2C524.dir_port': ['9030'], - '01A9258A46E97FF8B2CAC7910577862C14F2C524.has_extrainfo': ['false'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.address': ['5.9.110.236'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.or_port': ['9001'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.dir_port': ['9030'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.nickname': ['rueckgrat'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.has_extrainfo': ['true'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.orport6_address': ['2a01:4f8:162:51e2::2'], - '0756B7CD4DFC8182BE23143FAC0642F515182CEB.orport6_port': ['9001'], - } - - with tempfile.NamedTemporaryFile(prefix = 'fallbacks.') as tmp: - stem.descriptor.remote.FallbackDirectory._write(expected, 'abc', 'def', header, tmp.name) - - conf = stem.util.conf.Config() - conf.load(tmp.name) - self.assertEqual(excepted_config, dict(conf)) - - self.assertEqual(expected, stem.descriptor.remote.FallbackDirectory.from_cache(tmp.name)) - - @patch(URL_OPEN, _dirport_mock(b'')) - def test_fallback_directories_from_remote_empty(self): - self.assertRaisesRegexp(IOError, 'did not have any content', stem.descriptor.remote.FallbackDirectory.from_remote) - - @patch(URL_OPEN, _dirport_mock(b'\n'.join(FALLBACK_DIR_CONTENT.splitlines()[1:]))) - def test_fallback_directories_from_remote_no_header(self): - self.assertRaisesRegexp(IOError, 'does not have a type field indicating it is fallback directory metadata', stem.descriptor.remote.FallbackDirectory.from_remote) - - @patch(URL_OPEN, _dirport_mock(FALLBACK_DIR_CONTENT.replace(b'version=2.0.0', b'version'))) - def test_fallback_directories_from_remote_malformed_header(self): - self.assertRaisesRegexp(IOError, 'Malformed fallback directory header line: /* version */', stem.descriptor.remote.FallbackDirectory.from_remote) - - def test_fallback_directories_from_str(self): - expected = stem.descriptor.remote.FallbackDirectory( - address = '5.9.110.236', - or_port = 9001, - dir_port = 9030, - fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', - nickname = 'rueckgrat', - has_extrainfo = True, - orport_v6 = ('2a01:4f8:162:51e2::2', 9001), - ) - - self.assertEqual(expected, stem.descriptor.remote.FallbackDirectory._from_str(FALLBACK_ENTRY)) - - def test_fallback_directories_from_str_malformed(self): - test_values = { - FALLBACK_ENTRY.replace(b'id=0756B7CD4DFC8182BE23143FAC0642F515182CEB', b''): 'Malformed fallback address line:', - FALLBACK_ENTRY.replace(b'5.9.110.236', b'5.9.110'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv4 address: 5.9.110', - FALLBACK_ENTRY.replace(b':9030', b':7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid dir_port: 7814713228', - FALLBACK_ENTRY.replace(b'orport=9001', b'orport=7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid or_port: 7814713228', - FALLBACK_ENTRY.replace(b'ipv6=[2a01', b'ipv6=[:::'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv6 address: ::::4f8:162:51e2::2', - FALLBACK_ENTRY.replace(b'nickname=rueckgrat', b'nickname=invalid~nickname'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid nickname: invalid~nickname', - } - - for entry, expected in test_values.items(): - self.assertRaisesRegexp(ValueError, expected, stem.descriptor.remote.FallbackDirectory._from_str, entry) diff --git a/test/unit/directory/__init__.py b/test/unit/directory/__init__.py new file mode 100644 index 00000000..78e42d7c --- /dev/null +++ b/test/unit/directory/__init__.py @@ -0,0 +1,7 @@ +""" +Unit tests for stem.directory. +""" + +__all__ = [ + 'fallback', +] diff --git a/test/unit/directory/fallback.py b/test/unit/directory/fallback.py new file mode 100644 index 00000000..c95d74b7 --- /dev/null +++ b/test/unit/directory/fallback.py @@ -0,0 +1,194 @@ +""" +Unit tests for stem.directory.Fallback. +""" + +import io +import tempfile +import unittest + +import stem.directory +import stem.util.conf + +try: + # added in python 2.7 + from collections import OrderedDict +except ImportError: + from stem.util.ordereddict import OrderedDict + +try: + # added in python 3.3 + from unittest.mock import patch, Mock +except ImportError: + from mock import patch, Mock + +URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen' + +FALLBACK_DIR_CONTENT = b"""\ +/* type=fallback */ +/* version=2.0.0 */ +/* timestamp=20170526090242 */ +/* ===== */ +/* Whitelist & blacklist excluded 1326 of 1513 candidates. */ +/* Checked IPv4 DirPorts served a consensus within 15.0s. */ +/* +Final Count: 151 (Eligible 187, Target 392 (1963 * 0.20), Max 200) +Excluded: 36 (Same Operator 27, Failed/Skipped Download 9, Excess 0) +Bandwidth Range: 1.3 - 40.0 MByte/s +*/ +/* +Onionoo Source: details Date: 2017-05-16 07:00:00 Version: 4.0 +URL: https:onionoo.torproject.orgdetails?fields=fingerprint%2Cnickname%2Ccontact%2Clast_changed_address_or_port%2Cconsensus_weight%2Cadvertised_bandwidth%2Cor_addresses%2Cdir_address%2Crecommended_version%2Cflags%2Ceffective_family%2Cplatform&flag=V2Dir&type=relay&last_seen_days=-0&first_seen_days=30- +*/ +/* +Onionoo Source: uptime Date: 2017-05-16 07:00:00 Version: 4.0 +URL: https:onionoo.torproject.orguptime?first_seen_days=30-&flag=V2Dir&type=relay&last_seen_days=-0 +*/ +/* ===== */ +"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" +" ipv6=[2a01:4f8:162:51e2::2]:9001" +/* nickname=rueckgrat */ +/* extrainfo=1 */ +/* ===== */ +, +"193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524" +/* nickname= */ +/* extrainfo=0 */ +/* ===== */ +""" + +FALLBACK_ENTRY = b"""\ +"5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" +" ipv6=[2a01:4f8:162:51e2::2]:9001" +/* nickname=rueckgrat */ +/* extrainfo=1 */ +""" + + +class TestFallback(unittest.TestCase): + def test_from_cache(self): + # quick sanity test that we can load cached content + fallback_directories = stem.directory.Fallback.from_cache() + self.assertTrue(len(fallback_directories) > 10) + self.assertEqual('5.39.92.199', fallback_directories['0BEA4A88D069753218EAAAD6D22EA87B9A1319D6'].address) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(FALLBACK_DIR_CONTENT))) + def test_from_remote(self): + fallback_directories = stem.directory.Fallback.from_remote() + header = OrderedDict((('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'))) + + expected = { + '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.directory.Fallback( + address = '5.9.110.236', + or_port = 9001, + dir_port = 9030, + fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', + nickname = 'rueckgrat', + has_extrainfo = True, + orport_v6 = ('2a01:4f8:162:51e2::2', 9001), + header = header, + ), + '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.directory.Fallback( + address = '193.171.202.146', + or_port = 9001, + dir_port = 9030, + fingerprint = '01A9258A46E97FF8B2CAC7910577862C14F2C524', + nickname = None, + has_extrainfo = False, + orport_v6 = None, + header = header, + ), + } + + self.assertEqual(expected, fallback_directories) + + def test_persistence(self): + header = OrderedDict((('type', 'fallback'), ('version', '2.0.0'), ('timestamp', '20170526090242'))) + + expected = { + '0756B7CD4DFC8182BE23143FAC0642F515182CEB': stem.directory.Fallback( + address = '5.9.110.236', + or_port = 9001, + dir_port = 9030, + fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', + nickname = 'rueckgrat', + has_extrainfo = True, + orport_v6 = ('2a01:4f8:162:51e2::2', 9001), + header = header, + ), + '01A9258A46E97FF8B2CAC7910577862C14F2C524': stem.directory.Fallback( + address = '193.171.202.146', + or_port = 9001, + dir_port = 9030, + fingerprint = '01A9258A46E97FF8B2CAC7910577862C14F2C524', + nickname = None, + has_extrainfo = False, + orport_v6 = None, + header = header, + ), + } + + excepted_config = { + 'tor_commit': ['abc'], + 'stem_commit': ['def'], + 'header.type': ['fallback'], + 'header.version': ['2.0.0'], + 'header.timestamp': ['20170526090242'], + '01A9258A46E97FF8B2CAC7910577862C14F2C524.address': ['193.171.202.146'], + '01A9258A46E97FF8B2CAC7910577862C14F2C524.or_port': ['9001'], + '01A9258A46E97FF8B2CAC7910577862C14F2C524.dir_port': ['9030'], + '01A9258A46E97FF8B2CAC7910577862C14F2C524.has_extrainfo': ['false'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.address': ['5.9.110.236'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.or_port': ['9001'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.dir_port': ['9030'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.nickname': ['rueckgrat'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.has_extrainfo': ['true'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.orport6_address': ['2a01:4f8:162:51e2::2'], + '0756B7CD4DFC8182BE23143FAC0642F515182CEB.orport6_port': ['9001'], + } + + with tempfile.NamedTemporaryFile(prefix = 'fallbacks.') as tmp: + stem.directory.Fallback._write(expected, 'abc', 'def', header, tmp.name) + + conf = stem.util.conf.Config() + conf.load(tmp.name) + self.assertEqual(excepted_config, dict(conf)) + + self.assertEqual(expected, stem.directory.Fallback.from_cache(tmp.name)) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(b''))) + def test_from_remote_empty(self): + self.assertRaisesRegexp(IOError, 'did not have any content', stem.directory.Fallback.from_remote) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'\n'.join(FALLBACK_DIR_CONTENT.splitlines()[1:])))) + def test_from_remote_no_header(self): + self.assertRaisesRegexp(IOError, 'does not have a type field indicating it is fallback directory metadata', stem.directory.Fallback.from_remote) + + @patch(URL_OPEN, Mock(return_value = io.BytesIO(FALLBACK_DIR_CONTENT.replace(b'version=2.0.0', b'version')))) + def test_from_remote_malformed_header(self): + self.assertRaisesRegexp(IOError, 'Malformed fallback directory header line: /* version */', stem.directory.Fallback.from_remote) + + def test_from_str(self): + expected = stem.directory.Fallback( + address = '5.9.110.236', + or_port = 9001, + dir_port = 9030, + fingerprint = '0756B7CD4DFC8182BE23143FAC0642F515182CEB', + nickname = 'rueckgrat', + has_extrainfo = True, + orport_v6 = ('2a01:4f8:162:51e2::2', 9001), + ) + + self.assertEqual(expected, stem.directory.Fallback._from_str(FALLBACK_ENTRY)) + + def test_from_str_malformed(self): + test_values = { + FALLBACK_ENTRY.replace(b'id=0756B7CD4DFC8182BE23143FAC0642F515182CEB', b''): 'Malformed fallback address line:', + FALLBACK_ENTRY.replace(b'5.9.110.236', b'5.9.110'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv4 address: 5.9.110', + FALLBACK_ENTRY.replace(b':9030', b':7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid dir_port: 7814713228', + FALLBACK_ENTRY.replace(b'orport=9001', b'orport=7814713228'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid or_port: 7814713228', + FALLBACK_ENTRY.replace(b'ipv6=[2a01', b'ipv6=[:::'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid IPv6 address: ::::4f8:162:51e2::2', + FALLBACK_ENTRY.replace(b'nickname=rueckgrat', b'nickname=invalid~nickname'): '0756B7CD4DFC8182BE23143FAC0642F515182CEB has an invalid nickname: invalid~nickname', + } + + for entry, expected in test_values.items(): + self.assertRaisesRegexp(ValueError, expected, stem.directory.Fallback._from_str, entry)