tor-commits October 2020

tor-commits@lists.torproject.org

17 participants
2274 discussions

[stem/master] Test benchmark_server_descriptor_stem example
by atagar＠torproject.org 02 Oct '20

02 Oct '20

commit fc902a4917b3fb96e6c11b71a8f24737bbae221f Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Sep 21 16:28:30 2020 -0700 Test benchmark_server_descriptor_stem example --- test/unit/examples.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/test/unit/examples.py b/test/unit/examples.py index 12e3d68f..6a00d9b7 100644 --- a/test/unit/examples.py +++ b/test/unit/examples.py @@ -33,20 +33,25 @@ Relay BD4172533C3F7271ABCCD9F057E06FD91547C42B """ +EXPECTED_BENCHMARK_SERVER_DESC_PREFIX = """\ +Finished measure_average_advertised_bandwidth('%s') + Total time: 0 seconds + Processed server descriptors: 5 + Average advertised bandwidth: 313183 + Time per server descriptor: +""".rstrip() -def run_example(module): + +def import_example(module_name): """ - Invoke the given example, returning its stdout. + Import this example module. """ original_path = list(sys.path) sys.path.append(EXAMPLE_DIR) try: - with patch('sys.stdout', new_callable = io.StringIO) as stdout_mock: - importlib.import_module(module) - - return stdout_mock.getvalue() + return importlib.import_module(module_name) finally: sys.path = original_path @@ -70,7 +75,8 @@ class TestExamples(unittest.TestCase): self.fail("Changed our examples directory? The following are untested: %s" % ', '.join(missing)) @patch('stem.descriptor.remote.get_bandwidth_file') - def test_bandwidth_stats(self, get_bandwidth_file_mock): + @patch('sys.stdout', new_callable = io.StringIO) + def test_bandwidth_stats(self, stdout_mock, get_bandwidth_file_mock): get_bandwidth_file_mock().run.return_value = [BandwidthFile.create({ 'content': [ 'bw=1 bw_mean=807445 bw_median=911047 consensus_bandwidth=1190000 node_id=$FDCF49562E65B1CC219410009BD48A9EED387C77', @@ -78,10 +84,17 @@ class TestExamples(unittest.TestCase): ], })] - self.assertEqual(EXPECTED_BANDWIDTH_STATS, run_example('bandwidth_stats')) + import_example('bandwidth_stats') + self.assertEqual(EXPECTED_BANDWIDTH_STATS, stdout_mock.getvalue()) - def test_benchmark_server_descriptor_stem(self): - pass + @patch('sys.stdout', new_callable = io.StringIO) + def test_benchmark_server_descriptor_stem(self, stdout_mock): + path = os.path.join(test.STEM_BASE, 'test', 'unit', 'descriptor', 'data', 'collector', 'server-descriptors-2005-12-cropped.tar') + + module = import_example('benchmark_server_descriptor_stem') + module.measure_average_advertised_bandwidth(path) + + self.assertTrue(stdout_mock.getvalue().startswith(EXPECTED_BENCHMARK_SERVER_DESC_PREFIX % path)) def test_benchmark_stem(self): pass

1 0

[stem/master] Drop extra benchmark download links
by atagar＠torproject.org 02 Oct '20

02 Oct '20

commit da24bb2d89cbb860ed946a658281bf9a110d4960 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Sep 21 16:50:10 2020 -0700 Drop extra benchmark download links I recently added download links for our examples, but the benchmark scripts are an exception. First, some of these links were broken. Second, we already provide a link above which we display a *subset* of (since the scripts are pretty long). --- docs/tutorials/mirror_mirror_on_the_wall.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/tutorials/mirror_mirror_on_the_wall.rst b/docs/tutorials/mirror_mirror_on_the_wall.rst index 80ae1104..6b383141 100644 --- a/docs/tutorials/mirror_mirror_on_the_wall.rst +++ b/docs/tutorials/mirror_mirror_on_the_wall.rst @@ -407,7 +407,6 @@ Stem Example * `Stem Benchmark Script <../.../../_static/example/benchmark_stem.py>`_ .. literalinclude:: /_static/example/benchmark_server_descriptor_stem.py - :caption: `[Download] <../_static/example/benchmark_server_descriptor_stem.py>`__ :language: python Metrics-lib Example @@ -416,7 +415,6 @@ Metrics-lib Example * `Metrics-lib Benchmark Script <../.../../_static/example/benchmark_metrics_lib.java>`_ .. literalinclude:: /_static/example/benchmark_server_descriptor_metrics_lib.java - :caption: `[Download] <../_static/example/benchmark_server_descriptor_metrics_lib.py>`__ :language: java Zoossh Example @@ -425,5 +423,4 @@ Zoossh Example * `Zoossh Benchmark Script <../.../../_static/example/benchmark_zoossh.go>`_ .. literalinclude:: /_static/example/benchmark_server_descriptor_zoossh.go - :caption: `[Download] <../_static/example/benchmark_server_descriptor_zoossh.py>`__ :language: go

1 0

[stem/master] Stub example directory tests
by atagar＠torproject.org 02 Oct '20

02 Oct '20

commit 4451830442de64596c89e9d289718ebc03313e2c Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Sep 20 15:02:11 2020 -0700 Stub example directory tests Code within our example directory are untested. Stubbing a test module to remedy that. --- docs/_static/example/relay_connections.py | 0 test/settings.cfg | 1 + test/unit/doctest.py | 2 +- test/unit/examples.py | 173 ++++++++++++++++++++++++++++++ 4 files changed, 175 insertions(+), 1 deletion(-) diff --git a/docs/_static/example/relay_connections.py b/docs/_static/example/relay_connections.py old mode 100755 new mode 100644 diff --git a/test/settings.cfg b/test/settings.cfg index b1d26d92..a7bf1e1c 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -314,6 +314,7 @@ test.unit_tests |test.unit.interpreter.autocomplete.TestAutocompletion |test.unit.interpreter.help.TestHelpResponses |test.unit.interpreter.commands.TestInterpreterCommands +|test.unit.examples.TestExamples |test.unit.doctest.TestDocumentation test.integ_tests diff --git a/test/unit/doctest.py b/test/unit/doctest.py index 791e98b4..f72d845f 100644 --- a/test/unit/doctest.py +++ b/test/unit/doctest.py @@ -1,5 +1,5 @@ """ -Tests examples from our documentation. +Test inline examples from our documentation. """ import doctest diff --git a/test/unit/examples.py b/test/unit/examples.py new file mode 100644 index 00000000..ee59a137 --- /dev/null +++ b/test/unit/examples.py @@ -0,0 +1,173 @@ +""" +Exercise the code in our examples directory. +""" + +import os +import sys +import unittest + +import stem.util.system +import test + + +class TestExamples(unittest.TestCase): + def setUp(self): + self.original_path = list(sys.path) + self.example_dir = os.path.join(test.STEM_BASE, 'docs', '_static', 'example') + + sys.path.append(self.example_dir) + + def tearDown(self): + sys.path = self.original_path + + def test_runs_everything(self): + """ + Ensure we have tests for all our examples. + """ + + all_examples = set([os.path.basename(path)[:-3] for path in stem.util.system.files_with_suffix(self.example_dir, '.py')]) + tested_examples = set([method[5:] for method in dir(self) if method.startswith('test_') and method != 'test_runs_everything']) + + extra = sorted(tested_examples.difference(all_examples)) + missing = sorted(all_examples.difference(tested_examples)) + + if extra: + self.fail("Changed our examples directory? We test the following which are not present: %s" % ', '.join(extra)) + + if missing: + self.fail("Changed our examples directory? The following are untested: %s" % ', '.join(missing)) + + def test_bandwidth_stats(self): + pass + + def test_benchmark_server_descriptor_stem(self): + pass + + def test_benchmark_stem(self): + pass + + def test_broken_listener(self): + pass + + def test_check_digests(self): + pass + + def test_client_usage_using_pycurl(self): + pass + + def test_client_usage_using_socksipy(self): + pass + + def test_collector_caching(self): + pass + + def test_collector_reading(self): + pass + + def test_compare_flags(self): + pass + + def test_create_descriptor(self): + pass + + def test_create_descriptor_content(self): + pass + + def test_current_descriptors(self): + pass + + def test_custom_path_selection(self): + pass + + def test_descriptor_from_orport(self): + pass + + def test_descriptor_from_tor_control_socket(self): + pass + + def test_descriptor_from_tor_data_directory(self): + pass + + def test_download_descriptor(self): + pass + + def test_ephemeral_hidden_services(self): + pass + + def test_event_listening(self): + pass + + def test_exit_used(self): + pass + + def test_fibonacci_multiprocessing(self): + pass + + def test_fibonacci_threaded(self): + pass + + def test_get_hidden_service_descriptor(self): + pass + + def test_hello_world(self): + pass + + def test_introduction_points(self): + pass + + def test_list_circuits(self): + pass + + def test_load_test(self): + pass + + def test_manual_config_options(self): + pass + + def test_outdated_relays(self): + pass + + def test_persisting_a_consensus(self): + pass + + def test_persisting_a_consensus_with_parse_file(self): + pass + + def test_queue_listener(self): + pass + + def test_read_with_parse_file(self): + pass + + def test_reading_twitter(self): + pass + + def test_relay_connections(self): + pass + + def test_resuming_ephemeral_hidden_service(self): + pass + + def test_running_hidden_service(self): + pass + + def test_saving_and_loading_descriptors(self): + pass + + def test_slow_listener(self): + pass + + def test_tor_descriptors(self): + pass + + def test_utilities(self): + pass + + def test_validate_descriptor_content(self): + pass + + def test_votes_by_bandwidth_authorities(self): + pass + + def test_words_with(self): + pass

1 0

[stem/master] Fix server descriptor fingerprint crypto check
by atagar＠torproject.org 02 Oct '20

02 Oct '20

commit b487951b3a21b758b94baf23eaac54182cf6bdec Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 26 15:13:24 2020 -0700 Fix server descriptor fingerprint crypto check Fixing a server descriptor bug where checking our fingerprint didn't honor our skip_crypto_validation argument. This bug made it difficult to create descriptors with a preset fingerprint... >>> RelayDescriptor.create({'fingerprint': '4F0C 867D F0EF 6816 0568 C826 838F 482C EA7C FE44'}) ... ValueError: Fingerprint does not match the hash of our signing key (fingerprint: 4f0c867df0ef68160568c826838f482cea7cfe44, signing key hash: 8f54270a36526ab35895d5f899b6ae4059faecb3) --- stem/descriptor/server_descriptor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index e49688e1..baa14969 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -762,13 +762,13 @@ class RelayDescriptor(ServerDescriptor): super(RelayDescriptor, self).__init__(raw_contents, validate) if validate: - if self.fingerprint: - key_hash = hashlib.sha1(_bytes_for_block(self.signing_key)).hexdigest() + if not skip_crypto_validation: + if self.fingerprint: + key_hash = hashlib.sha1(_bytes_for_block(self.signing_key)).hexdigest() - if key_hash != self.fingerprint.lower(): - raise ValueError('Fingerprint does not match the hash of our signing key (fingerprint: %s, signing key hash: %s)' % (self.fingerprint.lower(), key_hash)) + if key_hash != self.fingerprint.lower(): + raise ValueError('Fingerprint does not match the hash of our signing key (fingerprint: %s, signing key hash: %s)' % (self.fingerprint.lower(), key_hash)) - if not skip_crypto_validation: try: signed_digest = self._digest_for_signature(self.signing_key, self.signature)

1 0

[stem/master] Skip client usage tests
by atagar＠torproject.org 02 Oct '20

02 Oct '20

commit 632c13d84decb132cdf57c9b27fe54bc668ec86c Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Sep 23 16:36:28 2020 -0700 Skip client usage tests There's not much code within these that we can test (it would effectively be wholly mocks). --- test/settings.cfg | 1 + test/unit/examples.py | 46 ++++++++++++++++++++-------------------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/test/settings.cfg b/test/settings.cfg index a7bf1e1c..26561327 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -213,6 +213,7 @@ pyflakes.ignore test/require.py => 'cryptography.hazmat.primitives.ciphers.Ciphe pyflakes.ignore test/require.py => 'cryptography.hazmat.primitives.ciphers.modes' imported but unused pyflakes.ignore test/require.py => 'cryptography.hazmat.primitives.serialization.load_der_public_key' imported but unused pyflakes.ignore test/require.py => 'cryptography.hazmat.primitives.asymmetric.ed25519.Ed25519PublicKey' imported but unused +pyflakes.ignore test/unit/examples.py => * imported but unused pyflakes.ignore test/unit/response/events.py => 'from stem import *' used; unable to detect undefined names pyflakes.ignore test/unit/response/events.py => *may be undefined, or defined from star imports: stem pyflakes.ignore test/integ/interpreter.py => 'readline' imported but unused diff --git a/test/unit/examples.py b/test/unit/examples.py index ba75a98c..566611c5 100644 --- a/test/unit/examples.py +++ b/test/unit/examples.py @@ -4,7 +4,6 @@ Exercise the code in our examples directory. import base64 import binascii -import importlib import io import os import sys @@ -26,6 +25,13 @@ from unittest.mock import Mock, patch EXAMPLE_DIR = os.path.join(test.STEM_BASE, 'docs', '_static', 'example') DESC_DIR = os.path.join(test.STEM_BASE, 'test', 'unit', 'descriptor', 'data') +UNTESTED = ( + # client usage demos don't have much non-stem code + + 'client_usage_using_pycurl', + 'client_usage_using_socksipy', +) + EXPECTED_BANDWIDTH_STATS = """\ Relay FDCF49562E65B1CC219410009BD48A9EED387C77 bw = 1 @@ -86,21 +92,14 @@ Extrainfo descriptor digest is correct """ -def import_example(module_name): - """ - Import this example module. - """ - - original_path = list(sys.path) - sys.path.append(EXAMPLE_DIR) - - try: - return importlib.import_module(module_name) - finally: - sys.path = original_path +class TestExamples(unittest.TestCase): + def setUp(self): + self.original_path = list(sys.path) + sys.path.append(EXAMPLE_DIR) + def tearDown(self): + sys.path = self.original_path -class TestExamples(unittest.TestCase): def test_runs_everything(self): """ Ensure we have tests for all our examples. @@ -110,7 +109,7 @@ class TestExamples(unittest.TestCase): tested_examples = set([method[5:] for method in dir(self) if method.startswith('test_') and method != 'test_runs_everything']) extra = sorted(tested_examples.difference(all_examples)) - missing = sorted(all_examples.difference(tested_examples)) + missing = sorted(all_examples.difference(tested_examples).difference(UNTESTED)) if extra: self.fail("Changed our examples directory? We test the following which are not present: %s" % ', '.join(extra)) @@ -128,21 +127,22 @@ class TestExamples(unittest.TestCase): ], })] - import_example('bandwidth_stats') + import bandwidth_stats self.assertEqual(EXPECTED_BANDWIDTH_STATS, stdout_mock.getvalue()) @patch('sys.stdout', new_callable = io.StringIO) def test_benchmark_server_descriptor_stem(self, stdout_mock): + import benchmark_server_descriptor_stem as module + path = os.path.join(DESC_DIR, 'collector', 'server-descriptors-2005-12-cropped.tar') expected_prefix = EXPECTED_SERVER_DESC_BENCHMARK_PREFIX % path - module = import_example('benchmark_server_descriptor_stem') module.measure_average_advertised_bandwidth(path) self.assertTrue(stdout_mock.getvalue().startswith(expected_prefix)) def test_benchmark_stem(self): - module = import_example('benchmark_stem') + import benchmark_stem as module with patch('sys.stdout', new_callable = io.StringIO) as stdout_mock: path = os.path.join(DESC_DIR, 'collector', 'server-descriptors-2005-12-cropped.tar') @@ -186,7 +186,7 @@ class TestExamples(unittest.TestCase): bw_event = ControlMessage.from_str('650 BW 15 25', 'EVENT', normalize = True) sleep_mock.side_effect = lambda duration: controller._handle_event(bw_event) - import_example('broken_listener') + import broken_listener self.assertEqual('start of broken_handler\n', stdout_mock.getvalue()) @@ -197,7 +197,7 @@ class TestExamples(unittest.TestCase): query.run.return_value = [desc] return Mock(return_value = query) - module = import_example('check_digests') + import check_digests as module fingerprint = 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB' extrainfo_desc = RelayExtraInfoDescriptor.create() @@ -229,12 +229,6 @@ class TestExamples(unittest.TestCase): module.validate_relay(fingerprint) self.assertEqual(EXPECTED_CHECK_DIGESTS_BAD % server_desc.digest(), stdout_mock.getvalue()) - def test_client_usage_using_pycurl(self): - pass - - def test_client_usage_using_socksipy(self): - pass - def test_collector_caching(self): pass

1 0

[stem/master] Test collector_caching example
by atagar＠torproject.org 02 Oct '20

02 Oct '20

commit 085018b9bace37a9758d5a0fbe447086612a8e8a Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Sep 25 17:21:56 2020 -0700 Test collector_caching example Ooph. Despite looking simple it took me a few hours to get these mocks right... --- test/unit/examples.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/test/unit/examples.py b/test/unit/examples.py index 566611c5..521a7937 100644 --- a/test/unit/examples.py +++ b/test/unit/examples.py @@ -91,6 +91,12 @@ Server descriptor digest invalid, expected A106452D87BD7B803B6CE916291ED368DC5BD Extrainfo descriptor digest is correct """ +EXPECTED_COLLECTOR_CACHING = """\ + krypton (3E2F63E2356F52318B536A12B6445373808A5D6C) + dizum (7EA6EAD6FD83083C538F44038BBFA077587DD755) + flubber (5C2124E6C5DD75C3C17C03EEA5A51812773DE671) +""" + class TestExamples(unittest.TestCase): def setUp(self): @@ -229,8 +235,26 @@ class TestExamples(unittest.TestCase): module.validate_relay(fingerprint) self.assertEqual(EXPECTED_CHECK_DIGESTS_BAD % server_desc.digest(), stdout_mock.getvalue()) - def test_collector_caching(self): - pass + @patch('stem.descriptor.collector.File.download', Mock()) + @patch('stem.descriptor.collector.CollecTor.files') + @patch('sys.stdout', new_callable = io.StringIO) + def test_collector_caching(self, stdout_mock, files_mock): + files_mock.return_value = [stem.descriptor.collector.File( + 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar', + ['server-descriptor 1.0'], + 1348620, + '0RrqB5aMY46vTeEHYqnbPVFGZQi1auJkzyHyt0NNDcw=', + '2005-12-15 01:42', + '2005-12-17 11:06', + '2016-06-24 08:12', + )] + + server_desc = list(stem.descriptor.parse_file(os.path.join(DESC_DIR, 'collector', 'server-descriptors-2005-12-cropped.tar'))) + + with patch('stem.descriptor.parse_file', Mock(return_value = server_desc)): + import collector_caching + + self.assertEqual(EXPECTED_COLLECTOR_CACHING, stdout_mock.getvalue()) def test_collector_reading(self): pass

1 0

[stem/master] Test check_digests example
by atagar＠torproject.org 02 Oct '20

02 Oct '20

commit 941c408c3a5c608e8272e629c61387f80ff1e2d4 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Sep 22 18:42:15 2020 -0700 Test check_digests example Yay! Caught our second example bug, this time a minor python 3 error with regard to iterating over a filter object. --- docs/_static/example/check_digests.py | 11 +++++-- test/unit/examples.py | 54 ++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/docs/_static/example/check_digests.py b/docs/_static/example/check_digests.py index 69f509cf..93b037c0 100644 --- a/docs/_static/example/check_digests.py +++ b/docs/_static/example/check_digests.py @@ -16,7 +16,7 @@ def download_descriptors(fingerprint): server_desc_query = stem.descriptor.remote.get_server_descriptors(fingerprint) extrainfo_query = stem.descriptor.remote.get_extrainfo_descriptors(fingerprint) - router_status_entries = filter(lambda desc: desc.fingerprint == fingerprint, conensus_query.run()) + router_status_entries = list(filter(lambda desc: desc.fingerprint == fingerprint, conensus_query.run())) if len(router_status_entries) != 1: raise OSError("Unable to find relay '%s' in the consensus" % fingerprint) @@ -27,8 +27,8 @@ def download_descriptors(fingerprint): extrainfo_query.run()[0], ) -if __name__ == '__main__': - fingerprint = raw_input("What relay fingerprint would you like to validate?\n") + +def validate_relay(fingerprint): print('') # blank line if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): @@ -50,3 +50,8 @@ if __name__ == '__main__': print("Extrainfo descriptor digest is correct") else: print("Extrainfo descriptor digest invalid, expected %s but is %s" % (server_desc.extra_info_digest, extrainfo_desc.digest())) + + +if __name__ == '__main__': + fingerprint = raw_input("What relay fingerprint would you like to validate?\n") + validate_relay(fingerprint) diff --git a/test/unit/examples.py b/test/unit/examples.py index 2f664fd3..ba75a98c 100644 --- a/test/unit/examples.py +++ b/test/unit/examples.py @@ -2,6 +2,8 @@ Exercise the code in our examples directory. """ +import base64 +import binascii import importlib import io import os @@ -11,9 +13,13 @@ import unittest import stem.socket import stem.util.system import test +import test.require from stem.control import Controller from stem.descriptor.bandwidth_file import BandwidthFile +from stem.descriptor.extrainfo_descriptor import RelayExtraInfoDescriptor +from stem.descriptor.router_status_entry import RouterStatusEntryV3 +from stem.descriptor.server_descriptor import RelayDescriptor from stem.response import ControlMessage from unittest.mock import Mock, patch @@ -69,6 +75,16 @@ Finished measure_fraction_relays_exit_80_microdescriptors('%s') Time per microdescriptor: """.rstrip() +EXPECTED_CHECK_DIGESTS_OK = """ +Server descriptor digest is correct +Extrainfo descriptor digest is correct +""" + +EXPECTED_CHECK_DIGESTS_BAD = """ +Server descriptor digest invalid, expected A106452D87BD7B803B6CE916291ED368DC5BD091 but is %s +Extrainfo descriptor digest is correct +""" + def import_example(module_name): """ @@ -174,8 +190,44 @@ class TestExamples(unittest.TestCase): self.assertEqual('start of broken_handler\n', stdout_mock.getvalue()) + @test.require.cryptography def test_check_digests(self): - pass + def download_of(desc): + query = Mock() + query.run.return_value = [desc] + return Mock(return_value = query) + + module = import_example('check_digests') + fingerprint = 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB' + + extrainfo_desc = RelayExtraInfoDescriptor.create() + server_desc = RelayDescriptor.create({'extra-info-digest': extrainfo_desc.digest()}, sign = True) + + encoded_digest = base64.b64encode(binascii.unhexlify(server_desc.digest())).rstrip(b'=') + + consensus_desc = RouterStatusEntryV3.create({ + 'r': 'caerSidi p1aag7VwarGxqctS7/fS0y5FU+s %s 2012-08-06 11:19:31 71.35.150.29 9001 0' % encoded_digest.decode('utf-8'), + }) + + bad_consensus_desc = RouterStatusEntryV3.create({ + 'r': 'caerSidi p1aag7VwarGxqctS7/fS0y5FU+s oQZFLYe9e4A7bOkWKR7TaNxb0JE 2012-08-06 11:19:31 71.35.150.29 9001 0', + }) + + with patch('stem.descriptor.remote.get_server_descriptors', download_of(server_desc)): + with patch('stem.descriptor.remote.get_extrainfo_descriptors', download_of(extrainfo_desc)): + # correctly signed descriptors + + with patch('stem.descriptor.remote.get_consensus', download_of(consensus_desc)): + with patch('sys.stdout', new_callable = io.StringIO) as stdout_mock: + module.validate_relay(fingerprint) + self.assertEqual(EXPECTED_CHECK_DIGESTS_OK, stdout_mock.getvalue()) + + # incorrect server descriptor digest + + with patch('stem.descriptor.remote.get_consensus', download_of(bad_consensus_desc)): + with patch('sys.stdout', new_callable = io.StringIO) as stdout_mock: + module.validate_relay(fingerprint) + self.assertEqual(EXPECTED_CHECK_DIGESTS_BAD % server_desc.digest(), stdout_mock.getvalue()) def test_client_usage_using_pycurl(self): pass

1 0

[stem/master] Test benchmark_stem example
by atagar＠torproject.org 02 Oct '20

02 Oct '20

commit 4497148d0be8b267e032d8867c082a3588529581 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Sep 21 17:26:30 2020 -0700 Test benchmark_stem example Caught our first bug in our example code... of sorts. This adds a 'descriptor_type' argument to benchmark_stem.py because CollecTor microdescriptor archives contain a mixture of consensuses and microdescriptors. --- docs/_static/example/benchmark_stem.py | 11 +++--- test/unit/examples.py | 62 +++++++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/docs/_static/example/benchmark_stem.py b/docs/_static/example/benchmark_stem.py index 8b71d730..7094c3dd 100644 --- a/docs/_static/example/benchmark_stem.py +++ b/docs/_static/example/benchmark_stem.py @@ -60,7 +60,7 @@ def measure_fraction_relays_exit_80_microdescriptors(path): start_time = time.time() exits, count = 0, 0 - for desc in stem.descriptor.parse_file(path): + for desc in stem.descriptor.parse_file(path, descriptor_type = 'microdescriptor 1.0'): if desc.exit_policy.can_exit_to(port = 80): exits += 1 @@ -74,8 +74,9 @@ def measure_fraction_relays_exit_80_microdescriptors(path): print(' Time per microdescriptor: %0.5f seconds' % (runtime / count)) print('') -measure_average_advertised_bandwidth('/home/atagar/Desktop/server-descriptors-2015-11.tar') -measure_countries_v3_requests('/home/atagar/Desktop/extra-infos-2015-11.tar') -measure_average_relays_exit('/home/atagar/Desktop/consensuses-2015-11.tar') -measure_fraction_relays_exit_80_microdescriptors('/home/atagar/Desktop/microdescs-2015-11.tar') +if __name__ == '__main__': + measure_average_advertised_bandwidth('/home/atagar/Desktop/server-descriptors-2015-11.tar') + measure_countries_v3_requests('/home/atagar/Desktop/extra-infos-2015-11.tar') + measure_average_relays_exit('/home/atagar/Desktop/consensuses-2015-11.tar') + measure_fraction_relays_exit_80_microdescriptors('/home/atagar/Desktop/microdescs-2015-11.tar') diff --git a/test/unit/examples.py b/test/unit/examples.py index 6a00d9b7..124ca3c7 100644 --- a/test/unit/examples.py +++ b/test/unit/examples.py @@ -15,6 +15,7 @@ from stem.descriptor.bandwidth_file import BandwidthFile from unittest.mock import patch EXAMPLE_DIR = os.path.join(test.STEM_BASE, 'docs', '_static', 'example') +DESC_DIR = os.path.join(test.STEM_BASE, 'test', 'unit', 'descriptor', 'data') EXPECTED_BANDWIDTH_STATS = """\ Relay FDCF49562E65B1CC219410009BD48A9EED387C77 @@ -33,7 +34,7 @@ Relay BD4172533C3F7271ABCCD9F057E06FD91547C42B """ -EXPECTED_BENCHMARK_SERVER_DESC_PREFIX = """\ +EXPECTED_SERVER_DESC_BENCHMARK_PREFIX = """\ Finished measure_average_advertised_bandwidth('%s') Total time: 0 seconds Processed server descriptors: 5 @@ -41,6 +42,30 @@ Finished measure_average_advertised_bandwidth('%s') Time per server descriptor: """.rstrip() +EXPECTED_EXTRAINFO_BENCHMARK_PREFIX = """\ +Finished measure_countries_v3_requests('%s') + Total time: 0 seconds + Processed extrainfo descriptors: 7 + Number of countries: 6 + Time per extrainfo descriptor: +""".rstrip() + +EXPECTED_CONSENSUS_BENCHMARK_PREFIX = """\ +Finished measure_average_relays_exit('%s') + Total time: 0 seconds + Processed 2 consensuses with 243 router status entries + Total exits: 28 (0.12%%) + Time per consensus: +""".rstrip() + +EXPECTED_MICRODESC_BENCHMARK_PREFIX = """\ +Finished measure_fraction_relays_exit_80_microdescriptors('%s') + Total time: 0 seconds + Processed microdescriptors: 3 + Total exits to port 80: 1 (0.33%%) + Time per microdescriptor: +""".rstrip() + def import_example(module_name): """ @@ -89,15 +114,44 @@ class TestExamples(unittest.TestCase): @patch('sys.stdout', new_callable = io.StringIO) def test_benchmark_server_descriptor_stem(self, stdout_mock): - path = os.path.join(test.STEM_BASE, 'test', 'unit', 'descriptor', 'data', 'collector', 'server-descriptors-2005-12-cropped.tar') + path = os.path.join(DESC_DIR, 'collector', 'server-descriptors-2005-12-cropped.tar') + expected_prefix = EXPECTED_SERVER_DESC_BENCHMARK_PREFIX % path module = import_example('benchmark_server_descriptor_stem') module.measure_average_advertised_bandwidth(path) - self.assertTrue(stdout_mock.getvalue().startswith(EXPECTED_BENCHMARK_SERVER_DESC_PREFIX % path)) + self.assertTrue(stdout_mock.getvalue().startswith(expected_prefix)) def test_benchmark_stem(self): - pass + module = import_example('benchmark_stem') + + with patch('sys.stdout', new_callable = io.StringIO) as stdout_mock: + path = os.path.join(DESC_DIR, 'collector', 'server-descriptors-2005-12-cropped.tar') + expected_prefix = EXPECTED_SERVER_DESC_BENCHMARK_PREFIX % path + + module.measure_average_advertised_bandwidth(path) + self.assertTrue(stdout_mock.getvalue().startswith(expected_prefix)) + + with patch('sys.stdout', new_callable = io.StringIO) as stdout_mock: + path = os.path.join(DESC_DIR, 'collector', 'extra-infos-2019-04-cropped.tar') + expected_prefix = EXPECTED_EXTRAINFO_BENCHMARK_PREFIX % path + + module.measure_countries_v3_requests(path) + self.assertTrue(stdout_mock.getvalue().startswith(expected_prefix)) + + with patch('sys.stdout', new_callable = io.StringIO) as stdout_mock: + path = os.path.join(DESC_DIR, 'collector', 'consensuses-2018-06-cropped.tar') + expected_prefix = EXPECTED_CONSENSUS_BENCHMARK_PREFIX % path + + module.measure_average_relays_exit(path) + self.assertTrue(stdout_mock.getvalue().startswith(expected_prefix)) + + with patch('sys.stdout', new_callable = io.StringIO) as stdout_mock: + path = os.path.join(DESC_DIR, 'collector', 'microdescs-2019-05-cropped.tar') + expected_prefix = EXPECTED_MICRODESC_BENCHMARK_PREFIX % path + + module.measure_fraction_relays_exit_80_microdescriptors(path) + self.assertTrue(stdout_mock.getvalue().startswith(expected_prefix)) def test_broken_listener(self): pass

1 0

[stem/master] Test broken_listener example
by atagar＠torproject.org 02 Oct '20

02 Oct '20

commit f8e075cdf7782ad5bee036274d51b43d99cf2f34 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Sep 22 16:46:47 2020 -0700 Test broken_listener example Ooph! Despite being a tiny test for a tiny example this took me hours to get the mocks right. On reflection though this is pretty straight forward. is_alive() must be mocked so the 'with' clause doesn't attempt to connect to something. --- test/unit/examples.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/test/unit/examples.py b/test/unit/examples.py index 124ca3c7..2f664fd3 100644 --- a/test/unit/examples.py +++ b/test/unit/examples.py @@ -8,11 +8,14 @@ import os import sys import unittest +import stem.socket import stem.util.system import test +from stem.control import Controller from stem.descriptor.bandwidth_file import BandwidthFile -from unittest.mock import patch +from stem.response import ControlMessage +from unittest.mock import Mock, patch EXAMPLE_DIR = os.path.join(test.STEM_BASE, 'docs', '_static', 'example') DESC_DIR = os.path.join(test.STEM_BASE, 'test', 'unit', 'descriptor', 'data') @@ -153,8 +156,23 @@ class TestExamples(unittest.TestCase): module.measure_fraction_relays_exit_80_microdescriptors(path) self.assertTrue(stdout_mock.getvalue().startswith(expected_prefix)) - def test_broken_listener(self): - pass + @patch('time.sleep') + @patch('stem.control.Controller.authenticate', Mock()) + @patch('stem.control.Controller.is_alive', Mock(return_value = True)) + @patch('stem.control.Controller.from_port') + @patch('sys.stdout', new_callable = io.StringIO) + def test_broken_listener(self, stdout_mock, from_port_mock, sleep_mock): + controller = Controller(stem.socket.ControlSocket()) + from_port_mock.return_value = controller + + # emits a BW event when the example runs time.sleep() + + bw_event = ControlMessage.from_str('650 BW 15 25', 'EVENT', normalize = True) + sleep_mock.side_effect = lambda duration: controller._handle_event(bw_event) + + import_example('broken_listener') + + self.assertEqual('start of broken_handler\n', stdout_mock.getvalue()) def test_check_digests(self): pass

1 0

[stem/master] Move list_circuits test
by atagar＠torproject.org 02 Oct '20

02 Oct '20

commit faa569fcbe8a8d5bb1014c2961478a33465d113e Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Sep 26 18:08:45 2020 -0700 Move list_circuits test Yikes. We have *three* test modules for our website examples... * tutorial.py, made in 2012 * tutorial_examples.py, made in 2014 * examples.py, from this branch I've worked on Stem for so long clearly even I can't keep all our tests straight. These modules cover a subset of our tutorial examples, so beginning to merge them into examples.py. Unlike them, examples.py checks for new example scripts so we'll finally stop re-inventing this wheel. --- docs/_static/example/list_circuits.py | 6 ++-- stem/response/events.py | 4 +++ test/unit/examples.py | 58 +++++++++++++++++++++++++++++++++-- test/unit/tutorial_examples.py | 49 ----------------------------- 4 files changed, 63 insertions(+), 54 deletions(-) diff --git a/docs/_static/example/list_circuits.py b/docs/_static/example/list_circuits.py index b89e99fc..aff3b211 100644 --- a/docs/_static/example/list_circuits.py +++ b/docs/_static/example/list_circuits.py @@ -8,8 +8,8 @@ with Controller.from_port(port = 9051) as controller: if circ.status != CircStatus.BUILT: continue - print("") - print("Circuit %s (%s)" % (circ.id, circ.purpose)) + print('') + print('Circuit %s (%s)' % (circ.id, circ.purpose)) for i, entry in enumerate(circ.path): div = '+' if (i == len(circ.path) - 1) else '|' @@ -18,4 +18,4 @@ with Controller.from_port(port = 9051) as controller: desc = controller.get_network_status(fingerprint, None) address = desc.address if desc else 'unknown' - print(" %s- %s (%s, %s)" % (div, fingerprint, nickname, address)) + print(' %s- %s (%s, %s)' % (div, fingerprint, nickname, address)) diff --git a/stem/response/events.py b/stem/response/events.py index 300d02d0..d9d46d6a 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -417,6 +417,10 @@ class CircuitEvent(Event): my_id = getattr(self, 'id') their_id = getattr(other, 'id') + if my_id.isdigit() and their_id.isdigit(): + my_id = int(my_id) + their_id = int(their_id) + return method(my_id, their_id) if my_id != their_id else method(hash(self), hash(other)) def __gt__(self, other: Any) -> bool: diff --git a/test/unit/examples.py b/test/unit/examples.py index 7c801ece..30375ea3 100644 --- a/test/unit/examples.py +++ b/test/unit/examples.py @@ -104,6 +104,24 @@ EXPECTED_COLLECTOR_READING = """\ caerSidi (4F0C867DF0EF68160568C826838F482CEA7CFE44) """ +EXPECTED_LIST_CIRCUITS = """\ + +Circuit 4 (GENERAL) + |- B1FA7D51B8B6F0CB585D944F450E7C06EDE7E44C (ByTORAndTheSnowDog, 173.209.180.61) + |- 0DD9935C5E939CFA1E07B8DDA6D91C1A2A9D9338 (afo02, 87.238.194.176) + +- DB3B1CFBD3E4D97B84B548ADD5B9A31451EEC4CC (edwardsnowden3, 109.163.234.10) + +Circuit 6 (GENERAL) + |- B1FA7D51B8B6F0CB585D944F450E7C06EDE7E44C (ByTORAndTheSnowDog, 173.209.180.61) + |- EC01CB4766BADC1611678555CE793F2A7EB2D723 (sprockets, 46.165.197.96) + +- 9EA317EECA56BDF30CAEB208A253FB456EDAB1A0 (bolobolo1, 96.47.226.20) + +Circuit 10 (GENERAL) + |- B1FA7D51B8B6F0CB585D944F450E7C06EDE7E44C (ByTORAndTheSnowDog, 173.209.180.61) + |- 00C2C2A16AEDB51D5E5FB7D6168FC66B343D822F (ph3x, 86.59.119.83) + +- 65242C91BFF30F165DA4D132C81A9EBA94B71D62 (torexit16, 176.67.169.171) +""" + class TestExamples(unittest.TestCase): def setUp(self): @@ -328,8 +346,44 @@ class TestExamples(unittest.TestCase): def test_introduction_points(self): pass - def test_list_circuits(self): - pass + @patch('stem.control.Controller.from_port', spec = Controller) + @patch('sys.stdout', new_callable = io.StringIO) + def test_list_circuits(self, stdout_mock, from_port_mock): + def _get_circ_event(circ_id, hop1, hop2, hop3): + path = '$%s=%s,$%s=%s,$%s=%s' % (hop1[0], hop1[1], hop2[0], hop2[1], hop3[0], hop3[1]) + content = '650 CIRC %i BUILT %s PURPOSE=GENERAL' % (circ_id, path) + return ControlMessage.from_str(content, 'EVENT', normalize = True) + + path_1 = ('B1FA7D51B8B6F0CB585D944F450E7C06EDE7E44C', 'ByTORAndTheSnowDog') + path_2 = ('0DD9935C5E939CFA1E07B8DDA6D91C1A2A9D9338', 'afo02') + path_3 = ('DB3B1CFBD3E4D97B84B548ADD5B9A31451EEC4CC', 'edwardsnowden3') + path_4 = ('EC01CB4766BADC1611678555CE793F2A7EB2D723', 'sprockets') + path_5 = ('9EA317EECA56BDF30CAEB208A253FB456EDAB1A0', 'bolobolo1') + path_6 = ('00C2C2A16AEDB51D5E5FB7D6168FC66B343D822F', 'ph3x') + path_7 = ('65242C91BFF30F165DA4D132C81A9EBA94B71D62', 'torexit16') + + circuit_4 = _get_circ_event(4, path_1, path_2, path_3) + circuit_6 = _get_circ_event(6, path_1, path_4, path_5) + circuit_10 = _get_circ_event(10, path_1, path_6, path_7) + + controller = from_port_mock().__enter__() + controller.get_circuits.return_value = [circuit_4, circuit_6, circuit_10] + + r_line = 'caerSidi p1aag7VwarGxqctS7/fS0y5FU+s oQZFLYe9e4A7bOkWKR7TaNxb0JE 2012-08-06 11:19:31 %s 9001 0' + + controller.get_network_status.side_effect = lambda fingerprint, *args: { + path_1[0]: RouterStatusEntryV3.create({'r': r_line % '173.209.180.61'}), + path_2[0]: RouterStatusEntryV3.create({'r': r_line % '87.238.194.176'}), + path_3[0]: RouterStatusEntryV3.create({'r': r_line % '109.163.234.10'}), + path_4[0]: RouterStatusEntryV3.create({'r': r_line % '46.165.197.96'}), + path_5[0]: RouterStatusEntryV3.create({'r': r_line % '96.47.226.20'}), + path_6[0]: RouterStatusEntryV3.create({'r': r_line % '86.59.119.83'}), + path_7[0]: RouterStatusEntryV3.create({'r': r_line % '176.67.169.171'}), + }[fingerprint] + + import list_circuits + + self.assertEqual(EXPECTED_LIST_CIRCUITS, stdout_mock.getvalue()) def test_load_test(self): pass diff --git a/test/unit/tutorial_examples.py b/test/unit/tutorial_examples.py index 5ccf1ce2..d673662f 100644 --- a/test/unit/tutorial_examples.py +++ b/test/unit/tutorial_examples.py @@ -26,24 +26,6 @@ PURPOSE=%s' PATH_CONTENT = '$%s=%s,$%s=%s,$%s=%s' -LIST_CIRCUITS_OUTPUT = """\ - -Circuit 4 (GENERAL) - |- B1FA7D51B8B6F0CB585D944F450E7C06EDE7E44C (ByTORAndTheSnowDog, 173.209.180.61) - |- 0DD9935C5E939CFA1E07B8DDA6D91C1A2A9D9338 (afo02, 87.238.194.176) - +- DB3B1CFBD3E4D97B84B548ADD5B9A31451EEC4CC (edwardsnowden3, 109.163.234.10) - -Circuit 6 (GENERAL) - |- B1FA7D51B8B6F0CB585D944F450E7C06EDE7E44C (ByTORAndTheSnowDog, 173.209.180.61) - |- EC01CB4766BADC1611678555CE793F2A7EB2D723 (sprockets, 46.165.197.96) - +- 9EA317EECA56BDF30CAEB208A253FB456EDAB1A0 (bolobolo1, 96.47.226.20) - -Circuit 10 (GENERAL) - |- B1FA7D51B8B6F0CB585D944F450E7C06EDE7E44C (ByTORAndTheSnowDog, 173.209.180.61) - |- 00C2C2A16AEDB51D5E5FB7D6168FC66B343D822F (ph3x, 86.59.119.83) - +- 65242C91BFF30F165DA4D132C81A9EBA94B71D62 (torexit16, 176.67.169.171) -""" - EXIT_USED_OUTPUT = """\ Tracking requests for tor exits. Press 'enter' to end. @@ -117,37 +99,6 @@ def _get_router_status(address = None, port = None, nickname = None, fingerprint class TestTutorialExamples(unittest.TestCase): - @patch('sys.stdout', new_callable = io.StringIO) - @patch('stem.control.Controller.from_port', spec = Controller) - def test_list_circuits(self, from_port_mock, stdout_mock): - path_1 = ('B1FA7D51B8B6F0CB585D944F450E7C06EDE7E44C', 'ByTORAndTheSnowDog') - path_2 = ('0DD9935C5E939CFA1E07B8DDA6D91C1A2A9D9338', 'afo02') - path_3 = ('DB3B1CFBD3E4D97B84B548ADD5B9A31451EEC4CC', 'edwardsnowden3') - path_4 = ('EC01CB4766BADC1611678555CE793F2A7EB2D723', 'sprockets') - path_5 = ('9EA317EECA56BDF30CAEB208A253FB456EDAB1A0', 'bolobolo1') - path_6 = ('00C2C2A16AEDB51D5E5FB7D6168FC66B343D822F', 'ph3x') - path_7 = ('65242C91BFF30F165DA4D132C81A9EBA94B71D62', 'torexit16') - - circuit_4 = _get_circ_event(4, 'BUILT', path_1, path_2, path_3, 'GENERAL') - circuit_6 = _get_circ_event(6, 'BUILT', path_1, path_4, path_5, 'GENERAL') - circuit_10 = _get_circ_event(10, 'BUILT', path_1, path_6, path_7, 'GENERAL') - - controller = from_port_mock().__enter__() - controller.get_circuits.return_value = [circuit_4, circuit_6, circuit_10] - - controller.get_network_status.side_effect = lambda fingerprint, *args: { - path_1[0]: _get_router_status('173.209.180.61'), - path_2[0]: _get_router_status('87.238.194.176'), - path_3[0]: _get_router_status('109.163.234.10'), - path_4[0]: _get_router_status('46.165.197.96'), - path_5[0]: _get_router_status('96.47.226.20'), - path_6[0]: _get_router_status('86.59.119.83'), - path_7[0]: _get_router_status('176.67.169.171') - }[fingerprint] - - exec_documentation_example('list_circuits.py') - self.assertCountEqual(LIST_CIRCUITS_OUTPUT.splitlines(), stdout_mock.getvalue().splitlines()) - @patch('sys.stdout', new_callable = io.StringIO) @patch('stem.control.Controller.from_port', spec = Controller) def test_exit_used(self, from_port_mock, stdout_mock):

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits October 2020