June 2018 - tor-commits - lists.torproject.org

[sbws/master] Move timestmap util tests, and fix them to actually pass
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 3aa5444c67b4a05059c73810849d882538c6ebbb Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 20:58:27 2018 -0400 Move timestmap util tests, and fix them to actually pass --- tests/{ => unit}/util/test_timestamp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/util/test_timestamp.py b/tests/unit/util/test_timestamp.py similarity index 86% rename from tests/util/test_timestamp.py rename to tests/unit/util/test_timestamp.py index 0ccf2ec..8ebc2f6 100644 --- a/tests/util/test_timestamp.py +++ b/tests/unit/util/test_timestamp.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """Test timestamp conversion util functions""" -from datetime import datetime +from datetime import datetime, timezone from sbws.util.timestamp import (dt_obj_to_isodt_str, unixts_to_dt_obj, unixts_to_isodt_str, unixts_to_str) @@ -8,7 +8,7 @@ from sbws.util.timestamp import (dt_obj_to_isodt_str, unixts_to_dt_obj, isodt_str = '2018-05-23T12:55:04' dt_obj = datetime.strptime(isodt_str, '%Y-%m-%dT%H:%M:%S') -unixts = int(dt_obj.timestamp()) +unixts = int(dt_obj.replace(tzinfo=timezone.utc).timestamp()) def test_dt_obj_to_isodt_str():

1 0

[sbws/master] log.warn is deprecated, use log.warning
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 16f1ae040709c9063c2f8b43f223b163290fc870 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 09:35:15 2018 -0400 log.warn is deprecated, use log.warning --- sbws/lib/resultdump.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index aab27e7..feadfdb 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -116,8 +116,8 @@ def trim_results_ip_changed(result_dict, on_changed_ipv4=False, new_results_dict[fp] = results return new_results_dict if on_changed_ipv6 is True: - log.warn("Reseting bandwidth results when IPv6 changes," - " is not yet implemented.") + log.warning("Reseting bandwidth results when IPv6 changes," + " is not yet implemented.") return result_dict

1 0

[sbws/master] Merge pull request #191 from pastly/issue155
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit cf3e47851895617bfd173c9154526290a1fcd7d3 Merge: 104fbe7 467f98a Author: juga0 <juga(a)riseup.net> Date: Sat Jun 23 18:22:28 2018 +0000 Merge pull request #191 from pastly/issue155 Implement the "MaxAdvertisedBandwidth" hack ... CHANGELOG.md | 2 + sbws/globals.py | 2 +- sbws/lib/resultdump.py | 14 +- sbws/lib/v3bwfile.py | 20 ++- scripts/tools/sbws-http-server.py | 76 +++++++++ tests/integration/core/test_scanner.py | 92 +++++++++++ tests/integration/net.tar | Bin 191488 -> 215040 bytes tests/unit/core/test_generate.py | 284 --------------------------------- tests/unit/lib/data/results.txt | 4 +- tests/unit/lib/test_results.py | 4 +- tests/unit/lib/test_v3bwfile.py | 3 +- tox.ini | 5 +- 12 files changed, 212 insertions(+), 294 deletions(-)

1 0

[sbws/master] Add more info to a comment
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit eebfd257adb10fbde1e9101aba8680442aec0360 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 09:28:18 2018 -0400 Add more info to a comment --- sbws/config.default.ini | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index d3abcda..80ddc8b 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -23,13 +23,11 @@ http_timeout = 10 # CircuitBuildTimeout doesn't handle the case of a TLS connection to a relay # taking forever, and probably other not-yet-discovered cases. circuit_timeout = 10 -# Whether or not to reset the bandwidth measurements when the the relay's IPv4 -# changes +# Whether or not to reset the bandwidth measurements when the relay's IP +# address changes. If it changes, we only consider results for the relay that +# we obtained while the relay was located at its most recent IP address. +# This is NOT implemented for IPv6. reset_bw_ipv4_changes = on - -# Whether or not to reset the bandwidth measurements when the the relay's IPv6 -# changes. -# This is currently not implemented. reset_bw_ipv6_changes = off [scanner]

1 0

[sbws/master] Changelog entry for integration test
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit d5ea67c9d99c8b0c2a98f06847204ca9a82cac65 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 21 21:15:20 2018 -0400 Changelog entry for integration test --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3982de..18bffbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. (GH#154 GHPR#199) - Maintenance script to help us find functions that are (probably) no longer being called. +- Integration test(s) for RelayPrioritizer (GHPR#206) ### Fixed

1 0

[sbws/master] Add changelog entry
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 079932618723daf136dbcb81e70bc9dced9b6c0d Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 09:39:21 2018 -0400 Add changelog entry --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e40965..f0cf0b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added + +- Ability to only consider results for a given relay valid if they came from + when that relay is using its most recent known IP address. Thanks Juga. +(GH#154 GHPR#199) + ### Changed - Change the path where the Bandwidth List files are generated: now they are

1 0

[sbws/master] Update tests to handle new result format and version
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit c238a9387dbd44e79a6e7cd3436d456805e0966d Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 20:39:50 2018 -0400 Update tests to handle new result format and version --- tests/unit/lib/data/results.txt | 4 ++-- tests/unit/lib/test_results.py | 4 +++- tests/unit/lib/test_v3bwfile.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/unit/lib/data/results.txt b/tests/unit/lib/data/results.txt index d2005f6..359f213 100644 --- a/tests/unit/lib/data/results.txt +++ b/tests/unit/lib/data/results.txt @@ -1,2 +1,2 @@ -{"version": 3, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} -{"version": 3, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} +{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_average_bandwidth": 1000000000} +{"version": 4, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} diff --git a/tests/unit/lib/test_results.py b/tests/unit/lib/test_results.py index 3d4770e..328d90a 100644 --- a/tests/unit/lib/test_results.py +++ b/tests/unit/lib/test_results.py @@ -103,7 +103,8 @@ def test_ResultSuccess_from_dict(time_mock): scanner_nick = 'sbwsscanner' nick = 'Mooooooo' relay_ip = '169.254.100.1' - relay = Result.Relay(fp1, nick, relay_ip, ed25519) + relay_average_bandwidth = 1 * 1024 * 1024 + relay = Result.Relay(fp1, nick, relay_ip, ed25519, relay_average_bandwidth) rtts = [5, 25] downloads = [{'duration': 4, 'amount': 40}] r1 = ResultSuccess(rtts, downloads, relay, circ, dest_url, scanner_nick) @@ -113,6 +114,7 @@ def test_ResultSuccess_from_dict(time_mock): 'dest_url': dest_url, 'scanner': scanner_nick, 'version': RESULT_VERSION, 'type': _ResultType.Success, 'time': t, 'master_key_ed25519': ed25519, + 'relay_average_bandwidth': relay_average_bandwidth, } r2 = Result.from_dict(d) assert isinstance(r1, ResultSuccess) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 2f4a631..bfe9d9b 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -78,7 +78,8 @@ RESULT_SUCCESS_DICT = { {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "nickname": "A", - "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s" + "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", + "relay_average_bandwidth": 1 * 1024 * 1024, } RESULT_SUCCESS_STR = str(RESULT_SUCCESS_DICT) RESULT_ERROR_STREAM_STR = str(RESULT_ERROR_STREAM_DICT)

1 0

[sbws/master] Fixup MaxAdvertisedBandwidth test
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit d5659732d57089ec2325880e1ce38b997f50b620 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 14:47:27 2018 -0400 Fixup MaxAdvertisedBandwidth test --- tests/integration/core/test_scanner.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/integration/core/test_scanner.py b/tests/integration/core/test_scanner.py index a9a467f..c1a8692 100644 --- a/tests/integration/core/test_scanner.py +++ b/tests/integration/core/test_scanner.py @@ -4,6 +4,7 @@ from sbws.util.config import get_config from sbws.lib.relaylist import RelayList from sbws.lib.destination import DestinationList from sbws.lib.resultdump import ResultSuccess +import logging def assert_within(value, target, radius): @@ -40,7 +41,8 @@ def get_everything_to_measure(dotsbws, cont, parser): def test_measure_relay_with_maxadvertisedbandwidth( - persistent_launch_tor, parser, persistent_empty_dotsbws): + persistent_launch_tor, parser, persistent_empty_dotsbws, caplog): + caplog.set_level(logging.DEBUG) cont = persistent_launch_tor dotsbws = persistent_empty_dotsbws.name d = get_everything_to_measure(dotsbws, cont, parser) @@ -57,10 +59,13 @@ def test_measure_relay_with_maxadvertisedbandwidth( result = result[0] assert isinstance(result, ResultSuccess) one_mbyte = 1 * 1024 * 1024 - allowed_error = 5 # bytes per second dls = result.downloads for dl in dls: - assert_within(dl['amount'] / dl['duration'], one_mbyte, allowed_error) + # This relay has MaxAdvertisedBandwidth set, but should not be limited + # to just 1 Mbyte. Assume and assert that all downloads where at least + # more than 10% faster than 1 MBps + assert dl['amount'] / dl['duration'] > one_mbyte * 1.1 + assert result.relay_average_bandwidth == one_mbyte def test_measure_relay_with_relaybandwidthrate(

1 0

[sbws/master] Make sure we only refresh our list of relays once
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 5df7396760a0703e948e97c74e2179b86aef6a90 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Fri Jun 22 09:53:17 2018 -0400 Make sure we only refresh our list of relays once GH: ref #205 --- sbws/lib/relaylist.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index f87d3d0..8b19a7c 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -8,6 +8,7 @@ import random import time import logging from sbws.globals import resolve +from threading import Lock log = logging.getLogger(__name__) @@ -124,12 +125,26 @@ class RelayList: def __init__(self, args, conf, controller): self._controller = controller self.rng = random.SystemRandom() + self._refresh_lock = Lock() self._refresh() + def _need_refresh(self): + return time.time() >= self._last_refresh + self.REFRESH_INTERVAL + @property def relays(self): - if time.time() >= self._last_refresh + self.REFRESH_INTERVAL: - self._refresh() + # See if we can get the list of relays without having to do a refresh, + # which is expensive and blocks other threads + if self._need_refresh(): + # Whelp we couldn't just get the list of relays because the list is + # stale. Wait for the lock so we can refresh it. + with self._refresh_lock: + # Now we have the lock ... but wait! Maybe someone else already + # did the refreshing. So check if it still needs refreshing. If + # not, we can do nothing. + if self._need_refresh(): + self._refresh() + assert not self._need_refresh() return self._relays @property

1 0

[sbws/master] Add integration test for RelayPrioritizer
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit b1d591c2cf90fe5bf70861cc289bc2b1ff5eb081 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 21 21:13:48 2018 -0400 Add integration test for RelayPrioritizer --- tests/integration/conftest.py | 4 +- tests/integration/lib/test_relayprioritizer.py | 91 ++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e35341d..94b2433 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,6 +4,7 @@ from sbws.util.parser import create_parser from sbws.util.config import get_config from sbws.util.stem import launch_tor import sbws.core.init +import os @pytest.fixture(scope='session') @@ -14,13 +15,14 @@ def parser(): @pytest.fixture(scope='session') def persistent_empty_dotsbws(parser): ''' - Creates a ~/.sbws with nothing in it but a config.ini + Creates a ~/.sbws with nothing in it but a config.ini and a datadir/ ''' d = TemporaryDirectory() args = parser.parse_args( '-d {} --log-level DEBUG init'.format(d.name).split()) conf = get_config(args) sbws.core.init.main(args, conf) + os.makedirs(os.path.join(d.name, 'datadir')) return d diff --git a/tests/integration/lib/test_relayprioritizer.py b/tests/integration/lib/test_relayprioritizer.py new file mode 100644 index 0000000..0267843 --- /dev/null +++ b/tests/integration/lib/test_relayprioritizer.py @@ -0,0 +1,91 @@ +from sbws.lib.resultdump import ResultDump +from sbws.lib.resultdump import (Result, ResultSuccess, ResultErrorCircuit) +from sbws.lib.relaylist import RelayList +from sbws.lib.relayprioritizer import RelayPrioritizer +from sbws.util.config import get_config +from threading import Event +from unittest.mock import patch + + +def static_time(value): + while True: + yield value + + +def get_global_stuff(dotsbws, cont, parser): + args = parser.parse_args( + '-d {} --log-level DEBUG'.format(dotsbws).split()) + conf = get_config(args) + rl = RelayList(args, conf, cont) + return { + 'args': args, + 'conf': conf, + 'rl': rl, + 'end': Event(), + } + + +def _build_result_for_relay(relay_nick, result_type, timestamp, rl): + relay = [r for r in rl.relays if r.nickname == relay_nick] + assert len(relay) == 1 + relay = relay[0] + other = [r for r in rl.relays if r.nickname != relay_nick][0] + circ = [relay.fingerprint, other.fingerprint] + url = 'http://example.com/sbws.bin' + nick = 'sbws_scanner' + if result_type == ResultSuccess: + rtts = [0.5, 0.5, 0.5] + dls = [ + {'amount': 1024, 'duration': 1}, + {'amount': 1024, 'duration': 1}, + {'amount': 1024, 'duration': 1}, + ] + return ResultSuccess(rtts, dls, relay, circ, url, nick, t=timestamp) + elif result_type == ResultErrorCircuit: + return ResultErrorCircuit( + relay, circ, url, nick, msg='Test error circ message', t=timestamp) + + +@patch('time.time') +def test_relayprioritizer_general( + time_mock, persistent_empty_dotsbws, parser, persistent_launch_tor): + now = 1000000 + time_mock.side_effect = static_time(now) + cont = persistent_launch_tor + dotsbws = persistent_empty_dotsbws.name + d = get_global_stuff(dotsbws, cont, parser) + args = d['args'] + conf = d['conf'] + end_event = d['end'] + rl = d['rl'] + rd = ResultDump(args, conf, end_event) + try: + rp = RelayPrioritizer(args, conf, rl, rd) + results = [ + _build_result_for_relay( + 'relay1', ResultSuccess, now - 100, rl), + _build_result_for_relay( + 'relay2', ResultSuccess, now - 200, rl), + _build_result_for_relay( + 'relay3', ResultSuccess, now - 300, rl), + _build_result_for_relay( + 'relay4', ResultSuccess, now - 400, rl), + _build_result_for_relay( + 'relay5', ResultSuccess, now - 500, rl), + ] + for result in results: + rd.store_result(result) + best_list = [_ for _ in rp.best_priority()] + # Of the relays for which we have added results to the ResultDump, + # relay1 has the lowest priority (it has the most recent result) and + # relay5 has the highest prioirty. The relays that we didn't add + # results for will have the highest priority, but don't test the order + # of them. Skip to the end of the list and check those guys since they + # should have a defined order. + for i in range(1, 5+1): + nick = 'relay{}'.format(i) + pos = i * -1 + relay = best_list[pos] + assert relay.nickname == nick + finally: + end_event.set()

1 0