June 2018 - tor-commits - lists.torproject.org

[sbws/master] Add relay1mbyteRBR to integration testnet
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 35acd72a487f31efe495acfe2bc64e34b0ff7967 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Tue Jun 19 22:19:00 2018 -0400 Add relay1mbyteRBR to integration testnet Has 'RelayBandwidthRate 1 MByte' set --- tests/integration/net.tar | Bin 204800 -> 215040 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/integration/net.tar b/tests/integration/net.tar index cde1f17..2870690 100644 Binary files a/tests/integration/net.tar and b/tests/integration/net.tar differ

1 0

[sbws/master] Move timestmap util tests, and fix them to actually pass
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 3aa5444c67b4a05059c73810849d882538c6ebbb Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 20:58:27 2018 -0400 Move timestmap util tests, and fix them to actually pass --- tests/{ => unit}/util/test_timestamp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/util/test_timestamp.py b/tests/unit/util/test_timestamp.py similarity index 86% rename from tests/util/test_timestamp.py rename to tests/unit/util/test_timestamp.py index 0ccf2ec..8ebc2f6 100644 --- a/tests/util/test_timestamp.py +++ b/tests/unit/util/test_timestamp.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """Test timestamp conversion util functions""" -from datetime import datetime +from datetime import datetime, timezone from sbws.util.timestamp import (dt_obj_to_isodt_str, unixts_to_dt_obj, unixts_to_isodt_str, unixts_to_str) @@ -8,7 +8,7 @@ from sbws.util.timestamp import (dt_obj_to_isodt_str, unixts_to_dt_obj, isodt_str = '2018-05-23T12:55:04' dt_obj = datetime.strptime(isodt_str, '%Y-%m-%dT%H:%M:%S') -unixts = int(dt_obj.timestamp()) +unixts = int(dt_obj.replace(tzinfo=timezone.utc).timestamp()) def test_dt_obj_to_isodt_str():

1 0

[sbws/master] log.warn is deprecated, use log.warning
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 16f1ae040709c9063c2f8b43f223b163290fc870 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 09:35:15 2018 -0400 log.warn is deprecated, use log.warning --- sbws/lib/resultdump.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index aab27e7..feadfdb 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -116,8 +116,8 @@ def trim_results_ip_changed(result_dict, on_changed_ipv4=False, new_results_dict[fp] = results return new_results_dict if on_changed_ipv6 is True: - log.warn("Reseting bandwidth results when IPv6 changes," - " is not yet implemented.") + log.warning("Reseting bandwidth results when IPv6 changes," + " is not yet implemented.") return result_dict

1 0

[sbws/master] Merge pull request #191 from pastly/issue155
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit cf3e47851895617bfd173c9154526290a1fcd7d3 Merge: 104fbe7 467f98a Author: juga0 <juga(a)riseup.net> Date: Sat Jun 23 18:22:28 2018 +0000 Merge pull request #191 from pastly/issue155 Implement the "MaxAdvertisedBandwidth" hack ... CHANGELOG.md | 2 + sbws/globals.py | 2 +- sbws/lib/resultdump.py | 14 +- sbws/lib/v3bwfile.py | 20 ++- scripts/tools/sbws-http-server.py | 76 +++++++++ tests/integration/core/test_scanner.py | 92 +++++++++++ tests/integration/net.tar | Bin 191488 -> 215040 bytes tests/unit/core/test_generate.py | 284 --------------------------------- tests/unit/lib/data/results.txt | 4 +- tests/unit/lib/test_results.py | 4 +- tests/unit/lib/test_v3bwfile.py | 3 +- tox.ini | 5 +- 12 files changed, 212 insertions(+), 294 deletions(-)

1 0

[sbws/master] Add more info to a comment
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit eebfd257adb10fbde1e9101aba8680442aec0360 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 09:28:18 2018 -0400 Add more info to a comment --- sbws/config.default.ini | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index d3abcda..80ddc8b 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -23,13 +23,11 @@ http_timeout = 10 # CircuitBuildTimeout doesn't handle the case of a TLS connection to a relay # taking forever, and probably other not-yet-discovered cases. circuit_timeout = 10 -# Whether or not to reset the bandwidth measurements when the the relay's IPv4 -# changes +# Whether or not to reset the bandwidth measurements when the relay's IP +# address changes. If it changes, we only consider results for the relay that +# we obtained while the relay was located at its most recent IP address. +# This is NOT implemented for IPv6. reset_bw_ipv4_changes = on - -# Whether or not to reset the bandwidth measurements when the the relay's IPv6 -# changes. -# This is currently not implemented. reset_bw_ipv6_changes = off [scanner]

1 0

[sbws/master] Changelog entry for integration test
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit d5ea67c9d99c8b0c2a98f06847204ca9a82cac65 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Thu Jun 21 21:15:20 2018 -0400 Changelog entry for integration test --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3982de..18bffbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. (GH#154 GHPR#199) - Maintenance script to help us find functions that are (probably) no longer being called. +- Integration test(s) for RelayPrioritizer (GHPR#206) ### Fixed

1 0

[sbws/master] Add changelog entry
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 079932618723daf136dbcb81e70bc9dced9b6c0d Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 09:39:21 2018 -0400 Add changelog entry --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e40965..f0cf0b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added + +- Ability to only consider results for a given relay valid if they came from + when that relay is using its most recent known IP address. Thanks Juga. +(GH#154 GHPR#199) + ### Changed - Change the path where the Bandwidth List files are generated: now they are

1 0

[sbws/master] Update tests to handle new result format and version
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit c238a9387dbd44e79a6e7cd3436d456805e0966d Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 20:39:50 2018 -0400 Update tests to handle new result format and version --- tests/unit/lib/data/results.txt | 4 ++-- tests/unit/lib/test_results.py | 4 +++- tests/unit/lib/test_v3bwfile.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/unit/lib/data/results.txt b/tests/unit/lib/data/results.txt index d2005f6..359f213 100644 --- a/tests/unit/lib/data/results.txt +++ b/tests/unit/lib/data/results.txt @@ -1,2 +1,2 @@ -{"version": 3, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} -{"version": 3, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} +{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_average_bandwidth": 1000000000} +{"version": 4, "time": 1523974147, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "error-stream", "msg": "Something bad happened while measuring bandwidth", "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s"} diff --git a/tests/unit/lib/test_results.py b/tests/unit/lib/test_results.py index 3d4770e..328d90a 100644 --- a/tests/unit/lib/test_results.py +++ b/tests/unit/lib/test_results.py @@ -103,7 +103,8 @@ def test_ResultSuccess_from_dict(time_mock): scanner_nick = 'sbwsscanner' nick = 'Mooooooo' relay_ip = '169.254.100.1' - relay = Result.Relay(fp1, nick, relay_ip, ed25519) + relay_average_bandwidth = 1 * 1024 * 1024 + relay = Result.Relay(fp1, nick, relay_ip, ed25519, relay_average_bandwidth) rtts = [5, 25] downloads = [{'duration': 4, 'amount': 40}] r1 = ResultSuccess(rtts, downloads, relay, circ, dest_url, scanner_nick) @@ -113,6 +114,7 @@ def test_ResultSuccess_from_dict(time_mock): 'dest_url': dest_url, 'scanner': scanner_nick, 'version': RESULT_VERSION, 'type': _ResultType.Success, 'time': t, 'master_key_ed25519': ed25519, + 'relay_average_bandwidth': relay_average_bandwidth, } r2 = Result.from_dict(d) assert isinstance(r1, ResultSuccess) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 2f4a631..bfe9d9b 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -78,7 +78,8 @@ RESULT_SUCCESS_DICT = { {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "nickname": "A", - "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s" + "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", + "relay_average_bandwidth": 1 * 1024 * 1024, } RESULT_SUCCESS_STR = str(RESULT_SUCCESS_DICT) RESULT_ERROR_STREAM_STR = str(RESULT_ERROR_STREAM_DICT)

1 0

[sbws/master] Fixup MaxAdvertisedBandwidth test
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit d5659732d57089ec2325880e1ce38b997f50b620 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Jun 20 14:47:27 2018 -0400 Fixup MaxAdvertisedBandwidth test --- tests/integration/core/test_scanner.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/integration/core/test_scanner.py b/tests/integration/core/test_scanner.py index a9a467f..c1a8692 100644 --- a/tests/integration/core/test_scanner.py +++ b/tests/integration/core/test_scanner.py @@ -4,6 +4,7 @@ from sbws.util.config import get_config from sbws.lib.relaylist import RelayList from sbws.lib.destination import DestinationList from sbws.lib.resultdump import ResultSuccess +import logging def assert_within(value, target, radius): @@ -40,7 +41,8 @@ def get_everything_to_measure(dotsbws, cont, parser): def test_measure_relay_with_maxadvertisedbandwidth( - persistent_launch_tor, parser, persistent_empty_dotsbws): + persistent_launch_tor, parser, persistent_empty_dotsbws, caplog): + caplog.set_level(logging.DEBUG) cont = persistent_launch_tor dotsbws = persistent_empty_dotsbws.name d = get_everything_to_measure(dotsbws, cont, parser) @@ -57,10 +59,13 @@ def test_measure_relay_with_maxadvertisedbandwidth( result = result[0] assert isinstance(result, ResultSuccess) one_mbyte = 1 * 1024 * 1024 - allowed_error = 5 # bytes per second dls = result.downloads for dl in dls: - assert_within(dl['amount'] / dl['duration'], one_mbyte, allowed_error) + # This relay has MaxAdvertisedBandwidth set, but should not be limited + # to just 1 Mbyte. Assume and assert that all downloads where at least + # more than 10% faster than 1 MBps + assert dl['amount'] / dl['duration'] > one_mbyte * 1.1 + assert result.relay_average_bandwidth == one_mbyte def test_measure_relay_with_relaybandwidthrate(

1 0

[sbws/master] Make sure we only refresh our list of relays once
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 5df7396760a0703e948e97c74e2179b86aef6a90 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Fri Jun 22 09:53:17 2018 -0400 Make sure we only refresh our list of relays once GH: ref #205 --- sbws/lib/relaylist.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index f87d3d0..8b19a7c 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -8,6 +8,7 @@ import random import time import logging from sbws.globals import resolve +from threading import Lock log = logging.getLogger(__name__) @@ -124,12 +125,26 @@ class RelayList: def __init__(self, args, conf, controller): self._controller = controller self.rng = random.SystemRandom() + self._refresh_lock = Lock() self._refresh() + def _need_refresh(self): + return time.time() >= self._last_refresh + self.REFRESH_INTERVAL + @property def relays(self): - if time.time() >= self._last_refresh + self.REFRESH_INTERVAL: - self._refresh() + # See if we can get the list of relays without having to do a refresh, + # which is expensive and blocks other threads + if self._need_refresh(): + # Whelp we couldn't just get the list of relays because the list is + # stale. Wait for the lock so we can refresh it. + with self._refresh_lock: + # Now we have the lock ... but wait! Maybe someone else already + # did the refreshing. So check if it still needs refreshing. If + # not, we can do nothing. + if self._need_refresh(): + self._refresh() + assert not self._need_refresh() return self._relays @property

1 0