June 2018 - tor-commits - lists.torproject.org

[sbws/master] Stop giving exception when threads can not start
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit b1dcfe4a448dfe515e82d1ccdc4ea719b1551cab Author: juga0 <juga(a)riseup.net> Date: Mon Jun 18 11:12:45 2018 +0000 Stop giving exception when threads can not start instead log the error and exit. Fixes #195 --- sbws/lib/resultdump.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index 32fd1cd..977a9e5 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -11,7 +11,7 @@ from queue import Empty from datetime import datetime from datetime import timedelta from enum import Enum -from sbws.globals import RESULT_VERSION +from sbws.globals import RESULT_VERSION, fail_hard from sbws.util.filelock import DirectoryLock from sbws.lib.relaylist import Relay @@ -444,7 +444,10 @@ class ResultDump: self.data_lock = RLock() self.thread = Thread(target=self.enter) self.queue = Queue() - self.thread.start() + try: + self.thread.start() + except RuntimeError as e: + fail_hard(e) def store_result(self, result): ''' Call from ResultDump thread '''

1 0

[sbws/master] Trim bandwidth results for relays that change IP
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 4e7bc3e5b3f4f17d9a5848f21176f4c60b96c098 Author: juga0 <juga(a)riseup.net> Date: Mon Jun 18 13:22:43 2018 +0000 Trim bandwidth results for relays that change IP --- sbws/lib/resultdump.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index 32fd1cd..d64d843 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -82,7 +82,44 @@ def trim_results(fresh_days, result_dict): return out_results -def load_recent_results_in_datadir(fresh_days, datadir, success_only=False): +def trim_results_ip_changed(result_dict, ipv4=True, ipv6=False): + """When there are results for the same relay with different IPs, + create a new results' dictionary without that relay's results using an + older IP. + + :param dict result_dict: a dictionary of results + :param bool ipv4: whether to trim the results when a relay's IPv4 changes + :param bool ipv6: whether to trim the results when a relay's IPv6 changes + :returns: a new results dictionary + """ + assert isinstance(result_dict, dict) + assert ipv4 is True or ipv6 is True + new_results_dict = {} + if ipv4 is True: + for fp in result_dict.keys(): + results = result_dict[fp] + # find if the results for a relay have more than one ipv4 + # address + ipv4s = set([result.address for result in results]) + if len(ipv4s) > 1: + # keep only the results for the last ip used + # probably we should not just discard all the results for + # a relay that change address + ordered_results = sorted(results, key=lambda r: r.time) + latest_address = ordered_results[-1].address + last_ip_results = [result for result in results + if result.address == latest_address] + new_results_dict[fp] = last_ip_results + else: + new_results_dict[fp] = results + if ipv6 is True: + # Not implemented + pass + return new_results_dict + + +def load_recent_results_in_datadir(fresh_days, datadir, success_only=False, + ipv4=True, ipv6=False): ''' Given a data directory, read all results files in it that could have results in them that are still valid. Trim them, and return the valid Results as a list '''

1 0

[sbws/master] Add config options for trimming results when ip changes
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit d4d4aab79254171e73237bc0571ed3a7899331e5 Author: juga0 <juga(a)riseup.net> Date: Mon Jun 18 13:32:17 2018 +0000 Add config options for trimming results when ip changes --- sbws/config.default.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index faa0ab1..ff72097 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -23,6 +23,8 @@ http_timeout = 10 # CircuitBuildTimeout doesn't handle the case of a TLS connection to a relay # taking forever, and probably other not-yet-discovered cases. circuit_timeout = 10 +reset_bw_ipv4_changes = on +reset_bw_ipv6_changes = off [scanner] # A human-readable string with chars in a-zA-Z0-9 to identify your scanner

1 0

[sbws/master] Call trim_results_ip_changed when loading results
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit c00bf3f760d7abcafb1ab6b1a98f2ebea63f577b Author: juga0 <juga(a)riseup.net> Date: Mon Jun 18 13:31:03 2018 +0000 Call trim_results_ip_changed when loading results --- sbws/lib/resultdump.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index d64d843..2124cec 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -144,6 +144,9 @@ def load_recent_results_in_datadir(fresh_days, datadir, success_only=False, results = merge_result_dicts(results, new_results) working_day += timedelta(days=1) results = trim_results(fresh_days, results) + # in time fresh days is possible that a relay changed ip, + # if that's the case, keep only the results for the last ip + results = trim_results_ip_changed(results, ipv4, ipv6) num_res = sum([len(results[fp]) for fp in results]) if num_res == 0: log.warning('Results files that are valid not found. ' @@ -492,6 +495,9 @@ class ResultDump: self.data[fp] = [] self.data[fp].append(result) self.data = trim_results(self.fresh_days, self.data) + # we probably do not want to remove the results for a relay + # that has changed address when storing the results + # it will be already done when loading def handle_result(self, result): ''' Call from ResultDump thread. If we are shutting down, ignores

1 0

[sbws/master] Add test for trim_results_ip_changed
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit f6a0876eebddc04d9158ca2f6f2fe80132c3def9 Author: juga0 <juga(a)riseup.net> Date: Mon Jun 18 13:29:58 2018 +0000 Add test for trim_results_ip_changed --- tests/unit/lib/test_resultdump.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/unit/lib/test_resultdump.py b/tests/unit/lib/test_resultdump.py new file mode 100644 index 0000000..05a21ae --- /dev/null +++ b/tests/unit/lib/test_resultdump.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +"""Unit tests for resultdump.""" +from sbws.lib.resultdump import trim_results_ip_changed, Result, ResultSuccess + + +TIME1 = 1529232277.9028733 +TIME2 = 1529232278.9028733 +FP1 = 'A' * 40 +FP2 = 'Z' * 40 +ED25519 = 'g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s' +CIRC = [FP1, FP2] +DEST_URL = 'http://example.com/sbws.bin' +NICK = 'A' +RELAY_ip1 = '169.254.100.1' +RELAY_ip2 = '169.254.100.2' +RELAY1 = Result.Relay(FP1, NICK, RELAY_ip1, ED25519) +RELAY2 = Result.Relay(FP1, NICK, RELAY_ip2, ED25519) +RTTS = [5, 25] +DOWNLOADS = [{'duration': 4, 'amount': 40}] + +RESULTSUCCESS1 = ResultSuccess(RTTS, DOWNLOADS, RELAY1, CIRC, DEST_URL, + 'sbws', t=TIME1) +RESULTSUCCESS2 = ResultSuccess(RTTS, DOWNLOADS, RELAY2, CIRC, DEST_URL, + 'sbws', t=TIME2) +RESULTDICT = {FP1: [RESULTSUCCESS1, RESULTSUCCESS2]} + + +def test_trim_results_ip_changed(): + expected_results_dict = {FP1: [RESULTSUCCESS2]} + results_dict = trim_results_ip_changed(RESULTDICT) + assert expected_results_dict == results_dict

1 0

[sbws/master] This sleep can probably be shorter
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 3791316a251ce9f9c5fea02abdb9de3fd7aefe51 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Tue Jun 19 22:17:48 2018 -0400 This sleep can probably be shorter --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2a34348..36d9283 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ commands = bash {envtmpdir}/net/start.sh bash -c "time python3 {envtmpdir}/net/wait.py {envtmpdir}/net/{auth,relay,exit}*" bash -c "python3 {toxinidir}/scripts/tools/sbws-http-server.py --port 28888 &>/dev/null &" - sleep 30 + sleep 15 wget -O/dev/null http://127.0.0.1:28888/sbws.bin coverage run -a --rcfile={toxinidir}/.coveragerc --source=sbws -m pytest -s {toxinidir}/tests/integration -vv bash {envtmpdir}/net/stop.sh

1 0

[sbws/master] Add comment to config vars for reseting bw on IP
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit d25799f08378ed66615098946fbca100fe2df67d Author: juga0 <juga(a)riseup.net> Date: Tue Jun 19 19:34:15 2018 +0000 Add comment to config vars for reseting bw on IP changes. --- sbws/config.default.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index ff72097..d3abcda 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -23,7 +23,13 @@ http_timeout = 10 # CircuitBuildTimeout doesn't handle the case of a TLS connection to a relay # taking forever, and probably other not-yet-discovered cases. circuit_timeout = 10 +# Whether or not to reset the bandwidth measurements when the the relay's IPv4 +# changes reset_bw_ipv4_changes = on + +# Whether or not to reset the bandwidth measurements when the the relay's IPv6 +# changes. +# This is currently not implemented. reset_bw_ipv6_changes = off [scanner]

1 0

[sbws/master] Validate new config options for reseting bw
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 6a55ae5a4f3edb1af0710a253ad1d2b1883465f5 Author: juga0 <juga(a)riseup.net> Date: Tue Jun 19 19:38:38 2018 +0000 Validate new config options for reseting bw --- sbws/util/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbws/util/config.py b/sbws/util/config.py index 1425dda..2ea9e94 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -137,10 +137,15 @@ def _validate_general(conf): floats = { 'http_timeout': {'minimum': 0.0, 'maximum': None}, } + bools = { + 'reset_bw_ipv4_changes': {}, + 'reset_bw_ipv6_changes': {}, + } all_valid_keys = list(ints.keys()) + list(floats.keys()) errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl)) errors.extend(_validate_section_ints(conf, sec, ints, err_tmpl)) errors.extend(_validate_section_floats(conf, sec, floats, err_tmpl)) + errors.extend(_validate_section_bools(conf, sec, bools, err_tmpl)) return errors

1 0

[sbws/master] Improve tests for reseting bw on IP changes
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 91d9e90f099ff94e18d350697e8872a4ab0c61cc Author: juga0 <juga(a)riseup.net> Date: Tue Jun 19 20:09:51 2018 +0000 Improve tests for reseting bw on IP changes --- tests/unit/lib/test_resultdump.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/unit/lib/test_resultdump.py b/tests/unit/lib/test_resultdump.py index 05a21ae..26794cb 100644 --- a/tests/unit/lib/test_resultdump.py +++ b/tests/unit/lib/test_resultdump.py @@ -11,10 +11,10 @@ ED25519 = 'g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s' CIRC = [FP1, FP2] DEST_URL = 'http://example.com/sbws.bin' NICK = 'A' -RELAY_ip1 = '169.254.100.1' -RELAY_ip2 = '169.254.100.2' -RELAY1 = Result.Relay(FP1, NICK, RELAY_ip1, ED25519) -RELAY2 = Result.Relay(FP1, NICK, RELAY_ip2, ED25519) +RELAY_IP1 = '169.254.100.1' +RELAY_IP2 = '169.254.100.2' +RELAY1 = Result.Relay(FP1, NICK, RELAY_IP1, ED25519) +RELAY2 = Result.Relay(FP1, NICK, RELAY_IP2, ED25519) RTTS = [5, 25] DOWNLOADS = [{'duration': 4, 'amount': 40}] @@ -22,10 +22,16 @@ RESULTSUCCESS1 = ResultSuccess(RTTS, DOWNLOADS, RELAY1, CIRC, DEST_URL, 'sbws', t=TIME1) RESULTSUCCESS2 = ResultSuccess(RTTS, DOWNLOADS, RELAY2, CIRC, DEST_URL, 'sbws', t=TIME2) -RESULTDICT = {FP1: [RESULTSUCCESS1, RESULTSUCCESS2]} +RESULTDICT_IP_CHANGED = {FP1: [RESULTSUCCESS1, RESULTSUCCESS2]} +RESULTDICT_IP_NOT_CHANGED = {FP1: [RESULTSUCCESS1, RESULTSUCCESS1]} def test_trim_results_ip_changed(): expected_results_dict = {FP1: [RESULTSUCCESS2]} - results_dict = trim_results_ip_changed(RESULTDICT) + results_dict = trim_results_ip_changed(RESULTDICT_IP_CHANGED) assert expected_results_dict == results_dict + + +def test_trim_results_ip_changed_false(): + results_dict = trim_results_ip_changed(RESULTDICT_IP_CHANGED) + assert RESULTDICT_IP_NOT_CHANGED == results_dict

1 0

[sbws/master] Generate bandwidth file without relays' old IPs
by pastly＠torproject.org 26 Jun '18

26 Jun '18

commit 64a33f63f91549a427ba16c52c75dd419c6079e7 Author: juga0 <juga(a)riseup.net> Date: Mon Jun 18 13:32:58 2018 +0000 Generate bandwidth file without relays' old IPs --- sbws/core/generate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 25653b1..69c224d 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -71,8 +71,11 @@ def main(args, conf): fail_hard('--scale-constant must be positive') fresh_days = conf.getint('general', 'data_period') + reset_bw_ipv4_changes = conf.getboolean('general', 'reset_bw_ipv4_changes') + reset_bw_ipv6_changes = conf.getboolean('general', 'reset_bw_ipv6_changes') results = load_recent_results_in_datadir( - fresh_days, datadir, success_only=True) + fresh_days, datadir, success_only=True, ipv4=reset_bw_ipv4_changes, + ipv6=reset_bw_ipv6_changes) if len(results) < 1: log.warning('No recent results, so not generating anything. (Have you ' 'ran sbws scanner recently?)')

1 0