October 2018 - tor-commits - lists.torproject.org

[sbws/master] Add statistics in the header before scaling too
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit f0437bf02c46d2d67e3da02d0b21813a2a3943ed Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 11:05:23 2018 +0000 Add statistics in the header before scaling too It can be the case that after selecting the results to consider and before scaling, there is not any. In this case the statistics should also be written in the header. --- sbws/lib/v3bwfile.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index b1bdbff..ef0c7f9 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -491,7 +491,14 @@ class V3BWFile(object): if line is not None: bw_lines_raw.append(line) if not bw_lines_raw: - log.info("There are not enough raw results to apply any scaling.") + log.info("After applying restrictions to the raw results, " + "there is not any. Scaling can not be applied.") + if num_net_relays is not None: + statsd, success = cls.measured_progress_stats(bw_lines_raw, + num_net_relays, state_fpath) + if not success: + header.add_stats(**statsd) + bw_lines = [] return cls(header, []) if scaling_method == SBWS_SCALING: bw_lines = cls.bw_sbws_scale(bw_lines_raw, scale_constant)

1 0

[sbws/master] Merge branch 'ticket28076_rmlink_or_header'
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit cf4fe8a110de2b0bd0f495dde71e27dc041e09e9 Merge: 73c01a4 9adb58f Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 13:49:18 2018 +0000 Merge branch 'ticket28076_rmlink_or_header' CHANGELOG.md | 8 ++++++ sbws/core/generate.py | 8 ++++-- sbws/globals.py | 1 + sbws/lib/v3bwfile.py | 69 +++++++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 79 insertions(+), 7 deletions(-) diff --cc CHANGELOG.md index 90da685,7d0df92..76c66ab --- a/CHANGELOG.md +++ b/CHANGELOG.md @@@ -16,9 -16,15 +16,17 @@@ and this project adheres to [Semantic V - Move ``examples/`` to ``docs/`` (#28040) - Number of results comparison and number of results away from each other are incorrect (#28041) +- Stop removing results that are not away from some other X secs (#28103) +- Use secs-away when provided instead of data_period (#28105) + ## Added + + - Write bw file only when the percentage of measured relays is bigger than 60% + (#28062) + - When the percentage of measured relays is less than the 60%, do not include + the relays in the bandwidth file and instead include some statistics in the + header (#28076) + ## [0.8.0] - 2018-10-08 **Important changes**: diff --cc sbws/lib/v3bwfile.py index 8d0e193,f1a2d25..52eaec7 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@@ -5,8 -5,8 +5,9 @@@ import copy import logging import os +from itertools import combinations from statistics import median, mean + from stem.descriptor import parse_file from sbws import __version__ from sbws.globals import (SPEC_VERSION, BW_LINE_SIZE, SBWS_SCALE_CONSTANT,

1 0

[sbws/master] Use num relays in the net to calculate percent
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 7ad67810cb165cc6399dfbaa2537140db8109ea0 Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 12:25:16 2018 +0000 Use num relays in the net to calculate percent --- sbws/lib/v3bwfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index c6fa089..f1a2d25 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -817,7 +817,7 @@ class V3BWFile(object): statsd['num_target_relays'] = round(statsd['num_net_relays'] * MIN_REPORT / 100) statsd['perc_measured_relays'] = round(len(bw_lines) * 100 - / statsd['num_target_relays']) + / statsd['num_net_relays']) statsd['perc_measured_targed'] = MIN_REPORT if statsd['num_measured_relays'] < statsd['num_target_relays']: log.warning('The percentage of the measured relays is less than'

1 0

[sbws/master] Modify entry, no option to rm the link
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 9adb58f9fafcca9b8794581484ee52cc60a5c1ac Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 13:30:36 2018 +0000 Modify entry, no option to rm the link --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c13dd5..7d0df92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Write bw file only when the percentage of measured relays is bigger than 60% (#28062) -- Add option to remove link when the percentage of measured relays is less than - the 60% and instead include some statistics in the header (#28076) +- When the percentage of measured relays is less than the 60%, do not include + the relays in the bandwidth file and instead include some statistics in the + header (#28076) ## [0.8.0] - 2018-10-08

1 0

[sbws/master] Add test about the measured relays statistics
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 2d705ee3029cebeb2254b5011d02a94ab285ba1e Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 12:24:32 2018 +0000 Add test about the measured relays statistics --- tests/unit/lib/test_v3bwfile.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index a53d5db..0618167 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -182,3 +182,33 @@ def test_results_away_each_other(datadir): values = results["CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC"] success_results = [r for r in values if isinstance(r, ResultSuccess)] assert len(success_results) < min_num + + +def test_measured_progress_stats(datadir): + num_net_relays = 3 + bw_lines_raw = [] + statsd_exp = {'perc_measured_relays': 100, 'perc_measured_targed': 60, + 'num_net_relays': 3, 'num_target_relays': 2, + 'num_measured_relays': 3} + min_perc_reached_before = None + results = load_result_file(str(datadir.join("results_away.txt"))) + for fp, values in results.items(): + # log.debug("Relay fp %s", fp) + line = V3BWLine.from_results(values) + if line is not None: + bw_lines_raw.append(line) + assert len(bw_lines_raw) == 3 + bw_lines = V3BWFile.bw_torflow_scale(bw_lines_raw) + assert len(bw_lines) == 3 + statsd, success = V3BWFile.measured_progress_stats(bw_lines, + num_net_relays, min_perc_reached_before) + assert success + assert statsd == statsd_exp + num_net_relays = 6 + statsd, success = V3BWFile.measured_progress_stats(bw_lines, + num_net_relays, min_perc_reached_before) + assert not success + statsd_exp = {'perc_measured_relays': 50, 'perc_measured_targed': 60, + 'num_net_relays': 6, 'num_target_relays': 4, + 'num_measured_relays': 3} + assert statsd_exp == statsd

1 0

[sbws/master] Implement get method in State
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit f41f9134973b47ebd6dc72e7142348ca755e55c2 Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 10:19:38 2018 +0000 Implement get method in State --- sbws/util/state.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sbws/util/state.py b/sbws/util/state.py index ce3dbb9..58da699 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -63,6 +63,13 @@ class State: self._state = self._read() return self._state.__len__() + def get(self, key): + if not isinstance(key, str): + raise TypeError( + 'Keys must be strings. %s is a %s' % (key, type(key))) + self._state = self._read() + return self._state.get(key) + def __getitem__(self, key): if not isinstance(key, str): raise TypeError(

1 0

[sbws/master] Add method to update state from progress
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 9c99e312b3f03bcddf1ecf7d7bba5aca4e08046f Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 12:13:37 2018 +0000 Add method to update state from progress This way the method to measure progress calculate only the statistics and it is easier to test, removing the logic about the state file, header stats and number of bw lines returned to this method instead. --- sbws/lib/v3bwfile.py | 48 ++++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index ef0c7f9..225724f 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -484,6 +484,7 @@ class V3BWFile(object): header = V3BWHeader.from_results(results, state_fpath) bw_lines_raw = [] num_net_relays = cls.read_num_net_relays(consensus_path) + state = State(state_fpath) for fp, values in results.items(): # log.debug("Relay fp %s", fp) line = V3BWLine.from_results(values, secs_recent, secs_away, @@ -493,12 +494,8 @@ class V3BWFile(object): if not bw_lines_raw: log.info("After applying restrictions to the raw results, " "there is not any. Scaling can not be applied.") - if num_net_relays is not None: - statsd, success = cls.measured_progress_stats(bw_lines_raw, - num_net_relays, state_fpath) - if not success: - header.add_stats(**statsd) - bw_lines = [] + cls.update_progress(cls, bw_lines_raw, header, num_net_relays, + state) return cls(header, []) if scaling_method == SBWS_SCALING: bw_lines = cls.bw_sbws_scale(bw_lines_raw, scale_constant) @@ -508,14 +505,7 @@ class V3BWFile(object): bw_lines = cls.bw_torflow_scale(bw_lines_raw, torflow_obs, torflow_cap, torflow_round_digs) # log.debug(bw_lines[-1]) - if num_net_relays is not None: - statsd, success = cls.measured_progress_stats(bw_lines, - consensus_path, state_fpath) - # add statistics about progress only when there are not enough - # measured relays. Should some stats be added always? - if not success: - header.add_stats(**statsd) - bw_lines = [] + cls.update_progress(cls, bw_lines, header, num_net_relays, state) else: bw_lines = cls.bw_kb(bw_lines_raw) # log.debug(bw_lines[-1]) @@ -817,7 +807,8 @@ class V3BWFile(object): return num @staticmethod - def measured_progress_stats(bw_lines, num_net_relays, state_fpath): + def measured_progress_stats(bw_lines, num_net_relays, + min_perc_reached_before): """ Statistics about measurements progress, to be included in the header. @@ -835,8 +826,6 @@ class V3BWFile(object): # measured relays is not either. assert isinstance(num_net_relays, int) assert isinstance(bw_lines, list) - assert isinstance(state_fpath, str) - state = State(state_fpath) statsd = {} statsd['num_measured_relays'] = len(bw_lines) statsd['num_net_relays'] = num_net_relays @@ -846,12 +835,7 @@ class V3BWFile(object): / statsd['num_net_relays']) statsd['perc_measured_targed'] = MIN_REPORT if statsd['num_measured_relays'] < statsd['num_target_relays']: - # the min percent of measured relays is not reached, - # write None in the state file and obtain whether it was reached - # before - min_perc_reached_before = state.get('min_perc_reached') - state['min_perc_reached'] = None - # if it was reached before, warn + # if min percent was was reached before, warn # otherwise, debug if min_perc_reached_before is not None: log.warning('The percentage of the measured relays is less ' @@ -862,9 +846,6 @@ class V3BWFile(object): 'than the %s%% of the relays in the network (%s).', MIN_REPORT, statsd['num_net_relays']) return statsd, False - # write in the state file the date the min percent of measured - # relays has been reached - state['min_perc_reached'] = now_isodt_str() return statsd, True @property @@ -906,6 +887,21 @@ class V3BWFile(object): ['sum_bw', 'mean_bw', 'median_bw', 'num', 'max_bw', 'min_bw']] + def update_progress(self, bw_lines, header, num_net_relays, state): + min_perc_reached_before = state.get('min_perc_reached') + if num_net_relays is not None: + statsd, success = self.measured_progress_stats( + bw_lines, num_net_relays, min_perc_reached_before) + # add statistics about progress only when there are not enough + # measured relays. Should some stats be added always? + if not success: + header.add_stats(**statsd) + bw_lines = [] + state['min_perc_reached'] = None + else: + state['min_perc_reached'] = now_isodt_str() + return bw_lines + def bw_line_for_node_id(self, node_id): """Returns the bandwidth line for a given node fingerprint.

1 0

[sbws/master] Stop to require consensus path to be str
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 1defb776b3db0d272ed255e6fea6acd2d0befa50 Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 15:15:24 2018 +0000 Stop to require consensus path to be str And catch exception. This way the tests do not need to be modified to include a consensus path. --- sbws/lib/v3bwfile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 225724f..c35c865 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -795,11 +795,10 @@ class V3BWFile(object): def read_num_net_relays(consensus_path): """Read the number of relays in the Network from the cached consensus file.""" - assert isinstance(consensus_path, str) num = None try: num = len(list(parse_file(consensus_path))) - except FileNotFoundError: + except (FileNotFoundError, AttributeError): log.info("It is not possible to obtain statistics about the " "percentage of measured relays because the cached " "consensus file is not found.")

1 0

[sbws/master] Merge branch 'ticket28141'
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit ed1d61c98044e9ed081a7462d94ae31630b75705 Merge: 1defb77 8787741 Author: juga0 <juga(a)riseup.net> Date: Wed Oct 24 09:39:59 2018 +0000 Merge branch 'ticket28141' CHANGELOG.md | 1 + sbws/globals.py | 10 +++++++++- sbws/lib/relaylist.py | 5 ++++- 3 files changed, 14 insertions(+), 2 deletions(-)

1 0

[sbws/master] Add test for update_progress
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 65c5095a44138ea87b159c88d391d56da9800237 Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 14:33:25 2018 +0000 Add test for update_progress --- tests/unit/lib/test_v3bwfile.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 0618167..3ced812 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -9,7 +9,7 @@ from sbws.lib.resultdump import Result, load_result_file, ResultSuccess from sbws.lib.v3bwfile import (V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, KEYVALUE_SEP_V110, num_results_of_type, V3BWFile) -from sbws.util.timestamp import now_fname +from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts timestamp = 1523974147 timestamp_l = str(timestamp) @@ -200,15 +200,37 @@ def test_measured_progress_stats(datadir): assert len(bw_lines_raw) == 3 bw_lines = V3BWFile.bw_torflow_scale(bw_lines_raw) assert len(bw_lines) == 3 - statsd, success = V3BWFile.measured_progress_stats(bw_lines, - num_net_relays, min_perc_reached_before) + statsd, success = V3BWFile.measured_progress_stats( + bw_lines, num_net_relays, min_perc_reached_before) assert success assert statsd == statsd_exp num_net_relays = 6 - statsd, success = V3BWFile.measured_progress_stats(bw_lines, - num_net_relays, min_perc_reached_before) + statsd, success = V3BWFile.measured_progress_stats( + bw_lines, num_net_relays, min_perc_reached_before) assert not success statsd_exp = {'perc_measured_relays': 50, 'perc_measured_targed': 60, 'num_net_relays': 6, 'num_target_relays': 4, 'num_measured_relays': 3} assert statsd_exp == statsd + + +def test_update_progress(datadir, tmpdir): + bw_lines_raw = [] + num_net_relays = 6 + state = {} + header = V3BWHeader(str(now_unixts())) + results = load_result_file(str(datadir.join("results_away.txt"))) + for fp, values in results.items(): + # log.debug("Relay fp %s", fp) + line = V3BWLine.from_results(values) + if line is not None: + bw_lines_raw.append(line) + bwfile = V3BWFile(header, []) + bwfile.update_progress(bw_lines_raw, header, num_net_relays, state) + assert header.perc_measured_relays == '50' + assert state.get('min_perc_reached') is None + num_net_relays = 3 + header = V3BWHeader(str(now_unixts())) + bwfile.update_progress(bw_lines_raw, header, num_net_relays, state) + assert state.get('min_perc_reached') == now_isodt_str() + assert not hasattr(header, 'perc_measured_relays')

1 0