June 2020 - tor-commits - lists.torproject.org

[sbws/master] fix: v3bwfile: linter error with new flake version
by juga＠torproject.org 29 Jun '20

29 Jun '20

commit ce0449a2efa4e341abb2bb985972423f878f0ef6 Author: juga0 <juga(a)riseup.net> Date: Wed May 20 15:39:46 2020 +0000 fix: v3bwfile: linter error with new flake version --- sbws/lib/bwfile_health.py | 2 ++ sbws/lib/v3bwfile.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/sbws/lib/bwfile_health.py b/sbws/lib/bwfile_health.py index 793e08e..95d37ee 100644 --- a/sbws/lib/bwfile_health.py +++ b/sbws/lib/bwfile_health.py @@ -1,4 +1,6 @@ """Expected bandwidth file values for KeyValues.""" +# flake8: noqa: E741 +# (E741 ambiguous variable name), when using l. import logging from stem import descriptor diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index a585bb7..071dc94 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- """Classes and functions that create the bandwidth measurements document (v3bw) used by bandwidth authorities.""" +# flake8: noqa: E741 +# (E741 ambiguous variable name), when using l. import copy import logging

1 0

[sbws/master] Merge branch 'maint-1.1_bug33871_scaling_03' into 'maint-1.1'
by juga＠torproject.org 29 Jun '20

29 Jun '20

commit 02861e103119b17821e78fcacbcfdf24f568ec69 Merge: 3033421 ce0449a Author: Georg Koppen <gk(a)torproject.org> Date: Mon Jun 15 07:14:43 2020 +0000 Merge branch 'maint-1.1_bug33871_scaling_03' into 'maint-1.1' Maint 1.1 bug33871 scaling 03 See merge request tpo/network-health/sbws!9 docs/source/index.rst | 1 + docs/source/torflow_aggr.rst | 429 +++++++++++++++++++++ sbws/lib/bwfile_health.py | 2 + sbws/lib/scaling.py | 19 + sbws/lib/v3bwfile.py | 386 ++++++++---------- tests/conftest.py | 1 + tests/unit/lib/data/results_0_consensus_bw.txt | 1 + tests/unit/lib/data/results_no_consensus_bw.txt | 1 + tests/unit/lib/data/results_no_desc_bw_avg.txt | 1 + tests/unit/lib/data/results_no_desc_bw_avg_obs.txt | 1 + tests/unit/lib/data/results_no_desc_bw_obs.txt | 1 + tests/unit/lib/test_scaling.py | 13 + tests/unit/lib/test_v3bwfile.py | 49 ++- 13 files changed, 680 insertions(+), 225 deletions(-)

1 0

[sbws/master] fix: v3bwfile: Warn about None bandwidth values
by juga＠torproject.org 29 Jun '20

29 Jun '20

commit d4929161a1284b8d1a28abc4ba6d09af61d68853 Author: juga0 <juga(a)riseup.net> Date: Sun May 17 09:46:45 2020 +0000 fix: v3bwfile: Warn about None bandwidth values since they are probably due a bug. --- sbws/lib/v3bwfile.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 8c5b9a9..46e6357 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -869,6 +869,7 @@ class V3BWLine(object): for r in reversed(results): if r.relay_average_bandwidth is not None: return r.relay_average_bandwidth + log.warning("Descriptor average bandwidth is None.") return None @staticmethod @@ -877,6 +878,7 @@ class V3BWLine(object): for r in reversed(results): if r.relay_burst_bandwidth is not None: return r.relay_burst_bandwidth + log.warning("Descriptor burst bandwidth is None.") return None @staticmethod @@ -885,6 +887,7 @@ class V3BWLine(object): for r in reversed(results): if r.consensus_bandwidth is not None: return r.consensus_bandwidth + log.warning("Consensus bandwidth is None.") return None @staticmethod @@ -893,6 +896,7 @@ class V3BWLine(object): for r in reversed(results): if r.consensus_bandwidth_is_unmeasured is not None: return r.consensus_bandwidth_is_unmeasured + log.warning("Consensus bandwidth is unmeasured is None.") return None @staticmethod @@ -912,6 +916,7 @@ class V3BWLine(object): for r in reversed(results): if r.relay_observed_bandwidth is not None: return r.relay_observed_bandwidth + log.warning("Descriptor observed bandwidth is None.") return None @property

1 0

[sbws/master] chg: v3bwfile: Calculate filtered bandwidth
by juga＠torproject.org 29 Jun '20

29 Jun '20

commit c030731ec25adeff750c8c00415b87521a10d581 Author: juga0 <juga(a)riseup.net> Date: Sat May 16 17:55:50 2020 +0000 chg: v3bwfile: Calculate filtered bandwidth for each relay, calculate the filtered mean for all relays and calculate the filtered ratio for each relay. --- sbws/lib/v3bwfile.py | 14 +++++++++++--- tests/unit/lib/test_scaling.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 245ffa6..9a303be 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -15,6 +15,7 @@ from sbws.globals import (SPEC_VERSION, BW_LINE_SIZE, SBWS_SCALE_CONSTANT, TORFLOW_SCALING, SBWS_SCALING, TORFLOW_BW_MARGIN, TORFLOW_OBS_LAST, TORFLOW_OBS_MEAN, PROP276_ROUND_DIG, MIN_REPORT, MAX_BW_DIFF_PERC) +from sbws.lib import scaling from sbws.lib.resultdump import ResultSuccess, _ResultType from sbws.util.filelock import DirectoryLock from sbws.util.timestamp import (now_isodt_str, unixts_to_isodt_str, @@ -631,8 +632,10 @@ class V3BWLine(object): assert node_id.startswith('$') self.node_id = node_id self.bw = bw + # For now, we do not want to add ``bw_filt`` to the bandwidth file, + # therefore it is set here but not added to ``BWLINE_KEYS_V1``. [setattr(self, k, v) for k, v in kwargs.items() - if k in BWLINE_KEYS_V1] + if k in BWLINE_KEYS_V1 + ["bw_filt"]] def __str__(self): return self.bw_strv1 @@ -762,7 +765,11 @@ class V3BWLine(object): if rtt: kwargs['rtt'] = rtt bw = cls.bw_median_from_results(results_recent) + # XXX: all the class functions could use the bw_measurements instead of + # obtaining them each time or use a class Measurements. + bw_measurements = scaling.bw_measurements_from_results(results_recent) kwargs['bw_mean'] = cls.bw_mean_from_results(results_recent) + kwargs['bw_filt'] = scaling.bw_filt(bw_measurements) kwargs['bw_median'] = cls.bw_median_from_results( results_recent) kwargs['desc_bw_avg'] = \ @@ -1311,7 +1318,8 @@ class V3BWFile(object): # mean (Torflow's strm_avg) mu = mean([l.bw_mean for l in bw_lines]) # filtered mean (Torflow's filt_avg) - muf = mean([max(l.bw_mean, mu) for l in bw_lines]) + muf = mean([l.bw_filt for l in bw_lines]) + # bw sum (Torflow's tot_net_bw or tot_sbw) sum_bw = sum([l.bw_mean for l in bw_lines]) # Torflow's clipping @@ -1369,7 +1377,7 @@ class V3BWFile(object): min_bandwidth = min(desc_bw, l.consensus_bandwidth) # Torflow's scaling ratio_stream = l.bw_mean / mu - ratio_stream_filtered = max(l.bw_mean, mu) / muf + ratio_stream_filtered = l.bw_filt / muf ratio = max(ratio_stream, ratio_stream_filtered) bw_scaled = ratio * min_bandwidth # round and convert to KB diff --git a/tests/unit/lib/test_scaling.py b/tests/unit/lib/test_scaling.py new file mode 100644 index 0000000..06fb71b --- /dev/null +++ b/tests/unit/lib/test_scaling.py @@ -0,0 +1,13 @@ +"""Unit tests for scaling.py.""" + +from sbws.lib import scaling + + +def test_bw_filt(): + bw_measurements = [ + 96700.00922329757, 70311.63051659254, 45531.743347556374, + 38913.97025485627, 55656.332364676025 + ] + fb = scaling.bw_filt(bw_measurements) + # This is greater than the mean, that is 61422.73714139576 + assert fb == 83505.81986994506

1 0

[sbws/master] chg: v3bwfile: Stop making mean minimum 1
by juga＠torproject.org 29 Jun '20

29 Jun '20

commit 88eb7ef53c77d1d377125effd088f4cd41f59235 Author: juga0 <juga(a)riseup.net> Date: Tue May 19 07:59:28 2020 +0000 chg: v3bwfile: Stop making mean minimum 1 --- sbws/lib/v3bwfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 9a303be..8c5b9a9 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -902,7 +902,8 @@ class V3BWLine(object): if r.relay_observed_bandwidth is not None: desc_bw_obs_ls.append(r.relay_observed_bandwidth) if desc_bw_obs_ls: - return max(round(mean(desc_bw_obs_ls)), 1) + return round(mean(desc_bw_obs_ls)) + log.warning("Descriptor observed bandwidth is None.") return None @staticmethod

1 0

[sbws/master] chg: scaling: Add filtered bandwidth function
by juga＠torproject.org 29 Jun '20

29 Jun '20

commit 5159662a50b6ae97588e87423d6c56a9014ff91f Author: juga0 <juga(a)riseup.net> Date: Sat May 16 17:07:19 2020 +0000 chg: scaling: Add filtered bandwidth function to calculate the filtered bandwidth for each relay. --- sbws/lib/scaling.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sbws/lib/scaling.py b/sbws/lib/scaling.py new file mode 100644 index 0000000..0fc3f84 --- /dev/null +++ b/sbws/lib/scaling.py @@ -0,0 +1,19 @@ +from statistics import mean + + +def bw_measurements_from_results(results): + return [ + dl['amount'] / dl['duration'] + for r in results for dl in r.downloads + ] + + +def bw_filt(bw_measurements): + """Filtered bandwidth for a relay. + + It is the equivalent to Torflow's ``filt_sbw``. + ``mu`` in this function is the equivalent to Torflow's ``sbw``. + """ + mu = mean(bw_measurements) + bws_gte_mean = filter(lambda bw: bw >= mu, bw_measurements) + return mean(bws_gte_mean)

1 0

[sbws/maint-1.1] chg: v3bwfile: Obtain consensus values from last consensus
by juga＠torproject.org 29 Jun '20

29 Jun '20

commit 6e1f57c6770d7a09facf0414029a6f8cb32e35c7 Author: juga0 <juga(a)riseup.net> Date: Sun May 17 13:12:37 2020 +0000 chg: v3bwfile: Obtain consensus values from last consensus --- sbws/lib/v3bwfile.py | 52 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 30e8a8b..e3a4cf2 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -642,7 +642,7 @@ class V3BWLine(object): @classmethod def from_results(cls, results, secs_recent=None, secs_away=None, - min_num=0): + min_num=0, router_statuses_d=None): """Convert sbws results to relays' Bandwidth Lines ``bs`` stands for Bytes/seconds @@ -756,6 +756,23 @@ class V3BWLine(object): 'recent_measurements_excluded_few_count' return (cls(node_id, 1, **kwargs), exclusion_reason) + # Use the last consensus if available, since the results' consensus + # values come from the moment the measurement was made. + if router_statuses_d and node_id in router_statuses_d: + consensus_bandwidth = \ + router_statuses_d[node_id].bandwidth * 1000 + consensus_bandwidth_is_unmeasured = \ + router_statuses_d[node_id].is_unmeasured + else: + consensus_bandwidth = \ + cls.consensus_bandwidth_from_results(results_recent) + consensus_bandwidth_is_unmeasured = \ + cls.consensus_bandwidth_is_unmeasured_from_results( + results_recent) + # If there is no last observed bandwidth, there won't be mean either. + desc_bw_obs_last = \ + cls.desc_bw_obs_last_from_results(results_recent) + # For any line not excluded, do not include vote and unmeasured # KeyValues del kwargs['vote'] @@ -776,15 +793,13 @@ class V3BWLine(object): cls.desc_bw_avg_from_results(results_recent) kwargs['desc_bw_bur'] = \ cls.desc_bw_bur_from_results(results_recent) - kwargs['consensus_bandwidth'] = \ - cls.consensus_bandwidth_from_results(results_recent) + kwargs['consensus_bandwidth'] = consensus_bandwidth kwargs['consensus_bandwidth_is_unmeasured'] = \ - cls.consensus_bandwidth_is_unmeasured_from_results( - results_recent) - kwargs['desc_bw_obs_last'] = \ - cls.desc_bw_obs_last_from_results(results_recent) + consensus_bandwidth_is_unmeasured + kwargs['desc_bw_obs_last'] = desc_bw_obs_last kwargs['desc_bw_obs_mean'] = \ cls.desc_bw_obs_mean_from_results(results_recent) + bwl = cls(node_id, bw, **kwargs) return bwl, None @@ -997,8 +1012,10 @@ class V3BWFile(object): destinations_countries, state_fpath) bw_lines_raw = [] bw_lines_excluded = [] - number_consensus_relays = cls.read_number_consensus_relays( - consensus_path) + router_statuses_d = cls.read_router_statuses(consensus_path) + # XXX: Use router_statuses_d to not parse again the file. + number_consensus_relays = \ + cls.read_number_consensus_relays(consensus_path) state = State(state_fpath) # Create a dictionary with the number of relays excluded by any of the @@ -1012,7 +1029,8 @@ class V3BWFile(object): for fp, values in results.items(): # log.debug("Relay fp %s", fp) line, reason = V3BWLine.from_results(values, secs_recent, - secs_away, min_num) + secs_away, min_num, + router_statuses_d) # If there is no reason it means the line will not be excluded. if not reason: bw_lines_raw.append(line) @@ -1417,6 +1435,20 @@ class V3BWFile(object): log.debug("Number of relays in the network %s", num) return num + @staticmethod + def read_router_statuses(consensus_path): + """Read the router statuses from the cached consensus file.""" + router_statuses_d = None + try: + router_statuses_d = dict([ + (r.fingerprint, r) + for r in parse_file(consensus_path) + ]) + except (FileNotFoundError, AttributeError): + log.warning("It is not possible to obtain the last consensus" + "cached file %s.", consensus_path) + return router_statuses_d + @staticmethod def measured_progress_stats(num_bw_lines, number_consensus_relays, min_perc_reached_before):

1 0

[sbws/maint-1.1] chg: v3bwfile: Calculate hlimit from scaled sum bw
by juga＠torproject.org 29 Jun '20

29 Jun '20

commit 5d0e63cb62ddae416671abd2e4b604e83c34cd1a Author: juga0 <juga(a)riseup.net> Date: Sun May 17 14:29:54 2020 +0000 chg: v3bwfile: Calculate hlimit from scaled sum bw instead of bw before scaling. Tests have finally correct value. For 1 result, only when the cap is 1, the value will be equal to the rounded bw because the cap does not limit it. --- sbws/lib/v3bwfile.py | 44 +++++++++++++++++++++++++++++------------ tests/unit/lib/test_v3bwfile.py | 16 +++++++-------- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index e3a4cf2..4ebfa68 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -1060,8 +1060,10 @@ class V3BWFile(object): cls.warn_if_not_accurate_enough(bw_lines, scale_constant) # log.debug(bw_lines[-1]) elif scaling_method == TORFLOW_SCALING: - bw_lines = cls.bw_torflow_scale(bw_lines_raw, torflow_obs, - torflow_cap, round_digs) + bw_lines = cls.bw_torflow_scale( + bw_lines_raw, torflow_obs, torflow_cap, round_digs, + router_statuses_d=router_statuses_d + ) # log.debug(bw_lines[-1]) # Update the header and log the progress. min_perc = cls.update_progress( @@ -1186,7 +1188,8 @@ class V3BWFile(object): @staticmethod def bw_torflow_scale(bw_lines, desc_bw_obs_type=TORFLOW_OBS_MEAN, cap=TORFLOW_BW_MARGIN, - num_round_dig=PROP276_ROUND_DIG, reverse=False): + num_round_dig=PROP276_ROUND_DIG, reverse=False, + router_statuses_d=None): """ Obtain final bandwidth measurements applying Torflow's scaling method. @@ -1344,15 +1347,12 @@ class V3BWFile(object): mu = mean([l.bw_mean for l in bw_lines]) # filtered mean (Torflow's filt_avg) muf = mean([l.bw_filt for l in bw_lines]) - - # bw sum (Torflow's tot_net_bw or tot_sbw) - sum_bw = sum([l.bw_mean for l in bw_lines]) - # Torflow's clipping - hlimit = sum_bw * cap - log.debug('sum %s', sum_bw) log.debug('mu %s', mu) log.debug('muf %s', muf) - log.debug('hlimit %s', hlimit) + + # Torflow's ``tot_net_bw``, sum of the scaled bandwidth for the relays + # that are in the last consensus + sum_bw = 0 for l in bw_lines_tf: # First, obtain the observed bandwidth, later check what to do # if it is 0 or None. @@ -1412,9 +1412,27 @@ class V3BWFile(object): ratio_stream = l.bw_mean / mu ratio_stream_filtered = l.bw_filt / muf ratio = max(ratio_stream, ratio_stream_filtered) - bw_scaled = ratio * min_bandwidth - # Cap maximum bw - bw_scaled = min(hlimit, bw_scaled) + + # Assign it to an attribute, so it's not lost before capping and + # rounding + l.bw = ratio * min_bandwidth + + # If the consensus is available, sum only the bw for the relays + # that are in the consensus + if router_statuses_d: + if l.node_id.replace("$", "") in router_statuses_d: + sum_bw += l.bw + # Otherwise sum all bw, for compatibility with tests that were not + # using the consensus file. + else: + sum_bw += l.bw + + # Cap maximum bw, only possible when the ``sum_bw`` is calculated. + # Torflow's clipping + hlimit = sum_bw * cap + log.debug("sum_bw: %s, hlimit: %s", sum_bw, hlimit) + for l in bw_lines_tf: + bw_scaled = min(hlimit, l.bw) # round and convert to KB bw_new = kb_round_x_sig_dig(bw_scaled, digits=num_round_dig) # avoid 0 diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 5567a37..07f59d7 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -330,7 +330,7 @@ def test_torflow_scale(mock_consensus, datadir, tmpdir, conf): state_fpath, scaling_method=TORFLOW_SCALING, round_digs=TORFLOW_ROUND_DIG) - assert v3bwfile.bw_lines[0].bw == 3 + assert v3bwfile.bw_lines[0].bw == 6 v3bwfile = V3BWFile.from_results(results, '', '', state_fpath, scaling_method=TORFLOW_SCALING, @@ -343,13 +343,13 @@ def test_torflow_scale(mock_consensus, datadir, tmpdir, conf): scaling_method=TORFLOW_SCALING, torflow_cap=1, round_digs=TORFLOW_ROUND_DIG) - assert v3bwfile.bw_lines[0].bw == 61 + assert v3bwfile.bw_lines[0].bw == 123 v3bwfile = V3BWFile.from_results(results, '', '', state_fpath, scaling_method=TORFLOW_SCALING, torflow_cap=1, round_digs=PROP276_ROUND_DIG) - assert v3bwfile.bw_lines[0].bw == 61 + assert v3bwfile.bw_lines[0].bw == 120 def test_torflow_scale_no_desc_bw_avg(datadir, conf, caplog): @@ -357,7 +357,7 @@ def test_torflow_scale_no_desc_bw_avg(datadir, conf, caplog): results = load_result_file(str(datadir.join("results_no_desc_bw_avg.txt"))) caplog.set_level(logging.DEBUG) v3bwfile = V3BWFile.from_results(results, '', '', state_fpath) - assert v3bwfile.bw_lines[0].bw == 3 + assert v3bwfile.bw_lines[0].bw == 26 def test_torflow_scale_no_desc_bw_obs(datadir, conf, caplog): @@ -365,7 +365,7 @@ def test_torflow_scale_no_desc_bw_obs(datadir, conf, caplog): results = load_result_file(str(datadir.join("results_no_desc_bw_obs.txt"))) caplog.set_level(logging.DEBUG) v3bwfile = V3BWFile.from_results(results, '', '', state_fpath) - assert v3bwfile.bw_lines[0].bw == 3 + assert v3bwfile.bw_lines[0].bw == 30 def test_torflow_scale_no_desc_bw_avg_obs(datadir, conf, caplog): @@ -375,7 +375,7 @@ def test_torflow_scale_no_desc_bw_avg_obs(datadir, conf, caplog): ) caplog.set_level(logging.DEBUG) v3bwfile = V3BWFile.from_results(results, '', '', state_fpath) - assert v3bwfile.bw_lines[0].bw == 3 + assert v3bwfile.bw_lines[0].bw == 30 def test_torflow_scale_no_consensus_bw(datadir, conf, caplog): @@ -385,7 +385,7 @@ def test_torflow_scale_no_consensus_bw(datadir, conf, caplog): ) caplog.set_level(logging.DEBUG) v3bwfile = V3BWFile.from_results(results, '', '', state_fpath) - assert v3bwfile.bw_lines[0].bw == 3 + assert v3bwfile.bw_lines[0].bw == 26 def test_torflow_scale_0_consensus_bw(datadir, conf, caplog): @@ -393,7 +393,7 @@ def test_torflow_scale_0_consensus_bw(datadir, conf, caplog): results = load_result_file(str(datadir.join("results_0_consensus_bw.txt"))) caplog.set_level(logging.DEBUG) v3bwfile = V3BWFile.from_results(results, '', '', state_fpath) - assert v3bwfile.bw_lines[0].bw == 3 + assert v3bwfile.bw_lines[0].bw == 26 def test_results_away_each_other(datadir):

1 0

[sbws/maint-1.1] fix: v3bwfile: Remove unneeded minimum 1
by juga＠torproject.org 29 Jun '20

29 Jun '20

commit d72fe2ee33237c507f461488c7bb879301895e62 Author: juga0 <juga(a)riseup.net> Date: Wed May 20 13:23:26 2020 +0000 fix: v3bwfile: Remove unneeded minimum 1 since rounding already returns 1 as minimum. --- sbws/lib/v3bwfile.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 4ebfa68..d64712c 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -1434,9 +1434,7 @@ class V3BWFile(object): for l in bw_lines_tf: bw_scaled = min(hlimit, l.bw) # round and convert to KB - bw_new = kb_round_x_sig_dig(bw_scaled, digits=num_round_dig) - # avoid 0 - l.bw = max(bw_new, 1) + l.bw = kb_round_x_sig_dig(bw_scaled, digits=num_round_dig) return sorted(bw_lines_tf, key=lambda x: x.bw, reverse=reverse) @staticmethod

1 0

[sbws/maint-1.1] chg: v3bwfile: Percentage difference with consensus
by juga＠torproject.org 29 Jun '20

29 Jun '20

commit d4fe347c8154335049898a4e687d5cf33e974f24 Author: juga0 <juga(a)riseup.net> Date: Tue May 19 08:11:11 2020 +0000 chg: v3bwfile: Percentage difference with consensus --- sbws/lib/v3bwfile.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index d64712c..263a285 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -1077,7 +1077,9 @@ class V3BWFile(object): bw_lines = cls.bw_kb(bw_lines_raw) # log.debug(bw_lines[-1]) # Not using the result for now, just warning - cls.is_max_bw_diff_perc_reached(bw_lines, max_bw_diff_perc) + cls.is_max_bw_diff_perc_reached( + bw_lines, max_bw_diff_perc, router_statuses_d + ) header.add_time_report_half_network() f = cls(header, bw_lines + bw_lines_excluded) return f @@ -1159,18 +1161,22 @@ class V3BWFile(object): @staticmethod def is_max_bw_diff_perc_reached(bw_lines, - max_bw_diff_perc=MAX_BW_DIFF_PERC): - # Since old versions were not storing consensus bandwidth, use getattr. - sum_consensus_bw = sum([l.consensus_bandwidth for l in bw_lines - if getattr(l, 'consensus_bandwidth', None)]) + max_bw_diff_perc=MAX_BW_DIFF_PERC, + router_statuses_d=None): + if router_statuses_d: + sum_consensus_bw = sum(list(map( + lambda x: x.bandwidth * 1000, + router_statuses_d.values() + ))) + else: + sum_consensus_bw = sum([ + l.consensus_bandwidth for l in bw_lines + if getattr(l, 'consensus_bandwidth', None) + ]) # Because the scaled bandwidth is in KB, but not the stored consensus # bandwidth, multiply by 1000. - # Do not count 1 bandwidths for the relays that were excluded - # and exclude also the bw of the relays that did not stored consensus, - # since they are not included either in the sum of the consensus. - sum_bw = sum([l.bw for l in bw_lines - if getattr(l, 'consensus_bandwidth', None) - and getattr(l, 'unmeasured', 0) == 0]) * 1000 + # Do not count the bandwidths for the relays that were excluded + sum_bw = sum([l.bw for l in bw_lines if getattr(l, "vote", 1)]) * 1000 # Percentage difference diff_perc = ( abs(sum_consensus_bw - sum_bw)

1 0