commit f365d679d72f1f35e59442bade90cfda3537a508 Author: juga0 <juga@riseup.net> Date: Sun May 17 11:16:54 2020 +0000 chg: v3bwfile: Change logic obtaining min bandwidth Take either the consenus bandwidth or the descriptor bandwidth if one of them is missing, do not scale when both are missing and ignore descriptor average and burst when they are missing. --- sbws/lib/v3bwfile.py | 51 ++++++++++++++-------- tests/unit/lib/data/results_0_consensus_bw.txt | 1 + tests/unit/lib/data/results_no_consensus_bw.txt | 1 + tests/unit/lib/data/results_no_desc_bw_avg.txt | 1 + tests/unit/lib/data/results_no_desc_bw_avg_obs.txt | 1 + tests/unit/lib/data/results_no_desc_bw_obs.txt | 1 + tests/unit/lib/test_v3bwfile.py | 43 ++++++++++++++++++ 7 files changed, 82 insertions(+), 17 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 244eb5b..8c58d64 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -1154,7 +1154,8 @@ class V3BWFile(object): # Percentage difference diff_perc = ( abs(sum_consensus_bw - sum_bw) - / ((sum_consensus_bw + sum_bw) / 2) + # Avoid ZeroDivisionError + / (max(1, (sum_consensus_bw + sum_bw)) / 2) ) * 100 log.info("The difference between the total consensus bandwidth (%s)" "and the total measured bandwidth (%s) is %s%%.", @@ -1356,23 +1357,39 @@ class V3BWFile(object): # descriptors' bandwidth-observed, because that penalises new # relays. # See https://trac.torproject.org/projects/tor/ticket/8494 - if l.desc_bw_bur is not None: - # Because in previous versions results were not storing - # desc_bw_bur - desc_bw = min(desc_bw_obs, l.desc_bw_bur, l.desc_bw_avg) + # If the observed bandwidth is None, it is not possible to + # calculate the minimum with the other descriptors. + # Only in this case, take the consensus bandwidth. + # In the case that descriptor average or burst are None, + # ignore them since it must be a bug in ``Resultdump``, already + # logged in x_bw/bandwidth_x_from_results, but scale. + if desc_bw_obs is not None: + if l.desc_bw_bur is not None: + if l.desc_bw_avg is not None: + desc_bw = min( + desc_bw_obs, l.desc_bw_bur, l.desc_bw_avg + ) + else: + desc_bw = min(desc_bw_obs, l.desc_bw_bur) + else: + if l.desc_bw_avg is not None: + desc_bw = min(desc_bw_obs, l.desc_bw_avg) + else: + desc_bw = desc_bw_obs + # If the relay is unmeasured and consensus bandwidth is None or + # 0, use the descriptor bandwidth + if l.consensus_bandwidth_is_unmeasured \ + or not l.consensus_bandwidth: + min_bandwidth = desc_bw_obs + else: + min_bandwidth = min(desc_bw, l.consensus_bandwidth) + elif l.consensus_bandwidth is not None: + min_bandwidth = l.consensus_bandwidth else: - desc_bw = min(desc_bw_obs, l.desc_bw_avg) - # In previous versions results were not storing consensus_bandwidth - if l.consensus_bandwidth_is_unmeasured \ - or l.consensus_bandwidth is None: - min_bandwidth = desc_bw - # If the relay is measured, use the minimum between the descriptors - # bandwidth and the consensus bandwidth, so that - # MaxAdvertisedBandwidth limits the consensus weight - # The consensus bandwidth in a measured relay has been obtained - # doing the same calculation as here - else: - min_bandwidth = min(desc_bw, l.consensus_bandwidth) + log.warning("Can not scale relay missing descriptor and" + " consensus bandwidth.") + continue + # Torflow's scaling ratio_stream = l.bw_mean / mu ratio_stream_filtered = l.bw_filt / muf diff --git a/tests/unit/lib/data/results_0_consensus_bw.txt b/tests/unit/lib/data/results_0_consensus_bw.txt new file mode 100644 index 0000000..a4164a7 --- /dev/null +++ b/tests/unit/lib/data/results_0_consensus_bw.txt @@ -0,0 +1 @@ +{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_burst_bandwidth": 1000000000, "relay_observed_bandwidth": 524288, "relay_average_bandwidth": 500000000, "consensus _bandwidth": 0, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} diff --git a/tests/unit/lib/data/results_no_consensus_bw.txt b/tests/unit/lib/data/results_no_consensus_bw.txt new file mode 100644 index 0000000..8da43d0 --- /dev/null +++ b/tests/unit/lib/data/results_no_consensus_bw.txt @@ -0,0 +1 @@ +{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_burst_bandwidth": 1000000000, "relay_observed_bandwidth": 524288, "relay_average_bandwidth": 500000000, "consensus _bandwidth": null, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} diff --git a/tests/unit/lib/data/results_no_desc_bw_avg.txt b/tests/unit/lib/data/results_no_desc_bw_avg.txt new file mode 100644 index 0000000..8825d7d --- /dev/null +++ b/tests/unit/lib/data/results_no_desc_bw_avg.txt @@ -0,0 +1 @@ +{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_burst_bandwidth": 1000000000, "relay_observed_bandwidth": 524288, "relay_average_bandwidth": null, "consensus_band width": 600000, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} diff --git a/tests/unit/lib/data/results_no_desc_bw_avg_obs.txt b/tests/unit/lib/data/results_no_desc_bw_avg_obs.txt new file mode 100644 index 0000000..7b4cff0 --- /dev/null +++ b/tests/unit/lib/data/results_no_desc_bw_avg_obs.txt @@ -0,0 +1 @@ +{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_burst_bandwidth": 1000000000, "relay_observed_bandwidth": null, "relay_average_bandwidth": null, "consensus_bandwi dth": 600000, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} diff --git a/tests/unit/lib/data/results_no_desc_bw_obs.txt b/tests/unit/lib/data/results_no_desc_bw_obs.txt new file mode 100644 index 0000000..e1c4f2d --- /dev/null +++ b/tests/unit/lib/data/results_no_desc_bw_obs.txt @@ -0,0 +1 @@ +{"version": 4, "time": 1523887747, "circ": ["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"], "type": "success", "rtts": [0.4596822261810303, 0.44872617721557617, 0.4563450813293457, 0.44872212409973145, 0.4561030864715576, 0.4765200614929199, 0.4495084285736084, 0.45711588859558105, 0.45520496368408203, 0.4635589122772217], "fingerprint": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "scanner": "IDidntEditTheSBWSConfig", "downloads": [{"amount": 590009, "duration": 6.1014368534088135}, {"amount": 590009, "duration": 8.391342878341675}, {"amount": 321663, "duration": 7.064587831497192}, {"amount": 321663, "duration": 8.266003131866455}, {"amount": 321663, "duration": 5.779450178146362}], "dest_url": "http://y.z", "nickname": "A", "address": "111.111.111.111", "master_key_ed25519": "g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s", "relay_burst_bandwidth": 1000000000, "relay_observed_bandwidth": null, "relay_average_bandwidth": 500000000, "consensus_b andwidth": 600000, "consensus_bandwidth_is_unmeasured": false, "relay_in_recent_consensus": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_measurement_attempt": ["2018-04-07T14:09:07", "2018-04-07T14:09:07"], "relay_recent_priority_list": ["2018-04-07T14:09:07", "2018-04-07T14:09:07", "2018-04-07T14:09:07"]} diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 1028370..1b46d69 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -351,6 +351,49 @@ def test_torflow_scale(mock_consensus, datadir, tmpdir, conf): assert v3bwfile.bw_lines[0].bw == 120 +def test_torflow_scale_no_desc_bw_avg(datadir, conf, caplog): + state_fpath = conf['paths']['state_fpath'] + results = load_result_file(str(datadir.join("results_no_desc_bw_avg.txt"))) + caplog.set_level(logging.DEBUG) + v3bwfile = V3BWFile.from_results(results, '', '', state_fpath) + assert v3bwfile.bw_lines[0].bw == 520 + + +def test_torflow_scale_no_desc_bw_obs(datadir, conf, caplog): + state_fpath = conf['paths']['state_fpath'] + results = load_result_file(str(datadir.join("results_no_desc_bw_obs.txt"))) + caplog.set_level(logging.DEBUG) + v3bwfile = V3BWFile.from_results(results, '', '', state_fpath) + assert v3bwfile.bw_lines[0].bw == 600 + + +def test_torflow_scale_no_desc_bw_avg_obs(datadir, conf, caplog): + state_fpath = conf['paths']['state_fpath'] + results = load_result_file( + str(datadir.join("results_no_desc_bw_avg_obs.txt")) + ) + caplog.set_level(logging.DEBUG) + v3bwfile = V3BWFile.from_results(results, '', '', state_fpath) + assert v3bwfile.bw_lines[0].bw == 600 + + +def test_torflow_scale_no_consensus_bw(datadir, conf, caplog): + state_fpath = conf['paths']['state_fpath'] + results = load_result_file(str( + datadir.join("results_no_consensus_bw.txt")) + ) + caplog.set_level(logging.DEBUG) + v3bwfile = V3BWFile.from_results(results, '', '', state_fpath) + assert v3bwfile.bw_lines[0].bw == 520 + + +def test_torflow_scale_0_consensus_bw(datadir, conf, caplog): + state_fpath = conf['paths']['state_fpath'] + results = load_result_file(str(datadir.join("results_0_consensus_bw.txt"))) + caplog.set_level(logging.DEBUG) + v3bwfile = V3BWFile.from_results(results, '', '', state_fpath) + assert v3bwfile.bw_lines[0].bw == 520 + def test_results_away_each_other(datadir): min_num = 2 secs_away = 86400 # 1d