tor-commits September 2018

tor-commits@lists.torproject.org

17 participants
3230 discussions

[sbws/master] Add tests for sbws and torflow scaling
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit f5cf65ed59ec4f2ed860d84da14c112604e8e773 Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 16:18:38 2018 +0000 Add tests for sbws and torflow scaling --- tests/unit/lib/test_v3bwfile.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index aed989f..b0001c1 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -146,5 +146,11 @@ def test_sbws_scale(datadir): def test_torflow_scale(datadir): results = load_result_file(str(datadir.join("results.txt"))) - v3bwfile = V3BWFile.from_results(results, scaling_method=SBWS_SCALING) - assert v3bwfile.bw_lines[0].bw == 8 + v3bwfile = V3BWFile.from_results(results, scaling_method=TORFLOW_SCALING) + assert v3bwfile.bw_lines[0].bw == 1000 + v3bwfile = V3BWFile.from_results(results, scaling_method=TORFLOW_SCALING, + torflow_cap=0.0001) + assert v3bwfile.bw_lines[0].bw == 1000 + v3bwfile = V3BWFile.from_results(results, scaling_method=TORFLOW_SCALING, + torflow_cap=1, torflow_round_digs=0) + assert v3bwfile.bw_lines[0].bw == 524

1 0

[sbws/master] Use new methods to obtain bw line from results
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit f6cf9bfe808117c3d7d5fd468e6dcd0919e52dc8 Author: juga0 <juga(a)riseup.net> Date: Wed Sep 5 12:25:15 2018 +0000 Use new methods to obtain bw line from results --- sbws/lib/v3bwfile.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index f3e9bc9..b67b329 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -275,18 +275,27 @@ class V3BWLine(object): kwargs.update(cls.result_types_from_results(results)) # useful args for scaling if success_results: + min_num_success_results = cls.min_num_results(success_results, + min_num) + if not min_num_success_results: + return None + results_away = \ + cls.results_away_each_other(min_num_success_results, secs_away) + results_recent = cls.results_recent_than(results_away, secs_recent) + if not results_recent: + return None # the most recent should be the last kwargs['desc_avg_bw_bs'] = \ - success_results[-1].relay_average_bandwidth - kwargs['rtt'] = cls.rtt_from_results(success_results) - bw = cls.bw_bs_median_from_results(success_results) - kwargs['bw_bs_mean'] = cls.bw_bs_mean_from_results(success_results) + results_recent[-1].relay_average_bandwidth + kwargs['rtt'] = cls.rtt_from_results(results_recent) + bw = cls.bw_bs_median_from_results(results_recent) + kwargs['bw_bs_mean'] = cls.bw_bs_mean_from_results(results_recent) kwargs['bw_bs_median'] = cls.bw_bs_median_from_results( - success_results) + results_recent) kwargs['desc_obs_bw_bs_last'] = \ - cls.desc_obs_bw_bs_last_from_results(success_results) + cls.desc_obs_bw_bs_last_from_results(results_recent) kwargs['desc_obs_bw_bs_mean'] = \ - cls.desc_obs_bw_bs_mean_from_results(success_results) + cls.desc_obs_bw_bs_mean_from_results(results_recent) bwl = cls(node_id, bw, **kwargs) return bwl return None

1 0

[sbws/master] Implement methods to filter results
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 0501e202073e5c70b72e589b9d17c3a7edf4e48f Author: juga0 <juga(a)riseup.net> Date: Wed Sep 5 12:23:30 2018 +0000 Implement methods to filter results * to have at least min num results * to have results that are more recent that x secs * to have results that are away from each other x secs --- sbws/lib/v3bwfile.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index fcc80d1..f3e9bc9 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -313,6 +313,33 @@ class V3BWLine(object): return bw_line @staticmethod + def min_num_results(results, min_num=0): + if len(results) > min_num: + return results + return None + + @staticmethod + def results_away_each_other(results, secs_away=None): + if secs_away is None or len(results) < 2: + return results + # the last one should be the most recent + results_away = [results[-1]] + # iterate over the rest of the results in reverse order + for r in reversed(results[:-1]): + if abs(results_away[0].time - r.time) > secs_away: + results_away.insert(0, r) + return results_away + + @staticmethod + def results_recent_than(results, secs_recent=None): + if secs_recent is None: + return results + results_recent = filter( + lambda x: (now_unixts() - x.time) < secs_recent, + results) + return list(results_recent) + + @staticmethod def bw_bs_median_from_results(results): return max(round(median([dl['amount'] / dl['duration'] for r in results for dl in r.downloads])), 1)

1 0

[sbws/master] Add function to obtain current unix timestamp
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 953e502271f7373c7476fade92f9464e5bdb4e3b Author: juga0 <juga(a)riseup.net> Date: Wed Sep 5 12:12:54 2018 +0000 Add function to obtain current unix timestamp --- sbws/util/timestamp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sbws/util/timestamp.py b/sbws/util/timestamp.py index d4629a3..eca78c5 100644 --- a/sbws/util/timestamp.py +++ b/sbws/util/timestamp.py @@ -42,6 +42,10 @@ def unixts_to_isodt_str(unixts): return dt_obj_to_isodt_str(unixts_to_dt_obj(unixts)) +def now_unixts(): + return datetime.utcnow().timestamp() + + def now_isodt_str(): """Return datetime now as ISO 8601 string in UTC time zone.""" return dt_obj_to_isodt_str(datetime.utcnow())

1 0

[sbws/master] Change Torflow scaling to use mean obs bw
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit ecac09767067538e4e1197d6f63542fe0b8ffb53 Author: juga0 <juga(a)riseup.net> Date: Wed Sep 5 12:30:54 2018 +0000 Change Torflow scaling to use mean obs bw --- sbws/lib/v3bwfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 5567c5c..a7a4991 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -566,7 +566,7 @@ class V3BWFile(object): 'allowed', (1 - accuracy_ratio) * 100, margin * 100) @staticmethod - def bw_torflow_scale(bw_lines, desc_obs_bws=TORFLOW_OBS_LAST, + def bw_torflow_scale(bw_lines, desc_obs_bws=TORFLOW_OBS_MEAN, cap=TORFLOW_BW_MARGIN, num_round_dig=TORFLOW_ROUND_DIG, reverse=False): """

1 0

[sbws/master] Add args to filter results when creating bw file
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 4eae70b5da1f755a113bb991bbca16a98833d099 Author: juga0 <juga(a)riseup.net> Date: Wed Sep 5 12:25:47 2018 +0000 Add args to filter results when creating bw file --- sbws/lib/v3bwfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index b67b329..66a865b 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -445,6 +445,7 @@ class V3BWFile(object): scaling_method=None, torflow_obs=TORFLOW_OBS_LAST, torflow_cap=TORFLOW_BW_MARGIN, torflow_round_digs=TORFLOW_ROUND_DIG, + secs_recent=None, secs_away=None, min_num=0, reverse=False): """Create V3BWFile class from sbws Results.

1 0

[sbws/master] Add forgotten parenthesis
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit d09dc0e505aa5ab22c24e729225a83c1228337a7 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 18:28:16 2018 +0000 Add forgotten parenthesis --- sbws/core/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index b937c5b..3c36915 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -82,7 +82,7 @@ def main(args, conf): bw_file = V3BWFile.from_results(results, state_fpath, args.scale_constant, scaling_method, torflow_cap=args.torflow_bw_margin, - torflow_round_digs=args.torflow_round_digs, + torflow_round_digs=args.torflow_round_digs) output = args.output or \ conf.getpath('paths', 'v3bw_fname').format(now_fname()) bw_file.write(output)

1 0

[sbws/master] Add changelog entry about rounding bw
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 95003b2236ebd64c908e36b8651e3be15a243df5 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 18:15:44 2018 +0000 Add changelog entry about rounding bw --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad2cab3..c1c2bd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Implement Torflow scaling (#27108) - Create methods to easy graph generation and obtain statistics to compare with current torflow results.(#27688) +- Implement rounding bw in bandwidth files to 2 insignificant digits(#27337) ### Fixed

1 0

[sbws/master] Import constants to generate bw files
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 42e439af691f4f6df6592133a51fd24df69d1018 Author: juga0 <juga(a)riseup.net> Date: Wed Sep 5 12:42:52 2018 +0000 Import constants to generate bw files filtering results --- sbws/core/generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 3c36915..4f80bb9 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -1,5 +1,6 @@ from sbws.globals import (fail_hard, SBWS_SCALE_CONSTANT, TORFLOW_SCALING, - SBWS_SCALING, TORFLOW_BW_MARGIN, TORFLOW_ROUND_DIG) + SBWS_SCALING, TORFLOW_BW_MARGIN, TORFLOW_ROUND_DIG, + DAY_SECS, NUM_MIN_RESULTS) from sbws.lib.v3bwfile import V3BWFile from sbws.lib.resultdump import load_recent_results_in_datadir from argparse import ArgumentDefaultsHelpFormatter

1 0

[sbws/master] Add cli args to filter results
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 52488eae1a22e112f1317241ca6d40ab17103c2e Author: juga0 <juga(a)riseup.net> Date: Wed Sep 5 12:43:21 2018 +0000 Add cli args to filter results when generating bw files --- sbws/core/generate.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 4f80bb9..20ccbfe 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -49,6 +49,16 @@ def gen_parser(sub): type=int, help="Number of most significant digits to round bw " "when scaling as Torflow. (Default: 3)") + p.add_argument('-p', '--secs-recent', default=None, + help="How many secs in the past are results being " + "still considered. Note this value will supersede " + "data_period in the configuration. (Default: None)") + p.add_argument('-a', '--secs-away', default=DAY_SECS, + help="How many secs results have to be away from each " + "other. (Default: 86400 - one day -)") + p.add_argument('-n', '--min-num', default=NUM_MIN_RESULTS, + help="Mininum number of a results to consider them." + "(Default: 2)") def main(args, conf):

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits September 2018