September 2018 - tor-commits - lists.torproject.org

[sbws/master] Add new constants for scaling
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit d3a94d02420239e5b35073784077017c22d25220 Author: juga0 <juga(a)riseup.net> Date: Thu Aug 30 12:01:15 2018 +0000 Add new constants for scaling --- sbws/globals.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sbws/globals.py b/sbws/globals.py index daa0a55..5e7de85 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -29,7 +29,15 @@ DEFAULT_LOG_CONFIG_PATH = os.path.join(PKG_DIR, 'config.log.default.ini') USER_CONFIG_PATH = os.path.join(os.path.expanduser('~'), '.sbws.ini') SOCKET_TIMEOUT = 60 # seconds -SCALE_CONSTANT = 7500 + +SBWS_SCALE_CONSTANT = 7500 +TORFLOW_SCALING = 1 +SBWS_SCALING = 2 +TORFLOW_BW_MARGIN = 0.05 +TORFLOW_OBS_LAST = 0 +TORFLOW_OBS_MEAN = 1 +TORFLOW_OBS_DECAYING = 3 + BW_LINE_SIZE = 510

1 0

[sbws/master] Change sbws scaling constant name
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 87b3861e6faa8f79527bf2eefbddeca1338a5091 Author: juga0 <juga(a)riseup.net> Date: Thu Aug 30 12:56:11 2018 +0000 Change sbws scaling constant name --- sbws/core/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 777b05e..fe277eb 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -1,4 +1,4 @@ -from sbws.globals import (fail_hard, SCALE_CONSTANT) +from sbws.globals import (fail_hard, SBWS_SCALE_CONSTANT) from sbws.lib.v3bwfile import V3BWFile from sbws.lib.resultdump import load_recent_results_in_datadir from argparse import ArgumentDefaultsHelpFormatter @@ -28,7 +28,7 @@ def gen_parser(sub): # time, torflow happened to generate output that averaged to 7500 bw units # per relay. We wanted the ability to try to be like torflow. See # https://lists.torproject.org/pipermail/tor-dev/2018-March/013049.html - p.add_argument('--scale-constant', default=SCALE_CONSTANT, type=int, + p.add_argument('--scale-constant', default=SBWS_SCALE_CONSTANT, type=int, help='When scaling bw weights, scale them using this const ' 'multiplied by the number of measured relays') p.add_argument('--scale', action='store_true',

1 0

[sbws/master] Use observed bw methods to init V3BWLine
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit e64ecd47e9ddd1e5eaf68a608a4989e1554626b3 Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 10:59:09 2018 +0000 Use observed bw methods to init V3BWLine --- sbws/lib/v3bwfile.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 28be1a9..4756e5e 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -264,6 +264,10 @@ class V3BWLine(object): kwargs['bw_bs_mean'] = cls.bw_bs_mean_from_results(success_results) kwargs['bw_bs_median'] = cls.bw_bs_median_from_results( success_results) + kwargs['desc_obs_bw_bs_last'] = \ + cls.desc_obs_bw_bs_last_from_results(success_results) + kwargs['desc_obs_bw_bs_mean'] = \ + cls.desc_obs_bw_bs_mean_from_results(success_results) bwl = cls(node_id, bw, **kwargs) return bwl return None

1 0

[sbws/master] Use obs bandwidth in example test bw line
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit e6965056815a7c6dfa9af66b7480fe9229917d18 Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 11:23:06 2018 +0000 Use obs bandwidth in example test bw line --- tests/unit/lib/test_v3bwfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index dafe015..1809e44 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -37,7 +37,8 @@ header_extra_ls = [timestamp_l, version_l, header_extra_str = LINE_SEP.join(header_extra_ls) + LINE_SEP bwl_str = "bw=56 bw_bs_mean=61423 bw_bs_median=55656 "\ - "desc_avg_bw_bs=1000000000 error_circ=0 error_misc=0 error_stream=1 " \ + "desc_avg_bw_bs=1000000000 desc_obs_bw_bs_last=524288 "\ + "desc_obs_bw_bs_mean=524288 error_circ=0 error_misc=0 error_stream=1 " \ "master_key_ed25519=g+Shk00y9Md0hg1S6ptnuc/wWKbADBgdjT0Kg+TSF3s " \ "nick=A " \ "node_id=$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA rtt=456 success=1 " \

1 0

[sbws/master] Add changelog entry about easy graphs and stats
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 1b17d599726d83093a6f52110ce1dd9c36d7af11 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 17:51:38 2018 +0000 Add changelog entry about easy graphs and stats --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 470c751..ad2cab3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add system physical requirements section to INSTALL (#26937) - Warn when there is not enough disk space (#26937) - Implement Torflow scaling (#27108) +- Create methods to easy graph generation and obtain statistics + to compare with current torflow results.(#27688) ### Fixed

1 0

[sbws/master] Add args to generate to scale as torflow too
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 028dc8bff6dd48a6b5a0682ba4904b70532842f0 Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 11:01:25 2018 +0000 Add args to generate to scale as torflow too --- sbws/core/generate.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 7d68be7..f996c93 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -1,4 +1,5 @@ -from sbws.globals import (fail_hard, SBWS_SCALE_CONSTANT) +from sbws.globals import (fail_hard, SBWS_SCALE_CONSTANT, TORFLOW_SCALING, + SBWS_SCALING) from sbws.lib.v3bwfile import V3BWFile from sbws.lib.resultdump import load_recent_results_in_datadir from argparse import ArgumentDefaultsHelpFormatter @@ -31,11 +32,14 @@ def gen_parser(sub): p.add_argument('--scale-constant', default=SBWS_SCALE_CONSTANT, type=int, help='When scaling bw weights, scale them using this const ' 'multiplied by the number of measured relays') - p.add_argument('--scale', action='store_true', + p.add_argument('--scale_sbws', action='store_true', help='If specified, do not use bandwidth values as they ' 'are, but scale them such that we have a budget of ' 'scale_constant * num_measured_relays = bandwidth to give ' 'out, and we do so proportionally') + p.add_argument('-t', '--scale_torflow', action='store_true', + help='If specified, do not use bandwidth values as they ' + 'are, but scale them in the way Torflow does.') def main(args, conf): @@ -46,6 +50,12 @@ def main(args, conf): fail_hard('%s does not exist', datadir) if args.scale_constant < 1: fail_hard('--scale-constant must be positive') + if args.scale_sbws: + scaling_method = SBWS_SCALING + elif args.scale_torflow: + scaling_method = TORFLOW_SCALING + else: + scaling_method = None fresh_days = conf.getint('general', 'data_period') reset_bw_ipv4_changes = conf.getboolean('general', 'reset_bw_ipv4_changes') @@ -59,7 +69,8 @@ def main(args, conf): 'ran sbws scanner recently?)') return state_fpath = conf.getpath('paths', 'state_fname') - bw_file = V3BWFile.from_results(results, state_fpath, args.scale_constant) + bw_file = V3BWFile.from_results(results, state_fpath, args.scale_constant, + scaling_method) output = args.output or \ conf.getpath('paths', 'v3bw_fname').format(now_fname()) bw_file.write(output)

1 0

[sbws/master] Add method to parse a V100 header
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 11ec984620b9ec0f12e0080e0e581c90ca8f92aa Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 11:39:31 2018 +0000 Add method to parse a V100 header --- sbws/lib/v3bwfile.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index dda4499..847c037 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -139,6 +139,17 @@ class V3BWHeader(object): assert isinstance(text, str) return self.from_lines_v110(text.split(LINE_SEP)) + @classmethod + def from_lines_v100(cls, lines): + """ + :param list lines: list of lines to parse + :returns: tuple of V3BWHeader object and non-header lines + """ + assert isinstance(lines, list) + h = cls(lines[0]) + # last line is new line + return h, lines[1:-1] + @staticmethod def generator_started_from_file(state_fpath): '''

1 0

[sbws/master] Merge pull request #252 from juga0/ticket27108_torflow_scale
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit d0ca906cde554196b01b1c218a6023539ddef31f Merge: 89d94e1 640a67e Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 17:40:06 2018 +0000 Merge pull request #252 from juga0/ticket27108_torflow_scale Ticket27108 torflow scale CHANGELOG.md | 1 + sbws/core/generate.py | 21 +++- sbws/globals.py | 10 +- sbws/lib/relaylist.py | 4 + sbws/lib/resultdump.py | 14 ++- sbws/lib/v3bwfile.py | 273 ++++++++++++++++++++++++++++++++++++++-- tests/unit/conftest.py | 59 +++++++-- tests/unit/lib/data/results.txt | 2 +- tests/unit/lib/test_v3bwfile.py | 17 ++- 9 files changed, 367 insertions(+), 34 deletions(-)

1 0

[sbws/master] Fix linter error
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 6a6c009093936559f8b18fe47d5d2269e6d7e5ee Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 17:52:48 2018 +0000 Fix linter error --- sbws/lib/v3bwfile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 5621557..844efb3 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -461,7 +461,6 @@ class V3BWFile(object): key=lambda l: l.bw) return cls(header, bw_lines) - @staticmethod def bw_kb(bw_lines, reverse=False): bw_lines_scaled = copy.deepcopy(bw_lines)

1 0

[sbws/master] Add methods to log stats
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit be90985edd6dff57d33be095cde1f3926382fd09 Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 14:07:53 2018 +0000 Add methods to log stats --- sbws/lib/v3bwfile.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 545cd77..5621557 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -724,6 +724,22 @@ class V3BWFile(object): def median_bw(self): return median([l.bw for l in self.bw_lines]) + @property + def max_bw(self): + return max([l.bw for l in self.bw_lines]) + + @property + def min_bw(self): + return min([l.bw for l in self.bw_lines]) + + @property + def info_stats(self): + if not self.bw_lines: + return + [log.info(': '.join([attr, str(getattr(self, attr))])) for attr in + ['sum_bw', 'mean_bw', 'median_bw', 'num', + 'max_bw', 'min_bw']] + def bw_line_for_node_id(self, node_id): """Returns the bandwidth line for a given node fingerprint.

1 0