September 2018 - tor-commits - lists.torproject.org

[sbws/master] Fix linter errors
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 78706632ce71693911f09dfd72ecedcd5fa782e9 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 17:28:57 2018 +0000 Fix linter errors --- sbws/lib/v3bwfile.py | 11 +++++------ tests/unit/lib/test_v3bwfile.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 4756e5e..34866e9 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -9,9 +9,8 @@ from statistics import median, mean from sbws import __version__ from sbws.globals import (SPEC_VERSION, BW_LINE_SIZE, SBWS_SCALE_CONSTANT, - TORFLOW_SCALING, SBWS_SCALING, TORFLOW_BW_MARGIN, - TORFLOW_OBS_LAST, TORFLOW_OBS_MEAN, - fail_hard) + SBWS_SCALING, TORFLOW_BW_MARGIN, TORFLOW_SCALING, + TORFLOW_OBS_LAST, TORFLOW_OBS_MEAN) from sbws.lib.resultdump import ResultSuccess, _ResultType from sbws.util.filelock import DirectoryLock from sbws.util.timestamp import now_isodt_str, unixts_to_isodt_str @@ -412,9 +411,9 @@ class V3BWFile(object): header = V3BWHeader.from_results(results, state_fpath) bw_lines_raw = [] for fp in results.keys(): - l = V3BWLine.from_results(results[fp]) - if l is not None: - bw_lines_raw.append(l) + line = V3BWLine.from_results(results[fp]) + if line is not None: + bw_lines_raw.append(line) if not bw_lines_raw: return cls(header, []) if scaling_method == SBWS_SCALING: diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 15b148a..aed989f 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -4,7 +4,7 @@ import json import os.path from sbws import __version__ as version -from sbws.globals import SPEC_VERSION, SBWS_SCALING, TORFLOW_SCALING +from sbws.globals import SPEC_VERSION, SBWS_SCALING from sbws.lib.resultdump import Result, load_result_file from sbws.lib.v3bwfile import (V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, KEYVALUE_SEP_V110, num_results_of_type,

1 0

[sbws/master] Add changelog entry about torflow scaling
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 640a67e0377dc7b73ccab4633aa801d173ab528e Author: juga0 <juga(a)riseup.net> Date: Thu Sep 13 17:35:39 2018 +0000 Add changelog entry about torflow scaling --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79f41a3..470c751 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add system physical requirements section to INSTALL (#26937) - Warn when there is not enough disk space (#26937) +- Implement Torflow scaling (#27108) ### Fixed

1 0

[sbws/master] Use renamed attr to show mean
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 54c7ec92df16e84486f130db7ab5ccf073d83ec2 Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 10:38:06 2018 +0000 Use renamed attr to show mean --- sbws/core/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index fe277eb..7d68be7 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -63,4 +63,4 @@ def main(args, conf): output = args.output or \ conf.getpath('paths', 'v3bw_fname').format(now_fname()) bw_file.write(output) - log.info('Mean bandwidth per line: %f "KiB"', bw_file.avg_bw) + log.info('Mean bandwidth per line: %f "KiB"', bw_file.mean_bw)

1 0

[sbws/master] Implement sbws and torflow scaling in tests
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 09268e4ac45eb9308f67c2ee3918cd4057d7086b Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 11:24:16 2018 +0000 Implement sbws and torflow scaling in tests --- tests/unit/lib/test_v3bwfile.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 1809e44..15b148a 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -4,7 +4,7 @@ import json import os.path from sbws import __version__ as version -from sbws.globals import SPEC_VERSION +from sbws.globals import SPEC_VERSION, SBWS_SCALING, TORFLOW_SCALING from sbws.lib.resultdump import Result, load_result_file from sbws.lib.v3bwfile import (V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, KEYVALUE_SEP_V110, num_results_of_type, @@ -136,3 +136,15 @@ def test_from_arg_results_write_read(datadir, tmpdir, conf, args): with open(output) as fd: v3bw = fd.read() assert v3bw == str(v3bwfile) + + +def test_sbws_scale(datadir): + results = load_result_file(str(datadir.join("results.txt"))) + v3bwfile = V3BWFile.from_results(results, scaling_method=SBWS_SCALING) + assert v3bwfile.bw_lines[0].bw == 8 + + +def test_torflow_scale(datadir): + results = load_result_file(str(datadir.join("results.txt"))) + v3bwfile = V3BWFile.from_results(results, scaling_method=SBWS_SCALING) + assert v3bwfile.bw_lines[0].bw == 8

1 0

[sbws/master] Add method to parse V100 file given a path
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 9bc81f48061d131e07c14e8c949c24446198d7f8 Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 11:40:10 2018 +0000 Add method to parse V100 file given a path --- sbws/lib/v3bwfile.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 847c037..65a99a8 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -450,6 +450,18 @@ class V3BWFile(object): bw_lines = [V3BWLine.from_bw_line_v110(line) for line in lines] return cls(header, bw_lines) + @classmethod + def from_v100_fpath(cls, fpath): + log.info('Parsing bandwidth file %s', fpath) + with open(fpath) as fd: + text = fd.read() + all_lines = text.split(LINE_SEP) + header, lines = V3BWHeader.from_lines_v100(all_lines) + bw_lines = sorted([V3BWLine.from_bw_line_v110(l) for l in lines], + key=lambda l: l.bw) + return cls(header, bw_lines) + + @staticmethod def bw_kb(bw_lines, reverse=False): bw_lines_scaled = copy.deepcopy(bw_lines)

1 0

[sbws/master] Implement method to scale as Torflow
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 090507ee36a4685055a4e829fd5b4a4428aa3b63 Author: juga0 <juga(a)riseup.net> Date: Thu Aug 30 12:55:23 2018 +0000 Implement method to scale as Torflow --- sbws/lib/v3bwfile.py | 191 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 189 insertions(+), 2 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 5ce1423..28be1a9 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -481,9 +481,196 @@ class V3BWFile(object): 'allowed', (1 - accuracy_ratio) * 100, margin * 100) @staticmethod - def bw_lines_torflow(bw_lines, desc_obs_bws=TORFLOW_OBS_LAST, + def bw_torflow_scale(bw_lines, desc_obs_bws=TORFLOW_OBS_LAST, cap=TORFLOW_BW_MARGIN, reverse=False): - pass + """ + Obtain final bandwidth measurements applying Torflow's scaling + method. + + From Torflow's README.spec.txt (section 2.2):: + + In this way, the resulting network status consensus bandwidth values # NOQA + are effectively re-weighted proportional to how much faster the node # NOQA + was as compared to the rest of the network. + + The variables and steps used in Torflow: + + **strm_bw**:: + + The strm_bw field is the average (mean) of all the streams for the relay # NOQA + identified by the fingerprint field. + strm_bw = sum(bw stream x)/|n stream| + + **filt_bw**:: + + The filt_bw field is computed similarly, but only the streams equal to # NOQA + or greater than the strm_bw are counted in order to filter very slow # NOQA + streams due to slow node pairings. + + **filt_sbw and strm_sbw**:: + + for rs in RouterStats.query.filter(stats_clause).\ + options(eagerload_all('router.streams.circuit.routers')).all(): # NOQA + tot_sbw = 0 + sbw_cnt = 0 + for s in rs.router.streams: + if isinstance(s, ClosedStream): + skip = False + #for br in badrouters: + # if br != rs: + # if br.router in s.circuit.routers: + # skip = True + if not skip: + # Throw out outliers < mean + # (too much variance for stddev to filter much) + if rs.strm_closed == 1 or s.bandwidth() >= rs.sbw: + tot_sbw += s.bandwidth() + sbw_cnt += 1 + + if sbw_cnt: rs.filt_sbw = tot_sbw/sbw_cnt + else: rs.filt_sbw = None + + **filt_avg, and strm_avg**:: + + Once we have determined the most recent measurements for each node, we # NOQA + compute an average of the filt_bw fields over all nodes we have measured. # NOQA + + :: + + filt_avg = sum(map(lambda n: n.filt_bw, nodes.itervalues()))/float(len(nodes)) # NOQA + strm_avg = sum(map(lambda n: n.strm_bw, nodes.itervalues()))/float(len(nodes)) # NOQA + + **true_filt_avg and true_strm_avg**:: + + for cl in ["Guard+Exit", "Guard", "Exit", "Middle"]: + true_filt_avg[cl] = filt_avg + true_strm_avg[cl] = strm_avg + + In the non-pid case, all types of nodes get the same avg + + **n.fbw_ratio and n.fsw_ratio**:: + + for n in nodes.itervalues(): + n.fbw_ratio = n.filt_bw/true_filt_avg[n.node_class()] + n.sbw_ratio = n.strm_bw/true_strm_avg[n.node_class()] + + **n.ratio**:: + + These averages are used to produce ratios for each node by dividing the # NOQA + measured value for that node by the network average. + + :: + + # Choose the larger between sbw and fbw + if n.sbw_ratio > n.fbw_ratio: + n.ratio = n.sbw_ratio + else: + n.ratio = n.fbw_ratio + + **desc_bw**: + + It is the ``observed bandwidth`` in the descriptor, NOT the ``average + bandwidth``:: + + return Router(ns.idhex, ns.nickname, bw_observed, dead, exitpolicy, + ns.flags, ip, version, os, uptime, published, contact, rate_limited, # NOQA + ns.orhash, ns.bandwidth, extra_info_digest, ns.unmeasured) + self.desc_bw = max(bw,1) # Avoid div by 0 + + **new_bw**:: + + These ratios are then multiplied by the most recent observed descriptor # NOQA + bandwidth we have available for each node, to produce a new value for # NOQA + the network status consensus process. + + :: + + n.new_bw = n.desc_bw*n.ratio + + The descriptor observed bandwidth is multiplied by the ratio. + With empirical results this ratio is ~[0.9, 8.9] + + **Limit the bandwidth to a maximum**:: + + NODE_CAP = 0.05 + + :: + + if n.new_bw > tot_net_bw*NODE_CAP: + plog("INFO", "Clipping extremely fast "+n.node_class()+" node "+n.idhex+"="+n.nick+ # NOQA + " at "+str(100*NODE_CAP)+"% of network capacity ("+ + str(n.new_bw)+"->"+str(int(tot_net_bw*NODE_CAP))+") "+ + " pid_error="+str(n.pid_error)+ + " pid_error_sum="+str(n.pid_error_sum)) + n.new_bw = int(tot_net_bw*NODE_CAP) + + However, tot_net_bw does not seems to be updated when not using pid. + This clipping would make faster relays to all have the same value. + + All of that can be expressed as: + + .. math:: + + bwnew_i &= + max\\left( + \\frac{bw_i}{\\mu}, + min \\left( + bw_i, + bw_i \\times \\mu + \\right) + \\times + \\frac{bw}{\\sum_{i=1}^{n} + min \\left(bw_i, + bw_i \\times \\mu + \\right)} + \\right) + \\times bwdescobs_i \\ + + &= + max\\left( + \\frac{bw_i}{\\frac{\\sum_{i=1}^{n}bw_i}{n}}, + min \\left( + bw_i, + bw_i \\times \\frac{\\sum_{i=1}^{n}bw_i}{n} + \\right) + \\times + \\frac{bw}{\\sum_{i=1}^{n} + min \\left(bw_i, + bw_i \\times \\frac{\\sum_{i=1}^{n}bw_i}{n} + \\right)} + \\right) + \\times bwdescobs_i + + """ + log.info("Calculating relays' bandwidth using Torflow method.") + bw_lines_tf = copy.deepcopy(bw_lines) + # mean (Torflow's strm_avg) + mu = mean([l.bw_bs_mean for l in bw_lines]) + # filtered mean (Torflow's filt_avg) + muf = mean([min(l.bw_bs_mean, mu) for l in bw_lines]) + # bw sum (Torflow's tot_net_bw or tot_sbw) + sum_bw = sum([l.bw_bs_mean for l in bw_lines]) + # Torflow's clipping, not being applied + # hlimit = sum_bw * TORFLOW_BW_MARGIN + log.debug('sum %s', sum_bw) + log.debug('mu %s', mu) + log.debug('muf %s', muf) + # log.debug('hlimit %s', hlimit) + for l in bw_lines_tf: + if desc_obs_bws == TORFLOW_OBS_LAST: + desc_obs_bw = l.desc_obs_bw_bs_last + elif desc_obs_bws == TORFLOW_OBS_MEAN: + desc_obs_bw = l.desc_obs_bw_bs_mean + # just applying the formula above: + l.bw = max(round( + max( + # ratio + l.bw_bs_mean / mu, + # ratio filtered + min(l.bw_bs_mean, mu) / muf + ) * desc_obs_bw + / 1000), 1) + return sorted(bw_lines_tf, key=lambda x: x.bw, reverse=reverse) @property def sum_bw(self):

1 0

[sbws/master] Make V3BWFile from results depend on the scaling method
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit f34d0c0df1ec4b2babd45f8ccfb71a7d8924c1d5 Author: juga0 <juga(a)riseup.net> Date: Thu Aug 30 12:49:47 2018 +0000 Make V3BWFile from results depend on the scaling method --- sbws/lib/v3bwfile.py | 65 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 22bf245..cdcbad8 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -8,7 +8,10 @@ import os from statistics import median, mean from sbws import __version__ -from sbws.globals import SPEC_VERSION, BW_LINE_SIZE, SCALE_CONSTANT +from sbws.globals import (SPEC_VERSION, BW_LINE_SIZE, SBWS_SCALE_CONSTANT, + TORFLOW_SCALING, SBWS_SCALING, TORFLOW_BW_MARGIN, + TORFLOW_OBS_LAST, TORFLOW_OBS_MEAN, + fail_hard) from sbws.lib.resultdump import ResultSuccess, _ResultType from sbws.util.filelock import DirectoryLock from sbws.util.timestamp import now_isodt_str, unixts_to_isodt_str @@ -38,7 +41,8 @@ BW_KEYVALUES_BASIC = ['node_id', 'bw'] BW_KEYVALUES_FILE = BW_KEYVALUES_BASIC + \ ['master_key_ed25519', 'nick', 'rtt', 'time', 'success', 'error_stream', 'error_circ', 'error_misc'] -BW_KEYVALUES_EXTRA_BWS = ['bw_bs_median', 'bw_bs_mean', 'desc_avg_bw_bs'] +BW_KEYVALUES_EXTRA_BWS = ['bw_bs_median', 'bw_bs_mean', 'desc_avg_bw_bs', + 'desc_obs_bw_bs_last', 'desc_obs_bw_bs_mean'] BW_KEYVALUES_EXTRA = BW_KEYVALUES_FILE + BW_KEYVALUES_EXTRA_BWS BW_KEYVALUES_INT = ['bw', 'rtt', 'success', 'error_stream', 'error_circ', 'error_misc'] + BW_KEYVALUES_EXTRA_BWS @@ -355,20 +359,52 @@ class V3BWFile(object): self.bw_lines = v3bwlines def __str__(self): - return str(self.header) + ''.join([str(bw_line) + return str(self.header) + ''.join([str(bw_line) or '' for bw_line in self.bw_lines]) @classmethod def from_results(cls, results, state_fpath='', - scale_constant=None): - bw_lines = [V3BWLine.from_results(results[fp]) for fp in results] - bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True) - if scale_constant is not None: - bw_lines = cls.bw_sbws_scale(bw_lines, scale_constant) + scale_constant=SBWS_SCALE_CONSTANT, + scaling_method=None, torflow_obs=TORFLOW_OBS_LAST, + reverse=False): + """Create V3BWFile class from sbws Results. + + :param dict results: see below + :param str state_fpath: path to the state file + :param int scaling_method: + Scaling method to obtain the bandwidth + Posiable values: {NONE, SBWS_SCALING, TORFLOW_SCALING} = {0, 1, 2} + :param int scale_constant: sbws scaling constant + :param int torflow_obs: method to choose descriptor observed bandwidth + :param bool reverse: whether to sort the bw lines descending or not + + Results are in the form:: + + {'relay_fp1': [Result1, Result2, ...], + 'relay_fp2': [Result1, Result2, ...]} + + """ + # TODO: change scaling_method to TORFLOW_SCALING before getting this + # in production + log.info('Processing results to generate a bandwidth list file.') + header = V3BWHeader.from_results(results, state_fpath) + bw_lines_raw = [] + for fp in results.keys(): + l = V3BWLine.from_results(results[fp]) + if l is not None: + bw_lines_raw.append(l) + if not bw_lines_raw: + return cls(header, []) + if scaling_method == SBWS_SCALING: + bw_lines = cls.bw_sbws_scale(bw_lines_raw, scale_constant) cls.warn_if_not_accurate_enough(bw_lines, scale_constant) + # log.debug(bw_lines[-1]) + elif scaling_method == TORFLOW_SCALING: + bw_lines = cls.bw_torflow_scale(bw_lines_raw, torflow_obs) + # log.debug(bw_lines[-1]) else: - bw_lines = cls.bw_kb(bw_lines) - header = V3BWHeader.from_results(results, state_fpath) + bw_lines = cls.bw_kb(bw_lines_raw) + # log.debug(bw_lines[-1]) f = cls(header, bw_lines) return f @@ -380,7 +416,7 @@ class V3BWFile(object): return sorted(bw_lines_scaled, key=lambda x: x.bw, reverse=reverse) @staticmethod - def bw_sbws_scale(bw_lines, scale_constant=SCALE_CONSTANT, + def bw_sbws_scale(bw_lines, scale_constant=SBWS_SCALE_CONSTANT, reverse=False): """Return a new V3BwLine list scaled using sbws method. @@ -417,7 +453,7 @@ class V3BWFile(object): @staticmethod def warn_if_not_accurate_enough(bw_lines, - scale_constant=SCALE_CONSTANT): + scale_constant=SBWS_SCALE_CONSTANT): margin = 0.001 accuracy_ratio = median([l.bw for l in bw_lines]) / scale_constant log.info('The generated lines are within {:.5}% of what they should ' @@ -426,6 +462,11 @@ class V3BWFile(object): log.warning('There was %f%% error and only +/- %f%% is ' 'allowed', (1 - accuracy_ratio) * 100, margin * 100) + @staticmethod + def bw_lines_torflow(bw_lines, desc_obs_bws=TORFLOW_OBS_LAST, + cap=TORFLOW_BW_MARGIN, reverse=False): + pass + @property def sum_bw(self): return sum([l.bw for l in self.bw_lines])

1 0

[sbws/master] Add method to parse V110 file from a given path
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 39219fc2d1d18cf1457cf7a9eddd6daefc1e6c17 Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 11:38:01 2018 +0000 Add method to parse V110 file from a given path --- sbws/lib/v3bwfile.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 34866e9..dda4499 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -429,6 +429,16 @@ class V3BWFile(object): f = cls(header, bw_lines) return f + @classmethod + def from_v110_fpath(cls, fpath): + log.info('Parsing bandwidth file %s', fpath) + with open(fpath) as fd: + text = fd.read() + all_lines = text.split(LINE_SEP) + header, lines = V3BWHeader.from_lines_v110(all_lines) + bw_lines = [V3BWLine.from_bw_line_v110(line) for line in lines] + return cls(header, bw_lines) + @staticmethod def bw_kb(bw_lines, reverse=False): bw_lines_scaled = copy.deepcopy(bw_lines)

1 0

[sbws/master] Use method to log stats
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 14f0406e6a6193501eb33b62b76365c0c18de74a Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 14:08:08 2018 +0000 Use method to log stats --- sbws/core/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index f996c93..113b05f 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -74,4 +74,4 @@ def main(args, conf): output = args.output or \ conf.getpath('paths', 'v3bw_fname').format(now_fname()) bw_file.write(output) - log.info('Mean bandwidth per line: %f "KiB"', bw_file.mean_bw) + bw_file.info_stats

1 0

[sbws/master] Add method to find bw of a node
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 02f0f815829ae483f441c7c563965e5899bcac99 Author: juga0 <juga(a)riseup.net> Date: Fri Aug 31 13:58:23 2018 +0000 Add method to find bw of a node useful to combine data when plotting --- sbws/lib/v3bwfile.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 04699c6..545cd77 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -724,6 +724,16 @@ class V3BWFile(object): def median_bw(self): return median([l.bw for l in self.bw_lines]) + def bw_line_for_node_id(self, node_id): + """Returns the bandwidth line for a given node fingerprint. + + Used to combine data when plotting. + """ + bwl = [l for l in self.bw_lines if l.node_id == node_id] + if bwl: + return bwl[0] + return None + def to_plt(self, attrs=['bw'], sorted_by=None): """Return bandwidth data in a format useful for matplotlib.

1 0