tor-commits October 2018

tor-commits@lists.torproject.org

17 participants
2350 discussions

[sbws/master] Write/read min_perc_reached in the state
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit fc5db6793205c52dd8dbc8d051e6199619dbfc00 Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 12:40:21 2018 +0000 Write/read min_perc_reached in the state To know whether the min percent of measured relays was reached before or not, and warn when it was. --- sbws/core/generate.py | 2 +- sbws/lib/v3bwfile.py | 34 +++++++++++++++++++++++++++------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 6cc418f..78b98b6 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -109,5 +109,5 @@ def main(args, conf): output = args.output or \ conf.getpath('paths', 'v3bw_fname').format(now_fname()) - bw_file.write(output, args.rm_link) + bw_file.write(output) bw_file.info_stats diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 52eaec7..b1ee357 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -490,6 +490,7 @@ class V3BWFile(object): if line is not None: bw_lines_raw.append(line) if not bw_lines_raw: + log.info("There are not enough raw results to apply any scaling.") return cls(header, []) if scaling_method == SBWS_SCALING: bw_lines = cls.bw_sbws_scale(bw_lines_raw, scale_constant) @@ -501,9 +502,9 @@ class V3BWFile(object): # log.debug(bw_lines[-1]) if consensus_path is not None: statsd, success = cls.measured_progress_stats(bw_lines, - consensus_path) - # add statistics about progress only when there are not enough - # measured relays. Should some stats be added always? + consensus_path, state_fpath) + # add statistics about progress only when there are not enough + # measured relays. Should some stats be added always? if not success: header.add_stats(**statsd) bw_lines = [] @@ -793,12 +794,14 @@ class V3BWFile(object): return sorted(bw_lines_tf, key=lambda x: x.bw, reverse=reverse) @staticmethod - def measured_progress_stats(bw_lines, consensus_path): + @staticmethod + def measured_progress_stats(bw_lines, consensus_path, state_fpath): """ Statistics about measurements progress, to be included in the header. :param list bw_lines: the bw_lines after scaling and applying filters. :param str consensus_path: the path to the cached consensus file. + :param str state_fpath: the path to the state file :returns dict, bool: Statistics about the progress made with measurements and whether the percentage of measured relays has been reached. @@ -810,6 +813,8 @@ class V3BWFile(object): # measured relays is not either. assert isinstance(consensus_path, str) assert isinstance(bw_lines, list) + assert isinstance(state_fpath, str) + state = State(state_fpath) statsd = {} statsd['num_measured_relays'] = len(bw_lines) statsd['num_net_relays'] = len(list(parse_file(consensus_path))) @@ -819,10 +824,25 @@ class V3BWFile(object): / statsd['num_net_relays']) statsd['perc_measured_targed'] = MIN_REPORT if statsd['num_measured_relays'] < statsd['num_target_relays']: - log.warning('The percentage of the measured relays is less than' - ' the %s%% of the relays in the network (%s).', - MIN_REPORT, statsd['num_net_relays']) + # the min percent of measured relays is not reached, + # write None in the state file and obtain whether it was reached + # before + min_perc_reached_before = state.get('min_perc_reached') + state['min_perc_reached'] = None + # if it was reached before, warn + # otherwise, debug + if min_perc_reached_before is not None: + log.warning('The percentage of the measured relays is less ' + 'than the %s%% of the relays in the network (%s).', + MIN_REPORT, statsd['num_net_relays']) + else: + log.info('The percentage of the measured relays is less ' + 'than the %s%% of the relays in the network (%s).', + MIN_REPORT, statsd['num_net_relays']) return statsd, False + # write in the state file the date the min percent of measured + # relays has been reached + state['min_perc_reached'] = now_isodt_str() return statsd, True @property

1 0

[sbws/master] Add method to read the number of relays in the net
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit ec43211f9583b42648fe5db0075c22f5f66e281c Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 10:54:44 2018 +0000 Add method to read the number of relays in the net So that the method to calculate statistics about the number of relays measured can be tested without having a tor data directory. --- sbws/lib/v3bwfile.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index b1ee357..b1bdbff 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -483,6 +483,7 @@ class V3BWFile(object): log.info('Processing results to generate a bandwidth list file.') header = V3BWHeader.from_results(results, state_fpath) bw_lines_raw = [] + num_net_relays = cls.read_num_net_relays(consensus_path) for fp, values in results.items(): # log.debug("Relay fp %s", fp) line = V3BWLine.from_results(values, secs_recent, secs_away, @@ -500,7 +501,7 @@ class V3BWFile(object): bw_lines = cls.bw_torflow_scale(bw_lines_raw, torflow_obs, torflow_cap, torflow_round_digs) # log.debug(bw_lines[-1]) - if consensus_path is not None: + if num_net_relays is not None: statsd, success = cls.measured_progress_stats(bw_lines, consensus_path, state_fpath) # add statistics about progress only when there are not enough @@ -794,8 +795,22 @@ class V3BWFile(object): return sorted(bw_lines_tf, key=lambda x: x.bw, reverse=reverse) @staticmethod + def read_num_net_relays(consensus_path): + """Read the number of relays in the Network from the cached consensus + file.""" + assert isinstance(consensus_path, str) + num = None + try: + num = len(list(parse_file(consensus_path))) + except FileNotFoundError: + log.info("It is not possible to obtain statistics about the " + "percentage of measured relays because the cached " + "consensus file is not found.") + log.debug("Number of relays in the network %s", num) + return num + @staticmethod - def measured_progress_stats(bw_lines, consensus_path, state_fpath): + def measured_progress_stats(bw_lines, num_net_relays, state_fpath): """ Statistics about measurements progress, to be included in the header. @@ -811,13 +826,13 @@ class V3BWFile(object): # network status or descriptors? # It will not be updated to the last consensus, but the list of # measured relays is not either. - assert isinstance(consensus_path, str) + assert isinstance(num_net_relays, int) assert isinstance(bw_lines, list) assert isinstance(state_fpath, str) state = State(state_fpath) statsd = {} statsd['num_measured_relays'] = len(bw_lines) - statsd['num_net_relays'] = len(list(parse_file(consensus_path))) + statsd['num_net_relays'] = num_net_relays statsd['num_target_relays'] = round(statsd['num_net_relays'] * MIN_REPORT / 100) statsd['perc_measured_relays'] = round(len(bw_lines) * 100

1 0

[sbws/master] Add entry about min percent warning
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 210fb2386f98ab0e57465fdd0b24314e12c2405f Author: juga0 <juga(a)riseup.net> Date: Tue Oct 23 14:35:07 2018 +0000 Add entry about min percent warning --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c66ab..148fce4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - When the percentage of measured relays is less than the 60%, do not include the relays in the bandwidth file and instead include some statistics in the header (#28076) +- When the percentage of measured relays is less than the 60% and it was more + before, warn about it (#28155) ## [0.8.0] - 2018-10-08

1 0

[sbws/master] Add function to obtain percentage measured relays
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit ed520c485a7bcd4ad9f1b1c3e4288b35303fae2e Author: juga0 <juga(a)riseup.net> Date: Tue Oct 16 12:52:55 2018 +0000 Add function to obtain percentage measured relays --- sbws/globals.py | 1 + sbws/lib/v3bwfile.py | 23 ++++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/sbws/globals.py b/sbws/globals.py index 52c31bf..9ac30d1 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -40,6 +40,7 @@ TORFLOW_OBS_DECAYING = 3 TORFLOW_ROUND_DIG = 3 DAY_SECS = 86400 NUM_MIN_RESULTS = 2 +MIN_REPORT = 60 BW_LINE_SIZE = 510 diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 4eecda6..58056b1 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -6,12 +6,13 @@ import copy import logging import os from statistics import median, mean +from stem.descriptor import parse_file from sbws import __version__ from sbws.globals import (SPEC_VERSION, BW_LINE_SIZE, SBWS_SCALE_CONSTANT, TORFLOW_SCALING, SBWS_SCALING, TORFLOW_BW_MARGIN, TORFLOW_OBS_LAST, TORFLOW_OBS_MEAN, - TORFLOW_ROUND_DIG) + TORFLOW_ROUND_DIG, MIN_REPORT) from sbws.lib.resultdump import ResultSuccess, _ResultType from sbws.util.filelock import DirectoryLock from sbws.util.timestamp import (now_isodt_str, unixts_to_isodt_str, @@ -65,6 +66,22 @@ def result_type_to_key(type_str): return type_str.replace('-', '_') +def is_min_percent_measured(consensus_path): + """""" + # cached-consensus should be updated every time that scanner get the + # network status or descriptors? + # It will not be updated to the last consensus, but the list of measured + # relays is not either. + descs = parse_file(consensus_path) + num_relays_net = len(list(descs)) + if num_relays_net * MIN_REPORT / 100: + log.warning('The percentage of the measured relays is less than the %s' + '%% of the relays in the network (%s).', + MIN_REPORT, num_relays_net) + return False + return True + + class V3BWHeader(object): """ Create a bandwidth measurements (V3bw) header @@ -700,7 +717,7 @@ class V3BWFile(object): All of that can be expressed as: .. math:: - + bwn_i &= max\\left( \\frac{bw_i}{\\mu}, @@ -724,7 +741,7 @@ class V3BWFile(object): \\right)} \\right) \\times bwobs_i \\ - + &= max\\left( \\frac{bw_i}{\\frac{\\sum_{i=1}^{n}bw_i}{n}},

1 0

[sbws/master] Add changelog entry about min percent relays
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit d1a23f14739ee188c0a13e31cd29ebe5a3c1974b Author: juga0 <juga(a)riseup.net> Date: Tue Oct 16 15:11:09 2018 +0000 Add changelog entry about min percent relays --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f58bfe3..778b5f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Number of results comparison and number of results away from each other are incorrect (#28041) +## Added + +- Write bw file only when the percentage of measured relays is bigger than 60% + (#28062) + ## [0.8.0] - 2018-10-08 **Important changes**:

1 0

[sbws/master] Add method to remove latest link
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 5b33d3a3e9e1658ce2393f0cdd3cb13332faaf94 Author: juga0 <juga(a)riseup.net> Date: Tue Oct 16 15:01:18 2018 +0000 Add method to remove latest link --- sbws/lib/v3bwfile.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 58056b1..c69e674 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -842,6 +842,17 @@ class V3BWFile(object): ys = [[getattr(l, k) for l in self.bw_lines] for k in attrs] return x, ys, attrs + def rm_link(self, output): + """""" + out_dir = os.path.dirname(output) + out_link = os.path.join(out_dir, 'latest.v3bw') + log.info('Removing %s link.', output) + with DirectoryLock(out_dir): + try: + os.unlink(out_link) + except FileNotFoundError: + pass + def write(self, output): if output == '/dev/stdout': log.info("Writing to stdout is not supported.")

1 0

[sbws/master] Add cli arg to rm the bw file link
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 199c92747068bd45184dddfab3d07b4ed2acb45f Author: juga0 <juga(a)riseup.net> Date: Thu Oct 18 21:47:34 2018 +0000 Add cli arg to rm the bw file link --- sbws/core/generate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 885569a..f147993 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -61,6 +61,9 @@ def gen_parser(sub): "other.") p.add_argument('-n', '--min-num', default=NUM_MIN_RESULTS, type=int, help="Mininum number of a results to consider them.") + p.add_argument('-l', '--rm-link', action='store_true', + help='If specified, remove latest.v3bw link when there ' + 'are no relays to include in it.') def main(args, conf):

1 0

[sbws/master] Write bw file only when there is a min percentage
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit 544936cd81abf4de58c84ad4d490fa3ff0e96064 Author: juga0 <juga(a)riseup.net> Date: Tue Oct 16 15:01:30 2018 +0000 Write bw file only when there is a min percentage --- sbws/core/generate.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 6a6226b..885569a 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -1,7 +1,7 @@ from sbws.globals import (fail_hard, SBWS_SCALE_CONSTANT, TORFLOW_SCALING, SBWS_SCALING, TORFLOW_BW_MARGIN, TORFLOW_ROUND_DIG, DAY_SECS, NUM_MIN_RESULTS) -from sbws.lib.v3bwfile import V3BWFile +from sbws.lib.v3bwfile import V3BWFile, is_min_percent_measured from sbws.lib.resultdump import load_recent_results_in_datadir from argparse import ArgumentDefaultsHelpFormatter import os @@ -99,7 +99,14 @@ def main(args, conf): secs_recent=args.secs_recent, secs_away=args.secs_away, min_num=args.min_num) + output = args.output or \ conf.getpath('paths', 'v3bw_fname').format(now_fname()) - bw_file.write(output) + # Check percentage of measured relays. + consensus_path = os.path.join(conf.getpath('tor', 'datadir'), + "cached-consensus") + if is_min_percent_measured(consensus_path): + bw_file.write(output) + else: + bw_file.rm_link(output) bw_file.info_stats

1 0

[sbws/master] Rm bwfile link when writting the file
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit c6a1ace87fb682d03cb7e695b2457ecdebd42a3b Author: juga0 <juga(a)riseup.net> Date: Thu Oct 18 21:55:53 2018 +0000 Rm bwfile link when writting the file --- sbws/lib/v3bwfile.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 83df447..1ada69e 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -462,7 +462,7 @@ class V3BWFile(object): torflow_cap=TORFLOW_BW_MARGIN, torflow_round_digs=TORFLOW_ROUND_DIG, secs_recent=None, secs_away=None, min_num=0, - reverse=False): + consensus_path=None, reverse=False): """Create V3BWFile class from sbws Results. :param dict results: see below @@ -500,6 +500,14 @@ class V3BWFile(object): bw_lines = cls.bw_torflow_scale(bw_lines_raw, torflow_obs, torflow_cap, torflow_round_digs) # log.debug(bw_lines[-1]) + if consensus_path is not None: + statsd, success = cls.measured_progress_stats(bw_lines, + consensus_path) + # add statistics about progress only when there are not enough + # measured relays. Should some stats be added always? + if not success: + header.add_stats(**statsd) + bw_lines = [] else: bw_lines = cls.bw_kb(bw_lines_raw) # log.debug(bw_lines[-1]) @@ -887,10 +895,13 @@ class V3BWFile(object): except FileNotFoundError: pass - def write(self, output): + def write(self, output, rm_link=False): if output == '/dev/stdout': log.info("Writing to stdout is not supported.") return + if not self.is_min_perc and rm_link: + self.rm_link(output) + return log.info('Writing v3bw file to %s', output) # To avoid inconsistent reads, the bandwidth data is written to an # archive path, then atomically symlinked to 'latest.v3bw'

1 0

[sbws/master] Create method to add new header attrs
by juga＠torproject.org 24 Oct '18

24 Oct '18

commit c5b342d25b989f7bbdd8b22b5e6871dbe4eb735a Author: juga0 <juga(a)riseup.net> Date: Thu Oct 18 21:53:24 2018 +0000 Create method to add new header attrs --- sbws/lib/v3bwfile.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index e33a7ec..83df447 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -229,6 +229,11 @@ class V3BWHeader(object): def num_lines(self): return len(self.__str__().split(LINE_SEP)) + def add_stats(self, **kwargs): + # Using kwargs because attributes might chage. + [setattr(self, k, str(v)) for k, v in kwargs.items() + if k in STATS_KEYVALUES] + class V3BWLine(object): """

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits October 2018