September 2018 - tor-commits - lists.torproject.org

[sbws/master] Complete info about required disk space
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 1e916ea2452d499dacae9ac1e64385cd73f10f20 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 6 16:55:51 2018 +0000 Complete info about required disk space --- sbws/util/fs.py | 77 +++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 16 deletions(-) diff --git a/sbws/util/fs.py b/sbws/util/fs.py index c7a5de7..0095f2a 100644 --- a/sbws/util/fs.py +++ b/sbws/util/fs.py @@ -5,37 +5,82 @@ import shutil log = logging.getLogger(__name__) +DISK_SPACE_TEXT = """ +Disk space requirements +----------------------- +v3bw files: the maximum space required is ~{mb_bw} MB, after {d_bw} days. +result files: the maximum space required is ~{mb_results} MB, after {d_r} days. +tor directory: the space required is ~{mb_tor} MB. +code and depenencies: the space required is ~{mb_code} MB +Total disk space required is: ~{mb_total} MB +""" + def sbws_required_disk_space(conf): """Disk space required by sbws files. Rough calculations. :param ConfigParser conf: sbws configuration - :returns: int, size in MiB + :returns: int, size in MB """ + text_dict = {} # Number of relays per line average size in Bytes size_v3bw_file = 7500 * 220 - # the minimum number of required v3bw files - num_v3bw_files = 2 + # default crontab configuration will run genenerate every hour + num_v3bw_files_day = 24 # ~1000 is the length of a line when the result is successfull # ~4550 is the number of lines of the biggest result file size_result_file = 4550 * 1000 - num_result_files = conf.getint('general', 'data_period') - # not counting compressed files - space_v3bw_files = size_v3bw_file * num_v3bw_files - space_result_files = size_result_file * num_result_files + num_result_files_day = 1 + space_v3bw_files_day = size_v3bw_file * num_v3bw_files_day + space_result_files_day = size_result_file * num_result_files_day + size_compressed_files = 600 * 1024 + # default crontab configuration will run cleanup once a day + # default cleanup configuration will compress v3bw files after 1 day + # and delete them after 7 days + v3bw_compress_after_days = conf.getint('cleanup', + 'v3bw_files_compress_after_days') + v3bw_delete_after_days = conf.getint('cleanup', + 'v3bw_files_delete_after_days') + v3bw_max_space_after_delete = \ + (space_v3bw_files_day * v3bw_compress_after_days) + \ + (size_compressed_files * num_v3bw_files_day * v3bw_delete_after_days) + text_dict['mb_bw'] = round(v3bw_max_space_after_delete / 1000 ** 2) + text_dict['d_bw'] = v3bw_delete_after_days + # default crontab configuration will run cleanup once a day + # default cleanup configuration will compress v3bw files after 1 day + # and delete them after 7 days + results_compress_after_days = conf.getint('cleanup', + 'data_files_compress_after_days') + results_delete_after_days = conf.getint('cleanup', + 'data_files_delete_after_days') + results_max_space_after_delete = \ + (space_result_files_day * results_compress_after_days) + \ + (size_compressed_files * num_v3bw_files_day * + results_delete_after_days) + text_dict['mb_results'] = round(results_max_space_after_delete / 1000 ** 2) + text_dict['d_r'] = results_delete_after_days # not counted rotated files and assuming that when it is not rotated the # size will be aproximately 10MiB - size_log_file = conf.getint('logging', 'to_file_max_bytes') or 10485760 \ - if conf.getboolean('logging', 'to_file') else 0 + space_log_files = 0 + if conf.getboolean('logging', 'to_file'): + size_log_file = conf.getint('logging', 'to_file_max_bytes') + num_log_files = conf.geting('logging', 'to_file_num_backups') + space_log_files = size_log_file * num_log_files + text_dict['mb_log'] = space_log_files # roughly, size of a current tor dir - space_tor_dir = 19828000 - # duplicate everything to warn early - size_total = (space_v3bw_files + size_log_file + space_result_files + - space_tor_dir) * 2 - # convert to MiB - size_total_mb = round(size_total / (1024 ** 2)) - return size_total_mb + size_tor_dir = 19828000 + text_dict['mb_tor'] = round(size_tor_dir / 1000 ** 2) + # roughly, the size of this code and dependencies + size_code_deps = 2097152 + text_dict['mb_code'] = round(size_code_deps / 1000 ** 2) + # Multiply per 2, just in case + size_total = (results_max_space_after_delete + + v3bw_max_space_after_delete + space_log_files + size_tor_dir + + size_code_deps) * 2 + text_dict['mb_total'] = round(size_total / 1000 ** 2) + space_text = DISK_SPACE_TEXT.format(**text_dict) + return space_text def df(path):

1 0

[sbws/master] Stop using function that makes disk system call
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit d6e44d30b1565ab5696a0bfd7bf20906f24db725 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 6 16:54:58 2018 +0000 Stop using function that makes disk system call since some systems might not allow it. --- sbws/core/scanner.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index fa8cc88..b1976ae 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -21,8 +21,6 @@ import logging import requests import random -from ..util.fs import is_low_space - rng = random.SystemRandom() end_event = Event() @@ -371,9 +369,6 @@ def gen_parser(sub): def main(args, conf): - if is_low_space(conf): - exit(1) - if conf.getint('scanner', 'measurement_threads') < 1: fail_hard('Number of measurement threads must be larger than 1')

1 0

[sbws/master] Add info about disk requirements.
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 54b00da7f92242e2680637746be0afb633feac55 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 6 16:59:23 2018 +0000 Add info about disk requirements. It will only be shown when running sbws without arguments. --- sbws/sbws.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbws/sbws.py b/sbws/sbws.py index 6f8752c..1ac13dd 100644 --- a/sbws/sbws.py +++ b/sbws/sbws.py @@ -14,6 +14,8 @@ from requests.__version__ import __version__ as requests_version import platform import logging +from sbws.util.fs import sbws_required_disk_space + log = logging.getLogger(__name__) @@ -51,6 +53,7 @@ def main(): log.critical(e) exit(1) configure_logging(args, conf) + parser.description = sbws_required_disk_space(conf) def_args = [args, conf] def_kwargs = {} known_commands = {

1 0

[sbws/master] Comment functions not being used
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit a1ccfa430a65d3aefa2468dcedbdf4be1a6fea0d Author: juga0 <juga(a)riseup.net> Date: Thu Sep 6 16:58:36 2018 +0000 Comment functions not being used --- sbws/util/fs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sbws/util/fs.py b/sbws/util/fs.py index 0095f2a..9877cf1 100644 --- a/sbws/util/fs.py +++ b/sbws/util/fs.py @@ -84,11 +84,15 @@ def sbws_required_disk_space(conf): def df(path): + # Not being used, since it makes a disk space system call and some + # systems might not allow it """Return space left on device where path is in MiB.""" return round(shutil.disk_usage(path).free / (1024 ** 2)) def is_low_space(conf): + # Not being used, since it makes a disk space system call and some + # systems might not allow it """Warn and return True when the space left on the device is less than what is needed for sbws and False otherwise needs. """

1 0

[sbws/master] Stop using function that makes disk system call
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 0da51525c4e679f0d7d04bbd81ef9768bb62153a Author: juga0 <juga(a)riseup.net> Date: Thu Sep 6 16:53:31 2018 +0000 Stop using function that makes disk system call since some systems might not allow it. --- sbws/core/generate.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 3c19ed4..f10ecc1 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -5,7 +5,6 @@ from argparse import ArgumentDefaultsHelpFormatter import os import logging from sbws.util.timestamp import now_fname -from ..util.fs import is_low_space log = logging.getLogger(__name__) @@ -41,8 +40,6 @@ def gen_parser(sub): def main(args, conf): os.makedirs(conf.getpath('paths', 'v3bw_dname'), exist_ok=True) - if is_low_space(conf): - exit(1) datadir = conf.getpath('paths', 'datadir') if not os.path.isdir(datadir):

1 0

[sbws/master] Reorder methods in BWV3Line
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 992ea07eb0eb3baf232ca9c0278794ec632bca6f Author: juga0 <juga(a)riseup.net> Date: Wed Aug 29 14:45:08 2018 +0000 Reorder methods in BWV3Line 1. magic methods 2. classmethods 3. staticmethods 4. properties 5. methods --- sbws/lib/v3bwfile.py | 130 +++++++++++++++++++++++++-------------------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index d663445..9324bde 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -249,39 +249,31 @@ class V3BWLine(object): [setattr(self, k, v) for k, v in kwargs.items() if k in BW_EXTRA_ARG_KEYVALUES] - @property - def bw_keyvalue_tuple_ls(self): - """Return list of KeyValue Bandwidth Line tuples.""" - # sort the list to generate determinist headers - keyvalue_tuple_ls = sorted([(k, v) for k, v in self.__dict__.items() - if k in BW_KEYVALUES]) - return keyvalue_tuple_ls - - @property - def bw_keyvalue_v110str_ls(self): - """Return list of KeyValue Bandwidth Line strings following - spec v1.1.0. - """ - bw_keyvalue_str = [KEYVALUE_SEP_V110 .join([k, str(v)]) - for k, v in self.bw_keyvalue_tuple_ls] - return bw_keyvalue_str - - @property - def bw_strv110(self): - """Return Bandwidth Line string following spec v1.1.0.""" - bw_line_str = BW_KEYVALUE_SEP_V110.join( - self.bw_keyvalue_v110str_ls) + LINE_SEP - if len(bw_line_str) > BW_LINE_SIZE: - # if this is the case, probably there are too many KeyValues, - # or the limit needs to be changed in Tor - log.warn("The bandwidth line %s is longer than %s", - len(bw_line_str), BW_LINE_SIZE) - return bw_line_str - def __str__(self): return self.bw_strv110 @classmethod + def from_results(cls, results): + success_results = [r for r in results if isinstance(r, ResultSuccess)] + # log.debug('len(success_results) %s', len(success_results)) + node_id = '$' + results[0].fingerprint + bw = cls.bw_from_results(success_results) + kwargs = dict() + kwargs['nick'] = results[0].nickname + if getattr(results[0], 'master_key_ed25519'): + kwargs['master_key_ed25519'] = results[0].master_key_ed25519 + kwargs['rtt'] = cls.rtt_from_results(success_results) + kwargs['time'] = cls.last_time_from_results(results) + kwargs.update(cls.result_types_from_results(results)) + bwl = cls(node_id, bw, **kwargs) + return bwl + + @classmethod + def from_data(cls, data, fingerprint): + assert fingerprint in data + return cls.from_results(data[fingerprint]) + + @classmethod def from_bw_line_v110(cls, line): assert isinstance(line, str) kwargs = dict([kv.split(KEYVALUE_SEP_V110) @@ -294,6 +286,24 @@ class V3BWLine(object): return bw_line @staticmethod + def last_time_from_results(results): + return unixts_to_isodt_str(round(max([r.time for r in results]))) + + @staticmethod + def rtt_from_results(results): + # convert from miliseconds to seconds + rtts = [(round(rtt * 1000)) for r in results for rtt in r.rtts] + rtt = round(median(rtts)) + return rtt + + @staticmethod + def result_types_from_results(results): + rt_dict = dict([(result_type_to_key(rt.value), + num_results_of_type(results, rt.value)) + for rt in _ResultType]) + return rt_dict + + @staticmethod def bw_from_results(results): median_bw = median([dl['amount'] / dl['duration'] for r in results for dl in r.downloads]) @@ -319,44 +329,34 @@ class V3BWLine(object): bw_kb = max(round(bw / 1024), 1) return bw_kb - @staticmethod - def last_time_from_results(results): - return unixts_to_isodt_str(round(max([r.time for r in results]))) - - @staticmethod - def rtt_from_results(results): - # convert from miliseconds to seconds - rtts = [(round(rtt * 1000)) for r in results for rtt in r.rtts] - rtt = round(median(rtts)) - return rtt - - @staticmethod - def result_types_from_results(results): - rt_dict = dict([(result_type_to_key(rt.value), - num_results_of_type(results, rt.value)) - for rt in _ResultType]) - return rt_dict + @property + def bw_keyvalue_tuple_ls(self): + """Return list of KeyValue Bandwidth Line tuples.""" + # sort the list to generate determinist headers + keyvalue_tuple_ls = sorted([(k, v) for k, v in self.__dict__.items() + if k in BW_KEYVALUES]) + return keyvalue_tuple_ls - @classmethod - def from_results(cls, results): - success_results = [r for r in results if isinstance(r, ResultSuccess)] - # log.debug('len(success_results) %s', len(success_results)) - node_id = '$' + results[0].fingerprint - bw = cls.bw_from_results(success_results) - kwargs = dict() - kwargs['nick'] = results[0].nickname - if getattr(results[0], 'master_key_ed25519'): - kwargs['master_key_ed25519'] = results[0].master_key_ed25519 - kwargs['rtt'] = cls.rtt_from_results(success_results) - kwargs['time'] = cls.last_time_from_results(results) - kwargs.update(cls.result_types_from_results(results)) - bwl = cls(node_id, bw, **kwargs) - return bwl + @property + def bw_keyvalue_v110str_ls(self): + """Return list of KeyValue Bandwidth Line strings following + spec v1.1.0. + """ + bw_keyvalue_str = [KEYVALUE_SEP_V110 .join([k, str(v)]) + for k, v in self.bw_keyvalue_tuple_ls] + return bw_keyvalue_str - @classmethod - def from_data(cls, data, fingerprint): - assert fingerprint in data - return cls.from_results(data[fingerprint]) + @property + def bw_strv110(self): + """Return Bandwidth Line string following spec v1.1.0.""" + bw_line_str = BW_KEYVALUE_SEP_V110.join( + self.bw_keyvalue_v110str_ls) + LINE_SEP + if len(bw_line_str) > BW_LINE_SIZE: + # if this is the case, probably there are too many KeyValues, + # or the limit needs to be changed in Tor + log.warn("The bandwidth line %s is longer than %s", + len(bw_line_str), BW_LINE_SIZE) + return bw_line_str class V3BWFile(object):

1 0

[sbws/master] Remove test since function not being used
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit e6b9b84a0a98d6e2a688a110515954bd1b90bdb3 Author: juga0 <juga(a)riseup.net> Date: Thu Sep 6 17:25:31 2018 +0000 Remove test since function not being used --- tests/unit/util/test_fs.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/tests/unit/util/test_fs.py b/tests/unit/util/test_fs.py deleted file mode 100644 index e659d83..0000000 --- a/tests/unit/util/test_fs.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Unit tests for fs module""" -from unittest.mock import patch - -from sbws.util import fs - - -def mock_df_zero(path): - return 0 - - -def mock_df_enough(path): - return 89 - - -@patch('sbws.util.fs.df', mock_df_zero) -def test_is_low_space_true(caplog, conf): - assert fs.is_low_space(conf) is True - assert ' is less than ' in caplog.records[-1].getMessage() - - -@patch('sbws.util.fs.df', mock_df_enough) -def test_is_low_space_false(conf): - assert fs.is_low_space(conf) is False

1 0

[sbws/master] Pass state file path and scale constant
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit cf9a55a359d8071a8bc2abdfeb1886804165c4bd Author: juga0 <juga(a)riseup.net> Date: Wed Aug 29 15:01:38 2018 +0000 Pass state file path and scale constant instead of conf and args, so that it's easier to test the method without creating ConfigParser and ArgumentParser objects. --- sbws/core/generate.py | 3 ++- sbws/lib/v3bwfile.py | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index f10ecc1..777b05e 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -58,7 +58,8 @@ def main(args, conf): log.warning('No recent results, so not generating anything. (Have you ' 'ran sbws scanner recently?)') return - bw_file = V3BWFile.from_arg_results(args, conf, results) + state_fpath = conf.getpath('paths', 'state_fname') + bw_file = V3BWFile.from_results(results, state_fpath, args.scale_constant) output = args.output or \ conf.getpath('paths', 'v3bw_fname').format(now_fname()) bw_file.write(output) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 576345e..ef9eadc 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -115,11 +115,11 @@ class V3BWHeader(object): return self.strv200 @classmethod - def from_results(cls, conf, results): + def from_results(cls, results, state_fpath=''): kwargs = dict() latest_bandwidth = cls.latest_bandwidth_from_results(results) earliest_bandwidth = cls.earliest_bandwidth_from_results(results) - generator_started = cls.generator_started_from_file(conf) + generator_started = cls.generator_started_from_file(state_fpath) timestamp = str(latest_bandwidth) kwargs['latest_bandwidth'] = unixts_to_isodt_str(latest_bandwidth) kwargs['earliest_bandwidth'] = unixts_to_isodt_str(earliest_bandwidth) @@ -159,12 +159,12 @@ class V3BWHeader(object): return self.from_lines_v110(text.split(LINE_SEP)) @staticmethod - def generator_started_from_file(conf): + def generator_started_from_file(state_fpath): ''' ISO formatted timestamp for the time when the scanner process most recently started. ''' - state = State(conf.getpath('paths', 'state_fname')) + state = State(state_fpath) if 'scanner_started' in state: return state['scanner_started'] else: @@ -376,12 +376,14 @@ class V3BWFile(object): for bw_line in self.bw_lines]) @classmethod - def from_arg_results(cls, args, conf, results): + def from_results(cls, results, state_fpath='', + scale_constant=None): bw_lines = [V3BWLine.from_results(results[fp]) for fp in results] bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True) - if args.scale: - bw_lines = scale_lines(bw_lines, args.scale_constant) - header = V3BWHeader.from_results(conf, results) + if scale_constant: + bw_lines = cls.bw_sbws_scale(bw_lines, scale_constant) + cls.warn_if_not_accurate_enough(bw_lines, scale_constant) + header = V3BWHeader.from_results(results, state_fpath) f = cls(header, bw_lines) return f

1 0

[sbws/master] Reorder methods in BWV3File
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 9ac61cc56cad551af3c7e1194ef94183520e4e86 Author: juga0 <juga(a)riseup.net> Date: Wed Aug 29 14:46:34 2018 +0000 Reorder methods in BWV3File 1. magic methods 2. classmethods 3. staticmethods 4. properties 5. methods --- sbws/lib/v3bwfile.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 9324bde..b068075 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -374,6 +374,16 @@ class V3BWFile(object): return str(self.header) + ''.join([str(bw_line) for bw_line in self.bw_lines]) + @classmethod + def from_arg_results(cls, args, conf, results): + bw_lines = [V3BWLine.from_results(results[fp]) for fp in results] + bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True) + if args.scale: + bw_lines = scale_lines(bw_lines, args.scale_constant) + header = V3BWHeader.from_results(conf, results) + f = cls(header, bw_lines) + return f + @property def total_bw(self): return total_bw(self.bw_lines) @@ -386,16 +396,6 @@ class V3BWFile(object): def avg_bw(self): return self.total_bw / self.num_lines - @classmethod - def from_arg_results(cls, args, conf, results): - bw_lines = [V3BWLine.from_results(results[fp]) for fp in results] - bw_lines = sorted(bw_lines, key=lambda d: d.bw, reverse=True) - if args.scale: - bw_lines = scale_lines(bw_lines, args.scale_constant) - header = V3BWHeader.from_results(conf, results) - f = cls(header, bw_lines) - return f - def write(self, output): if output == '/dev/stdout': log.info("Writing to stdout is not supported.")

1 0

[sbws/master] Stop including \n EOL when parsing V110 header
by juga＠torproject.org 18 Sep '18

18 Sep '18

commit 48fa53dd3a38f021654fba4d9be65be7d2af7ac4 Author: juga0 <juga(a)riseup.net> Date: Wed Aug 29 14:49:37 2018 +0000 Stop including \n EOL when parsing V110 header --- sbws/lib/v3bwfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index b068075..576345e 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -146,7 +146,8 @@ class V3BWHeader(object): for l in lines[:index_terminator] if l.split(KEYVALUE_SEP_V110)[0] in ALL_KEYVALUES]) h = cls(ts, **kwargs) - return h, lines[index_terminator + 1:] + # last line is new line + return h, lines[index_terminator + 1:-1] @classmethod def from_text_v110(self, text):

1 0