August 2018 - tor-commits - lists.torproject.org

[sbws/master] Cleanup clean all types of files unless otherwise
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 26fe323880eb4b6c06b8d8508fb5bbc854c37128 Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 16:33:49 2018 +0000 Cleanup clean all types of files unless otherwise specified --- sbws/core/cleanup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index bce90fb..3c4c8f8 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -26,6 +26,11 @@ def gen_parser(sub): formatter_class=ArgumentDefaultsHelpFormatter) p.add_argument('--dry-run', action='store_true', help='Don\'t actually compress or delete anything') + p.add_argument('--no-results', action='store_true', + help='Do not clean results files') + p.add_argument('--no-v3bw', action='store_true', + help='Do not clean v3bw files') + def _get_older_files_than(dname, num_days_ago, extensions):

1 0

[sbws/master] Replace option names for cleaning v3bw files
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 9379fdc26b3bfe1b510d767a0c9de2f520946e6b Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 16:18:52 2018 +0000 Replace option names for cleaning v3bw files --- sbws/config.default.ini | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index 860e8db..fd6a58c 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -14,8 +14,6 @@ usability_test_interval = 300 [general] # Days into the past that measurements are considered valid data_period = 5 -# Minutes into the past that the bandwidth list files need to be available (2h) -valid_mins_v3bw_files = 120 # Timeout in seconds to give to the python Requests library. It MUST be a # single float. Requests will use it both as the connect() timeout and the # timeout between bytes received from the server. See @@ -68,11 +66,10 @@ extra_lines = stale_days = 10 # After this many days, delete data files rotten_days = 90 -# After this many minutes, compress v3bw files -# Time during for the vote and time for the consensus (2h) -stale_mins_v3bw_files = 240 -# After this many minutes, delete v3bw files (1d) -rotten_mins_v3bw_files = 1440 +# After this many days, compress v3bw files (1d) +v3bw_files_compress_after_days = 1 +# After this many days, delete v3bw files (7d) +v3bw_files_delete_after_days = 7 [relayprioritizer] # Whether or not to measure authorities

1 0

[sbws/master] Replace options validation for cleaning v3bw files
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit d1286fe5369dea4a5fa56cd3b971606309302ca4 Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 16:19:25 2018 +0000 Replace options validation for cleaning v3bw files --- sbws/util/config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sbws/util/config.py b/sbws/util/config.py index a1d5a15..82ce0ec 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -174,8 +174,8 @@ def _validate_cleanup(conf): ints = { 'stale_days': {'minimum': 1, 'maximum': None}, 'rotten_days': {'minimum': 1, 'maximum': None}, - 'stale_mins_v3bw_files': {'minimum': 1, 'maximum': None}, - 'rotten_mins_v3bw_files': {'minimum': 1, 'maximum': None}, + 'v3bw_files_compress_after_days': {'minimum': 1, 'maximum': None}, + 'v3bw_files_delete_after_days': {'minimum': 1, 'maximum': None}, } all_valid_keys = list(ints.keys()) errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl)) @@ -189,7 +189,6 @@ def _validate_general(conf): err_tmpl = Template('$sec/$key ($val): $e') ints = { 'data_period': {'minimum': 1, 'maximum': None}, - 'valid_mins_v3bw_files': {'minimum': 1, 'maximum': None}, 'circuit_timeout': {'minimum': 1, 'maximum': None}, } floats = {

1 0

[sbws/master] Add function to find old v3bw files
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 1b7a24dfc132e0455ca43133f907dd51452ccfa0 Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 16:57:11 2018 +0000 Add function to find old v3bw files --- sbws/core/cleanup.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index 3c4c8f8..bd3300c 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -1,5 +1,6 @@ from sbws.util.filelock import DirectoryLock from sbws.globals import (fail_hard, is_initted) +from sbws.util.timestamp import unixts_to_dt_obj from argparse import ArgumentDefaultsHelpFormatter from datetime import datetime from datetime import timedelta @@ -32,6 +33,33 @@ def gen_parser(sub): help='Do not clean v3bw files') +def _get_files_mtime_older_than(dname, days_delta, extensions): + """Return files which modification time is older than days_delta + and which extension is one of the extensions.""" + assert os.path.isdir(dname) + assert isinstance(days_delta, int) + assert isinstance(extensions, list) + for ext in extensions: + assert isinstance(ext, str) + assert ext[0] == '.' + # Determine oldest allowed date + today = datetime.utcfromtimestamp(time.time()) + oldest_day = today - timedelta(days=days_delta) + for root, dirs, files in os.walk(dname): + for f in files: + fname = os.path.join(root, f) + _, ext = os.path.splitext(fname) + if ext not in extensions: + log.debug('Ignoring %s because it doesn\'t have extension ' + '%s', fname, ext) + continue + # using file modification time instead of parsing the name + # of the file. + filedt = unixts_to_dt_obj( + os.stat(fname, follow_symlinks=False).st_mtime) + if filedt < oldest_day and os.path.splitext: + yield fname + def _get_older_files_than(dname, num_days_ago, extensions): assert os.path.isdir(dname)

1 0

[sbws/master] Add function to delete files passed as arg
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 25f3fa9ccf3ad5d83cfd36ebf4f0e31b0cd191f6 Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 17:08:50 2018 +0000 Add function to delete files passed as arg --- sbws/core/cleanup.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index bd3300c..3deca57 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -1,3 +1,6 @@ +"""Util functions to cleanup disk space.""" +import types + from sbws.util.filelock import DirectoryLock from sbws.globals import (fail_hard, is_initted) from sbws.util.timestamp import unixts_to_dt_obj @@ -93,6 +96,17 @@ def _get_older_files_than(dname, num_days_ago, extensions): yield fname +def _delete_files(dname, files, dry_run=True): + """Delete the files passed as argument.""" + assert os.path.isdir(dname) + assert isinstance(files, types.GeneratorType) + with DirectoryLock(dname): + for fname in files: + log.info('Deleting %s', fname) + if not dry_run: + os.remove(fname) + + def _remove_rotten_files(datadir, rotten_days, dry_run=True): assert os.path.isdir(datadir) assert isinstance(rotten_days, int)

1 0

[sbws/master] Clean files depending on args
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 8926c2b945080dc930e8db039279b6c6aff78ee8 Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 17:15:58 2018 +0000 Clean files depending on args --- sbws/core/cleanup.py | 55 +++++++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index 04823b1..852b8e6 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -194,27 +194,34 @@ def main(args, conf): if not is_initted(args.directory): fail_hard('Sbws isn\'t initialized. Try sbws init') - datadir = conf['paths']['datadir'] - if not os.path.isdir(datadir): - fail_hard('%s does not exist', datadir) - - fresh_days = conf.getint('general', 'data_period') - stale_days = conf.getint('cleanup', 'stale_days') - rotten_days = conf.getint('cleanup', 'rotten_days') - if stale_days - 2 < fresh_days: - fail_hard('For safetly, cleanup/stale_days (%d) must be at least 2 ' - 'days larger than general/data_period (%d)', stale_days, - fresh_days) - if rotten_days < stale_days: - fail_hard('cleanup/rotten_days (%d) must be the same or larger than ' - 'cleanup/stale_days (%d)', rotten_days, stale_days) - - if stale_days / 2 < fresh_days: - log.warning( - 'cleanup/stale_days (%d) is less than twice ' - 'general/data_period (%d). For ease of parsing older results ' - 'if necessary, it is recommended to make stale_days at least ' - 'twice the data_period.', stale_days, fresh_days) - - _remove_rotten_files(datadir, rotten_days, dry_run=args.dry_run) - _compress_stale_files(datadir, stale_days, dry_run=args.dry_run) + if args.no_results and args.no_v3bw: + fail_hard('Nothing to clean.') + + if not args.no_results: + datadir = conf['paths']['datadir'] + if not os.path.isdir(datadir): + fail_hard('%s does not exist', datadir) + + fresh_days = conf.getint('general', 'data_period') + stale_days = conf.getint('cleanup', 'stale_days') + rotten_days = conf.getint('cleanup', 'rotten_days') + if stale_days - 2 < fresh_days: + fail_hard('For safetly, cleanup/stale_days (%d) must be at least ' + '2 days larger than general/data_period (%d)', + stale_days, fresh_days) + if rotten_days < stale_days: + fail_hard('cleanup/rotten_days (%d) must be the same or larger ' + 'than cleanup/stale_days (%d)', rotten_days, stale_days) + + if stale_days / 2 < fresh_days: + log.warning( + 'cleanup/stale_days (%d) is less than twice ' + 'general/data_period (%d). For ease of parsing older results ' + 'if necessary, it is recommended to make stale_days at least ' + 'twice the data_period.', stale_days, fresh_days) + + _remove_rotten_files(datadir, rotten_days, dry_run=args.dry_run) + _compress_stale_files(datadir, stale_days, dry_run=args.dry_run) + + if not args.no_v3bw: + _clean_v3bw_files(args, conf)

1 0

[sbws/master] Check conf options for cleaning v3bw files
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 1dbcd45262c1cbabcabb14596e8171e1497bf957 Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 17:13:45 2018 +0000 Check conf options for cleaning v3bw files --- sbws/core/cleanup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index d726a23..2099a62 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -155,6 +155,13 @@ def _compress_stale_files(datadir, stale_days, dry_run=True): os.remove(fname) +def _check_validity_periods_v3bw(compress_after_days, delete_after_days): + if 1 <= compress_after_days and compress_after_days < delete_after_days: + return True + fail_hard("v3bw files should only be compressed after 1 day and deleted " + "after a bigger number of days.") + + def main(args, conf): ''' Main entry point in to the cleanup command.

1 0

[sbws/master] Fix args for cleanup unit tests
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit c0ac5a3957abb15b8b307c4b4311b419f848e7ab Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 17:16:20 2018 +0000 Fix args for cleanup unit tests --- tests/unit/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 4eab658..7669318 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -119,7 +119,8 @@ def sbwshome_only_datadir(sbwshome_empty): def args(sbwshome_empty, parser): """Args with sbws home in the tests tmp dir.""" args = _PseudoArguments(directory=sbwshome_empty, output=sbwshome_empty, - scale=False, log_level='debug') + scale=False, log_level='debug', + no_results=False, no_v3bw=False) return args

1 0

[sbws/master] Clarify the extension is not in the list
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 758605703462a34edea9e69617c303903e017858 Author: juga0 <juga(a)riseup.net> Date: Mon Jul 23 20:00:41 2018 +0000 Clarify the extension is not in the list --- sbws/core/cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index 852b8e6..0a8daf3 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -53,7 +53,7 @@ def _get_files_mtime_older_than(dname, days_delta, extensions): fname = os.path.join(root, f) _, ext = os.path.splitext(fname) if ext not in extensions: - log.debug('Ignoring %s because it doesn\'t have extension ' + log.debug('Ignoring %s because its extension is not in ' '%s', fname, ext) continue # using file modification time instead of parsing the name

1 0

[sbws/master] Log list of allowed extensions, not the extension that the file has
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 84cb78d655e3fcf3e7d98bb9f157341e7a80ee34 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Aug 1 19:21:05 2018 -0400 Log list of allowed extensions, not the extension that the file has --- sbws/core/cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index d300997..3f677a0 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -54,7 +54,7 @@ def _get_files_mtime_older_than(dname, days_delta, extensions): _, ext = os.path.splitext(fname) if ext not in extensions: log.debug('Ignoring %s because its extension is not in ' - '%s', fname, ext) + '%s', fname, extensions) continue # using file modification time instead of parsing the name # of the file.

1 0