August 2018 - tor-commits - lists.torproject.org

[sbws/master] Log list of allowed extensions, not the extension that the file has
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 84cb78d655e3fcf3e7d98bb9f157341e7a80ee34 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Aug 1 19:21:05 2018 -0400 Log list of allowed extensions, not the extension that the file has --- sbws/core/cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index d300997..3f677a0 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -54,7 +54,7 @@ def _get_files_mtime_older_than(dname, days_delta, extensions): _, ext = os.path.splitext(fname) if ext not in extensions: log.debug('Ignoring %s because its extension is not in ' - '%s', fname, ext) + '%s', fname, extensions) continue # using file modification time instead of parsing the name # of the file.

1 0

[sbws/master] Python considers the extensions of "foo.bar.gz" to be ".gz"
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit a30be62758656ed2f0eedf25868d4a6a9032effb Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Aug 1 19:21:23 2018 -0400 Python considers the extensions of "foo.bar.gz" to be ".gz" Also this was missing the leading "." --- sbws/core/cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index 3f677a0..fe529a7 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -174,7 +174,7 @@ def _clean_v3bw_files(args, conf): # first delete so that the files to be deleted are not compressed first files_to_delete = _get_files_mtime_older_than(v3bw_dname, delete_after_days, - ['.v3bw', 'v3bw.gz']) + ['.v3bw', '.gz']) _delete_files(v3bw_dname, files_to_delete, dry_run=args.dry_run) files_to_compress = _get_files_mtime_older_than(v3bw_dname, compress_after_days,

1 0

[sbws/master] Delete also v3bw.gz files
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 846470f8ad5aee87bd7e70e065dbbea2eeb64323 Author: juga0 <juga(a)riseup.net> Date: Mon Jul 23 20:01:03 2018 +0000 Delete also v3bw.gz files --- sbws/core/cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index 0a8daf3..d300997 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -174,7 +174,7 @@ def _clean_v3bw_files(args, conf): # first delete so that the files to be deleted are not compressed first files_to_delete = _get_files_mtime_older_than(v3bw_dname, delete_after_days, - ['.v3bw']) + ['.v3bw', 'v3bw.gz']) _delete_files(v3bw_dname, files_to_delete, dry_run=args.dry_run) files_to_compress = _get_files_mtime_older_than(v3bw_dname, compress_after_days,

1 0

[sbws/master] Add function to clean v3bw files
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 451d7f34ef7c21da22771c4d22cafffb44f618de Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 17:14:43 2018 +0000 Add function to clean v3bw files --- sbws/core/cleanup.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index 2099a62..04823b1 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -162,6 +162,28 @@ def _check_validity_periods_v3bw(compress_after_days, delete_after_days): "after a bigger number of days.") +def _clean_v3bw_files(args, conf): + v3bw_dname = conf['paths']['v3bw_dname'] + if not os.path.isdir(v3bw_dname): + fail_hard('%s does not exist', v3bw_dname) + compress_after_days = conf.getint('cleanup', + 'v3bw_files_compress_after_days') + delete_after_days = conf.getint('cleanup', + 'v3bw_files_delete_after_days') + _check_validity_periods_v3bw(compress_after_days, delete_after_days) + # first delete so that the files to be deleted are not compressed first + files_to_delete = _get_files_mtime_older_than(v3bw_dname, + delete_after_days, + ['.v3bw']) + _delete_files(v3bw_dname, files_to_delete, dry_run=args.dry_run) + files_to_compress = _get_files_mtime_older_than(v3bw_dname, + compress_after_days, + ['.v3bw']) + # when dry_run is true, compress will also show all the files that + # would have been deleted, since they are not really deleted + _compress_files(v3bw_dname, files_to_compress, dry_run=args.dry_run) + + def main(args, conf): ''' Main entry point in to the cleanup command.

1 0

[sbws/master] Don't condition returning a file name on os.path.splitext existing
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 7866642820881fabe5c4758856a7eb0c4cb738ee Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Aug 1 19:23:38 2018 -0400 Don't condition returning a file name on os.path.splitext existing --- sbws/core/cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index fe529a7..a3e8cc6 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -60,7 +60,7 @@ def _get_files_mtime_older_than(dname, days_delta, extensions): # of the file. filedt = unixts_to_dt_obj( os.stat(fname, follow_symlinks=False).st_mtime) - if filedt < oldest_day and os.path.splitext: + if filedt < oldest_day: yield fname

1 0

[sbws/master] Revert "Add arg to clean v3bw files in tests"
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 98c8fdd1b1a78d7678400c9ce35a8c4b8c328675 Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 16:13:49 2018 +0000 Revert "Add arg to clean v3bw files in tests" This reverts commit edec4a38bb432f5cd697b7ec44b5e6a477b4074e. To do not modify existing cleaning code for result files. --- tests/unit/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 308eace..4eab658 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -119,7 +119,7 @@ def sbwshome_only_datadir(sbwshome_empty): def args(sbwshome_empty, parser): """Args with sbws home in the tests tmp dir.""" args = _PseudoArguments(directory=sbwshome_empty, output=sbwshome_empty, - scale=False, log_level='debug', v3bw=False) + scale=False, log_level='debug') return args

1 0

[sbws/master] Add function to compress files passed as arg
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 90d29317d2bcbaa4f61c9063448759d3749c4a00 Author: juga0 <juga(a)riseup.net> Date: Tue Jul 17 17:09:09 2018 +0000 Add function to compress files passed as arg --- sbws/core/cleanup.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index 3deca57..d726a23 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -121,6 +121,22 @@ def _remove_rotten_files(datadir, rotten_days, dry_run=True): os.remove(fname) +def _compress_files(dname, files, dry_run=True): + """Compress the files passed as argument.""" + assert os.path.isdir(dname) + assert isinstance(files, types.GeneratorType) + with DirectoryLock(dname): + for fname in files: + log.info('Compressing %s', fname) + if dry_run: + continue + with open(fname, 'rt') as in_fd: + out_fname = fname + '.gz' + with gzip.open(out_fname, 'wt') as out_fd: + shutil.copyfileobj(in_fd, out_fd) + os.remove(fname) + + def _compress_stale_files(datadir, stale_days, dry_run=True): assert os.path.isdir(datadir) assert isinstance(stale_days, int)

1 0

[sbws/master] Require files be in the given directory
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit c78c93b3576b877a1b181c5374db757c34da0f44 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Aug 1 19:25:25 2018 -0400 Require files be in the given directory --- sbws/core/cleanup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index a3e8cc6..ad5b286 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -103,6 +103,7 @@ def _delete_files(dname, files, dry_run=True): with DirectoryLock(dname): for fname in files: log.info('Deleting %s', fname) + assert os.path.commonprefix([dname, fname]) == dname if not dry_run: os.remove(fname) @@ -128,6 +129,7 @@ def _compress_files(dname, files, dry_run=True): with DirectoryLock(dname): for fname in files: log.info('Compressing %s', fname) + assert os.path.commonprefix([dname, fname]) == dname if dry_run: continue with open(fname, 'rt') as in_fd:

1 0

[sbws/master] Use juga's code to cleanup result files too
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 3a1789bddf7aecd74a9ed784f38ebb1a9f45f2de Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Aug 1 19:35:53 2018 -0400 Use juga's code to cleanup result files too --- sbws/core/cleanup.py | 132 +++++++++++++++++---------------------------------- 1 file changed, 43 insertions(+), 89 deletions(-) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index ad5b286..7dc7e78 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -7,7 +7,6 @@ from sbws.util.timestamp import unixts_to_dt_obj from argparse import ArgumentDefaultsHelpFormatter from datetime import datetime from datetime import timedelta -import re import os import gzip import shutil @@ -64,38 +63,6 @@ def _get_files_mtime_older_than(dname, days_delta, extensions): yield fname -def _get_older_files_than(dname, num_days_ago, extensions): - assert os.path.isdir(dname) - assert isinstance(num_days_ago, int) - assert isinstance(extensions, list) - for ext in extensions: - assert isinstance(ext, str) - assert ext[0] == '.' - # Determine oldest allowed date - today = datetime.utcfromtimestamp(time.time()) - oldest_day = today - timedelta(days=num_days_ago) - # Compile a regex that can extract a date from a file name that looks like - # /path/to/foo/YYYY-MM-DD*.extension - extensions = [re.escape(e) for e in extensions] - day_part = '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' - regex = re.compile(r'^.*/({}).*({})$' - .format(day_part, '|'.join(extensions))) - # Walk through all files in the given dname, find files that match the - # regex, and yield the ones that contain a date in the file name that is - # too old. - for root, dirs, files in os.walk(dname): - for f in files: - fname = os.path.join(root, f) - match = regex.match(fname) - if not match: - log.debug('Ignoring %s because it doesn\'t look like ' - 'YYYY-MM-DD', fname) - continue - d = datetime(*[int(n) for n in match.group(1).split('-')]) - if d < oldest_day: - yield fname - - def _delete_files(dname, files, dry_run=True): """Delete the files passed as argument.""" assert os.path.isdir(dname) @@ -108,20 +75,6 @@ def _delete_files(dname, files, dry_run=True): os.remove(fname) -def _remove_rotten_files(datadir, rotten_days, dry_run=True): - assert os.path.isdir(datadir) - assert isinstance(rotten_days, int) - # Hold the lock for basically the entire time just in case someone else - # moves files between when we get the list of files and when we try to - # delete them. - with DirectoryLock(datadir): - for fname in _get_older_files_than(datadir, rotten_days, - ['.txt', '.txt.gz']): - log.info('Deleting %s', fname) - if not dry_run: - os.remove(fname) - - def _compress_files(dname, files, dry_run=True): """Compress the files passed as argument.""" assert os.path.isdir(dname) @@ -139,24 +92,6 @@ def _compress_files(dname, files, dry_run=True): os.remove(fname) -def _compress_stale_files(datadir, stale_days, dry_run=True): - assert os.path.isdir(datadir) - assert isinstance(stale_days, int) - # Hold the lock for basically the entire time just in case someone else - # moves files between when we get the list of files and when we try to - # compress them. - with DirectoryLock(datadir): - for fname in _get_older_files_than(datadir, stale_days, ['.txt']): - log.info('Compressing %s', fname) - if dry_run: - continue - with open(fname, 'rt') as in_fd: - out_fname = fname + '.gz' - with gzip.open(out_fname, 'wt') as out_fd: - shutil.copyfileobj(in_fd, out_fd) - os.remove(fname) - - def _check_validity_periods_v3bw(compress_after_days, delete_after_days): if 1 <= compress_after_days and compress_after_days < delete_after_days: return True @@ -164,6 +99,26 @@ def _check_validity_periods_v3bw(compress_after_days, delete_after_days): "after a bigger number of days.") +def _check_validity_periods_results( + data_period, compress_after_days, delete_after_days): + if compress_after_days - 2 < data_period: + fail_hard( + 'For safetly, cleanup/stale_days (%d) must be at least 2 days ' + 'larger than general/data_period (%d)', compress_after_days, + data_period) + if delete_after_days < compress_after_days: + fail_hard( + 'cleanup/rotten_days (%d) must be the same or larger than ' + 'cleanup/stale_days (%d)', delete_after_days, compress_after_days) + if compress_after_days / 2 < data_period: + log.warning( + 'cleanup/stale_days (%d) is less than twice ' + 'general/data_period (%d). For ease of parsing older results ' + 'if necessary, it is recommended to make stale_days at least ' + 'twice the data_period.', compress_after_days, data_period) + return True + + def _clean_v3bw_files(args, conf): v3bw_dname = conf['paths']['v3bw_dname'] if not os.path.isdir(v3bw_dname): @@ -186,6 +141,28 @@ def _clean_v3bw_files(args, conf): _compress_files(v3bw_dname, files_to_compress, dry_run=args.dry_run) +def _clean_result_files(args, conf): + datadir = conf['paths']['datadir'] + if not os.path.isdir(datadir): + fail_hard('%s does not exist', datadir) + data_period = conf.getint('general', 'data_period') + compress_after_days = conf.getint('cleanup', 'stale_days') + delete_after_days = conf.getint('cleanup', 'rotten_days') + _check_validity_periods_results( + data_period, compress_after_days, delete_after_days) + + # first delete so that the files to be deleted are not compressed first + files_to_delete = _get_files_mtime_older_than( + datadir, delete_after_days, ['.txt', '.gz']) + _delete_files(datadir, files_to_delete, dry_run=args.dry_run) + + # when dry_run is true, compress will also show all the files that + # would have been deleted, since they are not really deleted + files_to_compress = _get_files_mtime_older_than( + datadir, compress_after_days, ['.txt']) + _compress_files(datadir, files_to_compress, dry_run=args.dry_run) + + def main(args, conf): ''' Main entry point in to the cleanup command. @@ -200,30 +177,7 @@ def main(args, conf): fail_hard('Nothing to clean.') if not args.no_results: - datadir = conf['paths']['datadir'] - if not os.path.isdir(datadir): - fail_hard('%s does not exist', datadir) - - fresh_days = conf.getint('general', 'data_period') - stale_days = conf.getint('cleanup', 'stale_days') - rotten_days = conf.getint('cleanup', 'rotten_days') - if stale_days - 2 < fresh_days: - fail_hard('For safetly, cleanup/stale_days (%d) must be at least ' - '2 days larger than general/data_period (%d)', - stale_days, fresh_days) - if rotten_days < stale_days: - fail_hard('cleanup/rotten_days (%d) must be the same or larger ' - 'than cleanup/stale_days (%d)', rotten_days, stale_days) - - if stale_days / 2 < fresh_days: - log.warning( - 'cleanup/stale_days (%d) is less than twice ' - 'general/data_period (%d). For ease of parsing older results ' - 'if necessary, it is recommended to make stale_days at least ' - 'twice the data_period.', stale_days, fresh_days) - - _remove_rotten_files(datadir, rotten_days, dry_run=args.dry_run) - _compress_stale_files(datadir, stale_days, dry_run=args.dry_run) + _clean_result_files(args, conf) if not args.no_v3bw: _clean_v3bw_files(args, conf)

1 0

[sbws/master] Stop using food terms
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 410d31e5fe1cc654597c99fe39426bac4d3af878 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Aug 1 19:39:49 2018 -0400 Stop using food terms --- sbws/config.default.ini | 4 ++-- sbws/core/cleanup.py | 24 ++++++++++++++---------- sbws/util/config.py | 4 ++-- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index fd6a58c..b8404da 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -63,9 +63,9 @@ extra_lines = [cleanup] # After this many days, compress data files -stale_days = 10 +data_files_compress_after_days = 10 # After this many days, delete data files -rotten_days = 90 +data_files_delete_after_days = 90 # After this many days, compress v3bw files (1d) v3bw_files_compress_after_days = 1 # After this many days, delete v3bw files (7d) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index 7dc7e78..18d8380 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -103,19 +103,21 @@ def _check_validity_periods_results( data_period, compress_after_days, delete_after_days): if compress_after_days - 2 < data_period: fail_hard( - 'For safetly, cleanup/stale_days (%d) must be at least 2 days ' - 'larger than general/data_period (%d)', compress_after_days, - data_period) + 'For safetly, cleanup/data_files_compress_after_days (%d) must be ' + 'at least 2 days larger than general/data_period (%d)', + compress_after_days, data_period) if delete_after_days < compress_after_days: fail_hard( - 'cleanup/rotten_days (%d) must be the same or larger than ' - 'cleanup/stale_days (%d)', delete_after_days, compress_after_days) + 'cleanup/data_files_delete_after_days (%d) must be the same or ' + 'larger than cleanup/data_files_compress_after_days (%d)', + delete_after_days, compress_after_days) if compress_after_days / 2 < data_period: log.warning( - 'cleanup/stale_days (%d) is less than twice ' + 'cleanup/data_files_compress_after_days (%d) is less than twice ' 'general/data_period (%d). For ease of parsing older results ' - 'if necessary, it is recommended to make stale_days at least ' - 'twice the data_period.', compress_after_days, data_period) + 'if necessary, it is recommended to make ' + 'data_files_compress_after_days at least twice the data_period.', + compress_after_days, data_period) return True @@ -146,8 +148,10 @@ def _clean_result_files(args, conf): if not os.path.isdir(datadir): fail_hard('%s does not exist', datadir) data_period = conf.getint('general', 'data_period') - compress_after_days = conf.getint('cleanup', 'stale_days') - delete_after_days = conf.getint('cleanup', 'rotten_days') + compress_after_days = conf.getint( + 'cleanup', 'data_files_compress_after_days') + delete_after_days = conf.getint( + 'cleanup', 'data_files_delete_after_days') _check_validity_periods_results( data_period, compress_after_days, delete_after_days) diff --git a/sbws/util/config.py b/sbws/util/config.py index 82ce0ec..2898d1b 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -172,8 +172,8 @@ def _validate_cleanup(conf): sec = 'cleanup' err_tmpl = Template('$sec/$key ($val): $e') ints = { - 'stale_days': {'minimum': 1, 'maximum': None}, - 'rotten_days': {'minimum': 1, 'maximum': None}, + 'data_files_compress_after_days': {'minimum': 1, 'maximum': None}, + 'data_files_delete_after_days': {'minimum': 1, 'maximum': None}, 'v3bw_files_compress_after_days': {'minimum': 1, 'maximum': None}, 'v3bw_files_delete_after_days': {'minimum': 1, 'maximum': None}, }

1 0