August 2018 - tor-commits - lists.torproject.org

[sbws/master] Combine Fixed sections in Unreleased
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit c36503c6e3d605157faf5a00464f0ee59a6e39d3 Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Aug 1 20:03:22 2018 -0400 Combine Fixed sections in Unreleased --- CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dd674f..ebb5bb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. (#26740) - State file for storing the last time `sbws scanner` was started, and able to be used for storing many other types of state in the future. (GH#166) +- Log files weren't rotating. Now they are. (#26881) ### Changed @@ -31,9 +32,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Stop using food terms for cleanup-related config options - Cleanup command now cleans up old v3bw files too (#26701) -### Fixed - -- Log files weren't rotating. Now they are. (#26881) ## [0.6.0] - 2018-07-11

1 0

[sbws/master] Remove sbws cleanup tests
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit c7ed801b14df4d1114c056d85d9f6e6e1f64f58f Author: Matt Traudt <sirmatt(a)ksu.edu> Date: Wed Aug 1 19:51:44 2018 -0400 Remove sbws cleanup tests --- tests/unit/core/test_cleanup.py | 236 ---------------------------------------- 1 file changed, 236 deletions(-) diff --git a/tests/unit/core/test_cleanup.py b/tests/unit/core/test_cleanup.py deleted file mode 100644 index 63735e2..0000000 --- a/tests/unit/core/test_cleanup.py +++ /dev/null @@ -1,236 +0,0 @@ -import pytest - -from sbws.globals import touch_file -import sbws.core.cleanup -from tests.unit.globals import monotonic_time -from unittest.mock import patch -import logging -import os - -log = logging.getLogger(__name__) - - -def test_cleanup_no_dotsbws(caplog, args, conf): - caplog.set_level(logging.DEBUG) - try: - sbws.core.cleanup.main(args, conf) - except SystemExit as e: - assert e.code == 1 - else: - assert None, 'Should have failed' - assert 'Try sbws init' in caplog.records[-1].getMessage() - - -def test_cleanup_no_datadir(sbwshome_config, args, conf, caplog): - ''' - An initialized but rather empty .sbws directory should fail about missing - ~/.sbws/datadir - ''' - try: - sbws.core.stats.main(args, conf) - except SystemExit as e: - assert e.code == 1 - else: - assert None, 'Should have failed' - assert '{}/datadir does not exist'.format( - os.path.abspath(sbwshome_config)) == caplog.records[-1].getMessage() - - -def test_cleanup_small_stale(sbwshome, caplog, args, conf): - conf['general']['data_period'] = '1' - conf['cleanup']['stale_days'] = '2' - conf['cleanup']['rotten_days'] = '3' - try: - sbws.core.cleanup.main(args, conf) - except SystemExit as e: - assert e.code == 1 - else: - assert None, 'Should have failed' - assert 'cleanup/stale_days (2) must be at least 2 days larger than ' +\ - 'general/data_period (1)' in caplog.records[-1].getMessage() - - -def test_cleanup_small_rotten(sbwshome, caplog, args, conf): - conf['general']['data_period'] = '1' - conf['cleanup']['stale_days'] = '5' - conf['cleanup']['rotten_days'] = '4' - try: - sbws.core.cleanup.main(args, conf) - except SystemExit as e: - assert e.code == 1 - else: - assert None, 'Should have failed' - assert 'cleanup/rotten_days (4) must be the same or larger than ' +\ - 'cleanup/stale_days (5)' in caplog.records[-1].getMessage() - - -def test_cleanup_medium_stale(sbwshome, caplog, args, conf): - args.dry_run = False - conf['general']['data_period'] = '10' - conf['cleanup']['stale_days'] = '19' - conf['cleanup']['rotten_days'] = '50' - sbws.core.cleanup.main(args, conf) - assert 'cleanup/stale_days (19) is less than twice ' +\ - 'general/data_period (10).' in caplog.records[-1].getMessage() - - -(a)pytest.mark.skip(reason="FIXME") -@patch('time.time') -def test_cleanup_only_compress_stale(time_mock, sbwshome, caplog, args, conf): - args.dry_run = True - caplog.set_level(logging.DEBUG) - conf['general']['data_period'] = '1' - conf['cleanup']['stale_days'] = '10' - conf['cleanup']['rotten_days'] = str(365*2) - now = 1041379200 # 1,041,379,200 is 1 Jan 2003 00:00:00 UTC - time_mock.side_effect = monotonic_time(start=now) - j = os.path.join - dd = j(sbwshome, 'datadir') - sub_a = j(dd, 'a') - sub_b = j(dd, 'b') - sub_ab = j(dd, 'a', 'b') - for dname in [sub_a, sub_b, sub_ab]: - os.makedirs(dname, exist_ok=True) - should_compress_fnames = [ - j(dd, '2002-01-01aaaa.txt'), - j(sub_a, '2002-10-01bbbb.txt'), - j(sub_b, '2002-10-10-cccc.txt'), - j(sub_a, '2002-10-10.dddd.txt'), - j(sub_ab, '2002-11-30.txt'), - ] - should_ignore_fnames = [ - j(sub_b, '2002-10-10.nottxt'), # wrong ext, should be ignored - j(sub_a, '200j-10-10.txt'), # not YYYY-MM-DD*.txt, should be ignored - ] - for fname in should_ignore_fnames + should_compress_fnames: - touch_file(fname) - sbws.core.cleanup.main(args, conf) - should_compress_fnames = [f + '.gz' for f in should_compress_fnames] - expected_fnames = should_ignore_fnames + should_compress_fnames +\ - [os.path.join(dd, '.lockfile')] - existing_fnames = [] - for root, dirs, files in os.walk(dd): - for fname in files: - existing_fnames.append(os.path.join(root, fname)) - expected_fnames.sort() - existing_fnames.sort() - assert expected_fnames == existing_fnames - - -@patch('time.time') -def test_cleanup_only_delete_rotten(time_mock, sbwshome, caplog, args, conf): - caplog.set_level(logging.DEBUG) - args.dry_run = False - conf['general']['data_period'] = '1' - conf['cleanup']['stale_days'] = '10' - conf['cleanup']['rotten_days'] = str(365*2) - now = 1041379200 # 1,041,379,200 is 1 Jan 2003 00:00:00 UTC - time_mock.side_effect = monotonic_time(start=now) - j = os.path.join - dd = j(sbwshome, 'datadir') - sub_a = j(dd, 'a') - sub_b = j(dd, 'b') - sub_ab = j(dd, 'a', 'b') - for dname in [sub_a, sub_b, sub_ab]: - os.makedirs(dname, exist_ok=True) - should_delete_fnames = [ - j(dd, '2000-01-01aaaa.txt'), - j(sub_a, '2000-10-01bbbb.txt'), - j(sub_b, '2000-10-10-cccc.txt'), - j(sub_a, '2000-10-10.dddd.txt'), - j(sub_a, '2000-10-11.eeee.txt.gz'), - j(dd, '2000-10-12.txt.gz'), - j(sub_ab, '2000-11-30.txt'), - ] - should_ignore_fnames = [ - j(dd, '2002-12-31.txt'), # too new, should be ignored - j(dd, '2003-01-01.txt'), # today, should be ignored - j(dd, '2003-02-10.txt'), # in the future, should be ignored - j(sub_b, '2000-10-10.nottxt'), # wrong ext, should be ignored - j(sub_a, '200j-10-10.txt'), # not YYYY-MM-DD*.txt, should be ignored - j(dd, '1999-1*-11.txt.gz'), # not YYYY-MM-DD*.txt.gz, should ignore - ] - for fname in should_ignore_fnames + should_delete_fnames: - touch_file(fname) - sbws.core.cleanup.main(args, conf) - expected_fnames = should_ignore_fnames + [os.path.join(dd, '.lockfile')] - existing_fnames = [] - for root, dirs, files in os.walk(dd): - for fname in files: - existing_fnames.append(os.path.join(root, fname)) - expected_fnames.sort() - existing_fnames.sort() - assert expected_fnames == existing_fnames - - -def test_cleanup_nothing_to_do(sbwshome, caplog, args, conf): - caplog.set_level(logging.DEBUG) - args.dry_run = False - try: - sbws.core.cleanup.main(args, conf) - except Exception as e: - assert None, 'Nothing bad should have happened, but this did: {}'\ - .format(e) - - -@patch('time.time') -def test_cleanup_compress_barely_stale(time_mock, sbwshome, caplog, args, - conf): - args.dry_run = False - caplog.set_level(logging.DEBUG) - conf['general']['data_period'] = '1' - conf['cleanup']['stale_days'] = '10' - conf['cleanup']['rotten_days'] = '30' - now = 1443571200 # 1,443,571,200 is 30 Sep 2015 00:00:00 UTC - time_mock.side_effect = monotonic_time(start=now) - dd = os.path.join(sbwshome, 'datadir') - fname_compress1 = os.path.join(dd, '2015-09-19.txt') - fname_compress2 = os.path.join(dd, '2015-09-20.txt') - fname_leave = os.path.join(dd, '2015-09-21.txt') - touch_file(fname_compress1) - touch_file(fname_compress2) - touch_file(fname_leave) - sbws.core.cleanup.main(args, conf) - expected_fnames = [ - fname_compress1 + '.gz', - fname_compress2 + '.gz', - fname_leave, - os.path.join(dd, '.lockfile'), - ] - existing_fnames = [] - for root, dirs, files in os.walk(dd): - for fname in files: - existing_fnames.append(os.path.join(root, fname)) - expected_fnames.sort() - existing_fnames.sort() - assert expected_fnames == existing_fnames - - -@patch('time.time') -def test_cleanup_delete_barely_rotten(time_mock, sbwshome, caplog, args, conf): - args.dry_run = False - caplog.set_level(logging.DEBUG) - conf['general']['data_period'] = '1' - conf['cleanup']['stale_days'] = '5' - conf['cleanup']['rotten_days'] = '20' - now = 1443571200 # 1,443,571,200 is 30 Sep 2015 00:00:00 UTC - time_mock.side_effect = monotonic_time(start=now) - dd = os.path.join(sbwshome, 'datadir') - fname_rotten1 = os.path.join(dd, '2015-09-09.txt') - fname_rotten2 = os.path.join(dd, '2015-09-10.txt') - fname_leave = os.path.join(dd, '2015-09-11.txt') - touch_file(fname_rotten1) - touch_file(fname_rotten2) - touch_file(fname_leave) - sbws.core.cleanup.main(args, conf) - expected_fnames = [ - fname_leave + '.gz', - os.path.join(dd, '.lockfile'), - ] - existing_fnames = [] - for root, dirs, files in os.walk(dd): - for fname in files: - existing_fnames.append(os.path.join(root, fname)) - expected_fnames.sort() - existing_fnames.sort() - assert expected_fnames == existing_fnames

1 0

[sbws/master] Add default sbws_home in config
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 74036c021b807e13683a1361194804527aba32b8 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 19:48:10 2018 +0000 Add default sbws_home in config --- sbws/config.default.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/sbws/config.default.ini b/sbws/config.default.ini index b8404da..17e3a25 100644 --- a/sbws/config.default.ini +++ b/sbws/config.default.ini @@ -1,4 +1,5 @@ [paths] +sbws_home = ~/.sbws datadir = ${sbws_home}/datadir v3bw_dname = ${sbws_home}/v3bw # The latest bandwidth file is atomically symlinked to

1 0

[sbws/master] Add instead minimal user config
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 860ec46ca6203c8c0b79681b08323c1e5ed40554 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 19:49:09 2018 +0000 Add instead minimal user config --- sbws/config.user.ini | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sbws/config.user.ini b/sbws/config.user.ini new file mode 100644 index 0000000..f3ca1da --- /dev/null +++ b/sbws/config.user.ini @@ -0,0 +1,10 @@ +# Minimum configuration that needs to be customized +[scanner] +# A human-readable string with chars in a-zA-Z0-9 to identify your scanner +nickname = sbws_default + +[destinations] +foo = off + +[destinations.foo] +url = https://example.com/does/not/exist.bin

1 0

[sbws/master] Remove unused config example
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 6c4b9174a923b98ea4c2ce0ce15af2942fe7e6d8 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 19:48:45 2018 +0000 Remove unused config example --- sbws/config.example.ini | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sbws/config.example.ini b/sbws/config.example.ini deleted file mode 100644 index 4fb7d3b..0000000 --- a/sbws/config.example.ini +++ /dev/null @@ -1,6 +0,0 @@ -# In `sbws init`, this is the starting point for the client's config that -# by default ends up at $HOME/.sbws/config.ini. -# -# Comments in here will not survive `sbws init`. -# -# Strongly prefer to put options in config.default.ini instead.

1 0

[sbws/master] Remove is_innited dependency
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit c21232f3407a30607e7e0bff8f954d183f8f02c8 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 19:50:16 2018 +0000 Remove is_innited dependency --- sbws/core/cleanup.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index 18d8380..4600cf8 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -2,7 +2,7 @@ import types from sbws.util.filelock import DirectoryLock -from sbws.globals import (fail_hard, is_initted) +from sbws.globals import fail_hard from sbws.util.timestamp import unixts_to_dt_obj from argparse import ArgumentDefaultsHelpFormatter from datetime import datetime @@ -174,11 +174,9 @@ def main(args, conf): :param argparse.Namespace args: command line arguments :param configparser.ConfigParser conf: parsed config files ''' - if not is_initted(args.directory): - fail_hard('Sbws isn\'t initialized. Try sbws init') - - if args.no_results and args.no_v3bw: - fail_hard('Nothing to clean.') + datadir = conf['paths']['datadir'] + if not os.path.isdir(datadir): + fail_hard('%s does not exist', datadir) if not args.no_results: _clean_result_files(args, conf)

1 0

[tor/release-0.3.4] Merge branch 'maint-0.2.9' into maint-0.3.2
by nickm＠torproject.org 09 Aug '18

09 Aug '18

commit 8982719f6ac06383c26283b60acf6b3d3fde613e Merge: 8e68fe7e1 fe19b40fe Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 9 08:47:24 2018 -0400 Merge branch 'maint-0.2.9' into maint-0.3.2 changes/geoip-2018-08-07 | 4 + src/config/geoip | 14291 ++++++++++++++++++++++++++------------------- src/config/geoip6 | 974 ++- 3 files changed, 9063 insertions(+), 6206 deletions(-)

1 0

[tor/release-0.3.4] Merge branch 'maint-0.3.3' into maint-0.3.4
by nickm＠torproject.org 09 Aug '18

09 Aug '18

commit b4362b99cf1857f2fdf4390796bbf64ac9f204f3 Merge: 0f0dac0bf fde551a38 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 9 08:47:25 2018 -0400 Merge branch 'maint-0.3.3' into maint-0.3.4 changes/geoip-2018-08-07 | 4 + src/config/geoip | 14291 ++++++++++++++++++++++++++------------------- src/config/geoip6 | 974 ++- 3 files changed, 9063 insertions(+), 6206 deletions(-)

1 0

[tor/release-0.3.4] Merge branch 'maint-0.3.2' into maint-0.3.3
by nickm＠torproject.org 09 Aug '18

09 Aug '18

commit fde551a387bf743dd974512f63405c6e63de49b4 Merge: c4d0d9bd2 8982719f6 Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 9 08:47:25 2018 -0400 Merge branch 'maint-0.3.2' into maint-0.3.3 changes/geoip-2018-08-07 | 4 + src/config/geoip | 14291 ++++++++++++++++++++++++++------------------- src/config/geoip6 | 974 ++- 3 files changed, 9063 insertions(+), 6206 deletions(-)

1 0

[tor/release-0.3.4] Merge branch 'maint-0.3.4' into release-0.3.4
by nickm＠torproject.org 09 Aug '18

09 Aug '18

commit 00dc2f7ee8be42fa35e2a6eabf32e98436771935 Merge: e1c878731 b4362b99c Author: Nick Mathewson <nickm(a)torproject.org> Date: Thu Aug 9 08:47:26 2018 -0400 Merge branch 'maint-0.3.4' into release-0.3.4 changes/geoip-2018-08-07 | 4 + src/config/geoip | 14291 ++++++++++++++++++++++++++------------------- src/config/geoip6 | 974 ++- 3 files changed, 9063 insertions(+), 6206 deletions(-)

1 0