August 2018 - tor-commits - lists.torproject.org

[sbws/master] Use config default paths
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 818f72ed6575ed45b4549dd96a24100630f1b4a2 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 20:06:58 2018 +0000 Use config default paths and create minimal user config when it does not exist. --- sbws/util/config.py | 67 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/sbws/util/config.py b/sbws/util/config.py index 2898d1b..3171ad2 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -1,3 +1,6 @@ +"""Util functions to manage sbws configuration files.""" +import shutil + from configparser import (ConfigParser, ExtendedInterpolation) from configparser import InterpolationMissingOptionError import os @@ -6,7 +9,8 @@ import logging.config from urllib.parse import urlparse from string import Template from tempfile import NamedTemporaryFile -from sbws.globals import PKG_DIR +from sbws.globals import (DEFAULT_CONFIG_PATH, DEFAULT_LOG_CONFIG_PATH, + MINIMUM_USER_CONFIG_PATH, USER_CONFIG_PATH) _ALPHANUM = 'abcdefghijklmnopqrstuvwxyz' _ALPHANUM += _ALPHANUM.upper() @@ -21,6 +25,12 @@ _LOG_LEVELS = ['debug', 'info', 'warning', 'error', 'critical'] log = logging.getLogger(__name__) +def _create_user_config_file(fname): + """Copy minimal user config to user config path.""" + if not os.path.isfile(fname): + shutil.copyfile(MINIMUM_USER_CONFIG_PATH, fname) + + def _read_config_file(conf, fname): assert os.path.isfile(fname) log.debug('Reading config file %s', fname) @@ -29,52 +39,57 @@ def _read_config_file(conf, fname): return conf +def _extend_config(conf, fpath): + """Extend ConfigParser from file configuration.""" + conf = _read_config_file(conf, fpath) + return conf + + def _get_default_config(): + """Return ConfigParser with default configuration.""" conf = ConfigParser(interpolation=ExtendedInterpolation()) - fname = os.path.join(PKG_DIR, 'config.default.ini') - assert os.path.isfile(fname) - conf = _read_config_file(conf, fname) - return conf + return _read_config_file(conf, DEFAULT_CONFIG_PATH) def _get_user_config(args, conf=None): + """Get user configuration. + If a custom path is passed, search for user configuration there. + In any case, create a minimal user configuration in user configuration path + , in case it needs to be overwritten. + """ if not conf: conf = ConfigParser(interpolation=ExtendedInterpolation()) else: assert isinstance(conf, ConfigParser) - fname = os.path.join(args.directory, 'config.ini') - if not os.path.isfile(fname): - return conf - conf = _read_config_file(conf, fname) - return conf - - -def _get_default_logging_config(args, conf=None): + if args.config: + if not os.path.isfile(args.config): + log.warning('Configuration file %s not found, ' + 'using default configuration, and creating a minimal ' + 'configuration in %s.', args.config, args.config) + _create_user_config_file(args.config) + return conf + return _extend_config(conf, args.config) + _create_user_config_file(USER_CONFIG_PATH) + return _extend_config(conf, USER_CONFIG_PATH) + + +def _get_default_logging_config(conf=None): + """Get default logging configuration.""" if not conf: conf = ConfigParser(interpolation=ExtendedInterpolation()) else: assert isinstance(conf, ConfigParser) - fname = os.path.join(PKG_DIR, 'config.log.default.ini') - assert os.path.isfile(fname) - conf = _read_config_file(conf, fname) - return conf + return _extend_config(conf, DEFAULT_LOG_CONFIG_PATH) def get_config(args): + """Get ConfigParser interpolating all configuration files.""" conf = _get_default_config() - conf = _get_default_logging_config(args, conf=conf) + conf = _get_default_logging_config(conf=conf) conf = _get_user_config(args, conf=conf) return conf -def get_user_example_config(): - conf = ConfigParser(interpolation=ExtendedInterpolation()) - fname = os.path.join(PKG_DIR, 'config.example.ini') - assert os.path.isfile(fname) - conf = _read_config_file(conf, fname) - return conf - - def _can_log_to_file(conf): ''' Checks all the known reasons for why we might not be able to log to a file,

1 0

[sbws/master] Remove is_innited dependency
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 14a2d956e8d61fa83d9671b4e0e9da5280ff62a7 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 19:50:37 2018 +0000 Remove is_innited dependency --- sbws/core/generate.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 481529c..f345f11 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -1,4 +1,4 @@ -from sbws.globals import (fail_hard, is_initted, SCALE_CONSTANT) +from sbws.globals import (fail_hard, SCALE_CONSTANT) from sbws.lib.v3bwfile import V3BWFile from sbws.lib.resultdump import load_recent_results_in_datadir from argparse import ArgumentDefaultsHelpFormatter @@ -39,9 +39,6 @@ def gen_parser(sub): def main(args, conf): - if not is_initted(args.directory): - fail_hard('Sbws isn\'t initialized. Try sbws init') - os.makedirs(conf['paths']['v3bw_dname'], exist_ok=True) datadir = conf['paths']['datadir']

1 0

[sbws/master] Remove is_innited dependency
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 88e1f05cf45d4d7bc98a023aa468bde346a98eff Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 19:51:57 2018 +0000 Remove is_innited dependency --- sbws/core/stats.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sbws/core/stats.py b/sbws/core/stats.py index 2b497ce..38f8978 100644 --- a/sbws/core/stats.py +++ b/sbws/core/stats.py @@ -1,4 +1,4 @@ -from sbws.globals import (fail_hard, is_initted) +from sbws.globals import fail_hard from sbws.lib.resultdump import Result from sbws.lib.resultdump import ResultError from sbws.lib.resultdump import ResultErrorCircuit @@ -155,8 +155,6 @@ def main(args, conf): :param argparse.Namespace args: command line arguments :param configparser.ConfigParser conf: parsed config files ''' - if not is_initted(args.directory): - fail_hard('Sbws isn\'t initialized. Try sbws init') datadir = conf['paths']['datadir'] if not os.path.isdir(datadir):

1 0

[sbws/master] Remove init dependency and create default dirs
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit b28c6c6fdbc8f348463ac0f301993084e136abf2 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 19:56:58 2018 +0000 Remove init dependency and create default dirs --- sbws/sbws.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sbws/sbws.py b/sbws/sbws.py index a97e428..64d126c 100644 --- a/sbws/sbws.py +++ b/sbws/sbws.py @@ -1,7 +1,8 @@ +import os + import sbws.core.cleanup import sbws.core.scanner import sbws.core.generate -import sbws.core.init import sbws.core.stats from sbws.util.config import get_config from sbws.util.config import validate_config @@ -16,6 +17,13 @@ import logging log = logging.getLogger(__name__) +def _ensure_dirs(conf): + log.debug('Ensuring all dirs exits.') + os.makedirs(conf['paths']['datadir']) + os.makedirs(conf['paths']['v3bw']) + os.makedirs(conf['paths']['log']) + + def _adjust_log_level(args, conf): if not args.log_level: return @@ -33,6 +41,7 @@ def main(): parser = create_parser() args = parser.parse_args() conf = get_config(args) + _ensure_dirs(conf) _adjust_log_level(args, conf) conf_valid, conf_errors = validate_config(conf) if not conf_valid: @@ -49,8 +58,6 @@ def main(): 'a': def_args, 'kw': def_kwargs}, 'generate': {'f': sbws.core.generate.main, 'a': def_args, 'kw': def_kwargs}, - 'init': {'f': sbws.core.init.main, - 'a': def_args, 'kw': def_kwargs}, 'stats': {'f': sbws.core.stats.main, 'a': def_args, 'kw': def_kwargs}, }

1 0

[sbws/master] Remove init and add config arg
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 15d1748764d4c4fd0daa04f3212639bbc22b4742 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 19:57:41 2018 +0000 Remove init and add config arg --- sbws/util/parser.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sbws/util/parser.py b/sbws/util/parser.py index f2fc828..1ef15cc 100644 --- a/sbws/util/parser.py +++ b/sbws/util/parser.py @@ -1,7 +1,6 @@ import sbws.core.cleanup import sbws.core.scanner import sbws.core.generate -import sbws.core.init import sbws.core.stats from sbws import __version__ @@ -22,12 +21,11 @@ def create_parser(): p.add_argument('--log-level', choices=['debug', 'info', 'warning', 'error', 'critical'], help='Override the sbws log level') - p.add_argument('-d', '--directory', default=_default_dot_sbws_dname(), - help='Name of the .sbws directory') + p.add_argument('-c', '--config', + help='Path the sbws config file') sub = p.add_subparsers(dest='command') sbws.core.cleanup.gen_parser(sub) sbws.core.scanner.gen_parser(sub) sbws.core.generate.gen_parser(sub) - sbws.core.init.gen_parser(sub) sbws.core.stats.gen_parser(sub) return p

1 0

[sbws/master] Remove init
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit c8eb041dc98e8d1536d458f4a3414a0472fa7b36 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 20:07:48 2018 +0000 Remove init --- sbws/core/init.py | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/sbws/core/init.py b/sbws/core/init.py deleted file mode 100644 index 39d4c17..0000000 --- a/sbws/core/init.py +++ /dev/null @@ -1,40 +0,0 @@ -from sbws.globals import (is_initted, fail_hard) -from sbws.util.config import get_user_example_config -from sbws.util.userquery import query_yes_no -from argparse import ArgumentDefaultsHelpFormatter -import os -import logging - -log = logging.getLogger(__name__) - - -def gen_parser(sub): - d = 'Initialize a directory so sbws can use it for configuration, '\ - 'data storage, etc. A common default directory is ~/.sbws but check '\ - 'the output of sbws -h to verify.' - sub.add_parser('init', formatter_class=ArgumentDefaultsHelpFormatter, - description=d) - - -def main(args, conf): - if is_initted(args.directory): - fail_hard('Directory already seems to be initted') - - if not os.path.isdir(args.directory): - log.info('Creating %s', args.directory) - os.makedirs(args.directory, exist_ok=False) - - # Create config.ini #### - fname = os.path.join(args.directory, 'config.ini') - if os.path.exists(fname) and not os.path.isfile(fname): - fail_hard('Don\'t know how to handle %s existing as a non-file', fname) - if os.path.isfile(fname) and not query_yes_no( - 'Is it okay to overwrite {}?'.format(fname), default=None): - fail_hard('Cannot continue') - c = get_user_example_config() - if 'paths' not in c: - c['paths'] = {} - c['paths']['sbws_home'] = args.directory - log.info('Creating %s based on example config', fname) - with open(fname, 'wt') as fd: - c.write(fd)

1 0

[sbws/master] Remove init and sbwshome with config
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 10e351a52f6bcba93e31ffbc1516f2023e348181 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 20:10:46 2018 +0000 Remove init and sbwshome with config --- tests/unit/core/test_stats.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/tests/unit/core/test_stats.py b/tests/unit/core/test_stats.py index b8b3fff..fb29117 100644 --- a/tests/unit/core/test_stats.py +++ b/tests/unit/core/test_stats.py @@ -1,27 +1,12 @@ import os.path -import sbws.core.init import sbws.core.stats from tests.unit.globals import monotonic_time from unittest.mock import patch import logging -def test_stats_uninitted(sbwshome_empty, args, conf, caplog): - ''' - An un-initialized .sbws directory should fail hard and exit immediately - ''' - try: - sbws.core.stats.main(args, conf) - except SystemExit as e: - assert e.code == 1 - else: - assert None, 'Should have failed' - assert 'Sbws isn\'t initialized. Try sbws init' == \ - caplog.records[-1].getMessage() - - -def test_stats_initted(sbwshome_config, args, conf, caplog): +def test_stats_initted(sbwshome_empty, args, conf, caplog): ''' An initialized but rather empty .sbws directory should fail about missing ~/.sbws/datadir @@ -33,10 +18,10 @@ def test_stats_initted(sbwshome_config, args, conf, caplog): else: assert None, 'Should have failed' assert '{}/datadir does not exist'.format( - os.path.abspath(sbwshome_config)) == caplog.records[-1].getMessage() + os.path.abspath(sbwshome_empty)) == caplog.records[-1].getMessage() -def test_stats_stale_result(sbwshome, args, conf, caplog, +def test_stats_stale_result(args, conf, caplog, sbwshome_success_result): ''' An initialized .sbws directory with no fresh results should say so and

1 0

[sbws/master] Remove init
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 53ab6eeb34a69755f0991ee6605bcb7414cdb081 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 20:11:06 2018 +0000 Remove init --- tests/unit/core/test_init.py | 45 -------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/tests/unit/core/test_init.py b/tests/unit/core/test_init.py deleted file mode 100644 index 77f3b0a..0000000 --- a/tests/unit/core/test_init.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Unit tests for sbws.core.init.""" -import logging -import os.path - -import sbws - - -def test_sbwshome_only_datadir(sbwshome_only_datadir, args, conf, caplog): - caplog.set_level(logging.DEBUG) - sbws.core.init.main(args, conf) - print(caplog.records[-1].getMessage()) - assert "Creating {} based on example config".format( - os.path.join(conf['paths']['sbws_home'], 'config.ini')) \ - in caplog.records[-1].getMessage() - assert os.path.isdir(conf['paths']['sbws_home']) - assert os.path.isdir(conf['paths']['datadir']) - assert os.path.isfile(os.path.join(conf['paths']['sbws_home'], - 'config.ini')) - - -def test_sbwshome_empty(args, conf, caplog): - caplog.set_level(logging.DEBUG) - sbws.core.init.main(args, conf) - assert "Creating {} based on example config".format( - os.path.join(conf['paths']['sbws_home'], 'config.ini')) \ - in caplog.records[-1].getMessage() - assert os.path.isdir(conf['paths']['sbws_home']) - assert os.path.isfile(os.path.join(conf['paths']['sbws_home'], - 'config.ini')) - - -def test_sbwshome(sbwshome, args, conf, caplog): - caplog.set_level(logging.DEBUG) - try: - sbws.core.init.main(args, conf) - except SystemExit as e: - assert e.code == 1 - else: - assert None, 'Should have failed' - assert "Directory already seems to be initted" \ - in caplog.records[-1].getMessage() - assert os.path.isdir(conf['paths']['sbws_home']) - assert os.path.isdir(conf['paths']['datadir']) - assert os.path.isfile(os.path.join(conf['paths']['sbws_home'], - 'config.ini'))

1 0

[sbws/master] Remove init
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 4d022d925c0363f93114a622972d30a14bf8deee Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 20:20:01 2018 +0000 Remove init --- tests/integration/test_sbws.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/tests/integration/test_sbws.py b/tests/integration/test_sbws.py deleted file mode 100644 index 5122f38..0000000 --- a/tests/integration/test_sbws.py +++ /dev/null @@ -1,22 +0,0 @@ -from tempfile import TemporaryFile -import subprocess -import os - - -def test_simple_init(tmpdir): - # out = None - err = None - with TemporaryFile('w+t') as stdout, TemporaryFile('w+t') as stderr: - retcode = subprocess.call( - 'sbws -d {} --log-level debug init'.format(tmpdir).split(), - stdout=stdout, stderr=stderr) - stdout.seek(0, 0) - stderr.seek(0, 0) - # out = stdout.read() - err = stderr.read() - assert retcode == 0 - assert len(err) == 0 - conf_fname = os.path.join(str(tmpdir), 'config.ini') - assert os.path.exists(conf_fname) - with open(conf_fname, 'rt') as fd: - assert len(fd.read()) > 0

1 0

[sbws/master] Remove sbws_home including config
by pastly＠torproject.org 09 Aug '18

09 Aug '18

commit 3f3e46fef627d53398502d8fdd1d1847c6cf2c89 Author: juga0 <juga(a)riseup.net> Date: Wed Jul 18 20:08:27 2018 +0000 Remove sbws_home including config there is not anymore sbws_home including config --- tests/unit/conftest.py | 46 ++++++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 7669318..e0fa938 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -4,7 +4,6 @@ import pytest from datetime import datetime import os -from sbws.core import init from sbws.globals import RESULT_VERSION from sbws.lib.resultdump import (ResultErrorStream, ResultSuccess, Result) from sbws.lib.resultdump import write_result_to_datadir @@ -116,11 +115,17 @@ def sbwshome_only_datadir(sbwshome_empty): @pytest.fixture(scope='function') -def args(sbwshome_empty, parser): +def test_config_path(tmpdir): + """""" + config = tmpdir.join('.sbws.ini') + return config + + +(a)pytest.fixture(scope='function') +def args(sbwshome_empty, parser, test_config_path): """Args with sbws home in the tests tmp dir.""" - args = _PseudoArguments(directory=sbwshome_empty, output=sbwshome_empty, - scale=False, log_level='debug', - no_results=False, no_v3bw=False) + args = _PseudoArguments(config=test_config_path, output=sbwshome_empty, + scale=False, log_level='debug') return args @@ -132,21 +137,6 @@ def conf(sbwshome_empty): return conf -(a)pytest.fixture(scope='function') -def sbwshome_config(sbwshome_empty, args, conf): - """Create sbws home inside of the tests tmp dir with only datadir.""" - init.main(args, conf) - return sbwshome_empty - - -(a)pytest.fixture(scope='function') -def sbwshome(sbwshome_only_datadir, args, conf): - """Create sbws home inside of the tests tmp dir.""" - os.makedirs(os.path.join(sbwshome_only_datadir, 'v3bw'), exist_ok=True) - init.main(args, conf) - return conf['paths']['sbws_home'] - - @pytest.fixture() def result(): return RESULT @@ -188,38 +178,38 @@ def resultdict_ip_changed_trimmed(): @pytest.fixture(scope='function') -def sbwshome_error_result(sbwshome, conf): +def sbwshome_error_result(sbwshome_only_datadir, conf): ''' Creates an ~/.sbws with a single fresh ResultError in it ''' dd = conf['paths']['datadir'] write_result_to_datadir(RESULT_ERROR_STREAM, dd) - return sbwshome + return sbwshome_only_datadir @pytest.fixture(scope='function') -def sbwshome_success_result(sbwshome, conf): +def sbwshome_success_result(sbwshome_only_datadir, conf): ''' Creates an ~/.sbws with a single fresh ResultSuccess in it ''' dd = conf['paths']['datadir'] write_result_to_datadir(RESULT_SUCCESS1, dd) - return sbwshome + return sbwshome_only_datadir @pytest.fixture(scope='function') -def sbwshome_success_result_one_relay(sbwshome, conf): +def sbwshome_success_result_one_relay(sbwshome_only_datadir, conf): ''' Creates an ~/.sbws with a a couple of fresh ResultSuccess for one relay ''' dd = conf['paths']['datadir'] write_result_to_datadir(RESULT_SUCCESS1, dd) write_result_to_datadir(RESULT_SUCCESS1, dd) - return sbwshome + return sbwshome_only_datadir @pytest.fixture(scope='function') -def sbwshome_success_result_two_relays(sbwshome, conf): +def sbwshome_success_result_two_relays(sbwshome_only_datadir, conf): ''' Creates an ~/.sbws with a a couple of fresh ResultSuccess for a couple or relays @@ -229,4 +219,4 @@ def sbwshome_success_result_two_relays(sbwshome, conf): write_result_to_datadir(RESULT_SUCCESS1, dd) write_result_to_datadir(RESULT_SUCCESS2, dd) write_result_to_datadir(RESULT_SUCCESS2, dd) - return sbwshome + return sbwshome_only_datadir

1 0