commit 886ec10ee51d63fba76c7f3a21d8b73208858ea6 Author: Damian Johnson <atagar@torproject.org> Date: Thu Jan 30 15:49:36 2020 -0800 Add an --exclude-test argument Implementing a great suggestion from teor for an --exclude-test argument... https://github.com/torproject/stem/issues/53 This can be used to skip both modules... % run_tests.py --unit --exclude-test interpreter.commands [ runs all the tests except the interpreter.commands module ] ... and individual tests... % run_tests.py --unit --test interpreter.commands --exclude-test interpreter.commands.test_events --verbose [ only runs interpreter.commands tests, but skips test_events ] --- run_tests.py | 55 ++++++++++++++++++++++++++++++++++++++++++------------- test/arguments.py | 5 ++++- test/settings.cfg | 7 ++++++- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/run_tests.py b/run_tests.py index d6ced384..91333084 100755 --- a/run_tests.py +++ b/run_tests.py @@ -111,40 +111,57 @@ def log_traceback(sig, frame): os._exit(-1) -def get_unit_tests(module_prefixes = None): +def get_unit_tests(module_prefixes, exclude): """ Provides the classes for our unit tests. :param list module_prefixes: only provide the test if the module starts with any of these substrings + :param list exclude: test modules explicitly excluded :returns: an **iterator** for our unit tests """ - return _get_tests(CONFIG['test.unit_tests'].splitlines(), module_prefixes) + return _get_tests(CONFIG['test.unit_tests'].splitlines(), module_prefixes, exclude) -def get_integ_tests(module_prefixes = None): +def get_integ_tests(module_prefixes, exclude): """ Provides the classes for our integration tests. :param list module_prefixes: only provide the test if the module starts with any of these substrings + :param list exclude: test modules explicitly excluded :returns: an **iterator** for our integration tests """ - return _get_tests(CONFIG['test.integ_tests'].splitlines(), module_prefixes) + return _get_tests(CONFIG['test.integ_tests'].splitlines(), module_prefixes, exclude) -def _get_tests(modules, module_prefixes): +def _get_tests(modules, module_prefixes, exclude): for import_name in modules: + cropped_name = test.arguments.crop_module_name(import_name) + cropped_name = cropped_name.rsplit('.', 1)[0] # exclude the class name + + if exclude: + # Check if '--exclude-test' says we should skip this whole module. The + # argument can also skip individual tests, but that must be handled + # elsewhere. + + skip = False + + for exclude_prefix in exclude: + if cropped_name.startswith(exclude_prefix): + skip = True + break + + if skip: + continue + if not module_prefixes: yield import_name else: - cropped_name = test.arguments.crop_module_name(import_name) - cropped_name = cropped_name.rsplit('.', 1)[0] # exclude the class name - for prefix in module_prefixes: if cropped_name.startswith(prefix): yield import_name @@ -258,8 +275,8 @@ def main(): test.output.print_divider('UNIT TESTS', True) error_tracker.set_category('UNIT TEST') - for test_class in get_unit_tests(args.specific_test): - run_result = _run_test(args, test_class, output_filters) + for test_class in get_unit_tests(args.specific_test, args.exclude_test): + run_result = _run_test(args, test_class, args.exclude_test, output_filters) test.output.print_logging(logging_buffer) skipped_tests += len(getattr(run_result, 'skipped', [])) @@ -277,8 +294,8 @@ def main(): println('Running tests...\n', STATUS) - for test_class in get_integ_tests(args.specific_test): - run_result = _run_test(args, test_class, output_filters) + for test_class in get_integ_tests(args.specific_test, args.exclude_test): + run_result = _run_test(args, test_class, args.exclude_test, output_filters) test.output.print_logging(logging_buffer) skipped_tests += len(getattr(run_result, 'skipped', [])) @@ -382,7 +399,7 @@ def _print_static_issues(static_check_issues): println() -def _run_test(args, test_class, output_filters): +def _run_test(args, test_class, exclude, output_filters): # When logging to a file we don't have stdout's test delimiters to correlate # logs with the test that generated them. @@ -423,6 +440,18 @@ def _run_test(args, test_class, output_filters): traceback.print_exc(exc) return None + # check if we should skip any individual tests within this module + + if exclude: + cropped_name = test.arguments.crop_module_name(test_class) + cropped_name = cropped_name.rsplit('.', 1)[0] # exclude the class name + + for prefix in exclude: + if prefix.startswith(cropped_name): + test_name = prefix.split('.')[-1] + + suite._tests = list(filter(lambda test: test.id().split('.')[-1] != test_name, suite._tests)) + test_results = io.StringIO() run_result = stem.util.test_tools.TimedTestRunner(test_results, verbosity = 2).run(suite) diff --git a/test/arguments.py b/test/arguments.py index 65434b8b..d0f0dc3f 100644 --- a/test/arguments.py +++ b/test/arguments.py @@ -27,6 +27,7 @@ DEFAULT_ARGS = { 'run_unit': False, 'run_integ': False, 'specific_test': [], + 'exclude_test': [], 'logging_runlevel': None, 'logging_path': None, 'tor_path': 'tor', @@ -38,7 +39,7 @@ DEFAULT_ARGS = { } OPT = 'auit:l:qvh' -OPT_EXPANDED = ['all', 'unit', 'integ', 'targets=', 'test=', 'log=', 'log-file=', 'tor=', 'quiet', 'verbose', 'help'] +OPT_EXPANDED = ['all', 'unit', 'integ', 'targets=', 'test=', 'exclude-test=', 'log=', 'log-file=', 'tor=', 'quiet', 'verbose', 'help'] def parse(argv): @@ -105,6 +106,8 @@ def parse(argv): args['attribute_targets'] = attribute_targets elif opt == '--test': args['specific_test'].append(crop_module_name(arg)) + elif opt == '--exclude-test': + args['exclude_test'].append(crop_module_name(arg)) elif opt in ('-l', '--log'): arg = arg.upper() diff --git a/test/settings.cfg b/test/settings.cfg index 9c746234..1ec8176e 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -68,7 +68,6 @@ msg.help | -a, --all runs unit, integ, and style checks (same as '-ui') | -u, --unit runs unit tests | -i, --integ runs integration tests -| --test TEST_NAME only run tests with this in the module name | | -t, --target TARGET comma separated list of integ targets (see below) | --tor PATH custom tor binary to run testing against @@ -77,6 +76,9 @@ msg.help | TRACE, DEBUG, INFO, NOTICE, WARN, ERROR | --log-file PATH logs to this file, otherwise logging is to stdout | +| --test TEST_NAME only run this test or or test module +| --exclude-test TEST_NAME skip this test or test module +| | -q, --quiet only present failures | -v, --verbose provides additional test output | -h, --help presents this help @@ -92,6 +94,9 @@ msg.help | run_tests.py --integ --test test.integ.util | Only run integration tests for the util modules. | +| run_tests.py --unit --test interpreter.commands --exclude-test interpreter.commands.test_events --verbose +| Only run interpreter.commands tests, but skip test_events. +| | Integration targets: |