[tor-commits] [stem/master] Add an --exclude-test argument
atagar at torproject.org
atagar at torproject.org
Thu Jan 30 23:58:21 UTC 2020
commit 886ec10ee51d63fba76c7f3a21d8b73208858ea6
Author: Damian Johnson <atagar at torproject.org>
Date: Thu Jan 30 15:49:36 2020 -0800
Add an --exclude-test argument
Implementing a great suggestion from teor for an --exclude-test argument...
https://github.com/torproject/stem/issues/53
This can be used to skip both modules...
% run_tests.py --unit --exclude-test interpreter.commands
[ runs all the tests except the interpreter.commands module ]
... and individual tests...
% run_tests.py --unit --test interpreter.commands --exclude-test interpreter.commands.test_events --verbose
[ only runs interpreter.commands tests, but skips test_events ]
---
run_tests.py | 55 ++++++++++++++++++++++++++++++++++++++++++-------------
test/arguments.py | 5 ++++-
test/settings.cfg | 7 ++++++-
3 files changed, 52 insertions(+), 15 deletions(-)
diff --git a/run_tests.py b/run_tests.py
index d6ced384..91333084 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -111,40 +111,57 @@ def log_traceback(sig, frame):
os._exit(-1)
-def get_unit_tests(module_prefixes = None):
+def get_unit_tests(module_prefixes, exclude):
"""
Provides the classes for our unit tests.
:param list module_prefixes: only provide the test if the module starts with
any of these substrings
+ :param list exclude: test modules explicitly excluded
:returns: an **iterator** for our unit tests
"""
- return _get_tests(CONFIG['test.unit_tests'].splitlines(), module_prefixes)
+ return _get_tests(CONFIG['test.unit_tests'].splitlines(), module_prefixes, exclude)
-def get_integ_tests(module_prefixes = None):
+def get_integ_tests(module_prefixes, exclude):
"""
Provides the classes for our integration tests.
:param list module_prefixes: only provide the test if the module starts with
any of these substrings
+ :param list exclude: test modules explicitly excluded
:returns: an **iterator** for our integration tests
"""
- return _get_tests(CONFIG['test.integ_tests'].splitlines(), module_prefixes)
+ return _get_tests(CONFIG['test.integ_tests'].splitlines(), module_prefixes, exclude)
-def _get_tests(modules, module_prefixes):
+def _get_tests(modules, module_prefixes, exclude):
for import_name in modules:
+ cropped_name = test.arguments.crop_module_name(import_name)
+ cropped_name = cropped_name.rsplit('.', 1)[0] # exclude the class name
+
+ if exclude:
+ # Check if '--exclude-test' says we should skip this whole module. The
+ # argument can also skip individual tests, but that must be handled
+ # elsewhere.
+
+ skip = False
+
+ for exclude_prefix in exclude:
+ if cropped_name.startswith(exclude_prefix):
+ skip = True
+ break
+
+ if skip:
+ continue
+
if not module_prefixes:
yield import_name
else:
- cropped_name = test.arguments.crop_module_name(import_name)
- cropped_name = cropped_name.rsplit('.', 1)[0] # exclude the class name
-
for prefix in module_prefixes:
if cropped_name.startswith(prefix):
yield import_name
@@ -258,8 +275,8 @@ def main():
test.output.print_divider('UNIT TESTS', True)
error_tracker.set_category('UNIT TEST')
- for test_class in get_unit_tests(args.specific_test):
- run_result = _run_test(args, test_class, output_filters)
+ for test_class in get_unit_tests(args.specific_test, args.exclude_test):
+ run_result = _run_test(args, test_class, args.exclude_test, output_filters)
test.output.print_logging(logging_buffer)
skipped_tests += len(getattr(run_result, 'skipped', []))
@@ -277,8 +294,8 @@ def main():
println('Running tests...\n', STATUS)
- for test_class in get_integ_tests(args.specific_test):
- run_result = _run_test(args, test_class, output_filters)
+ for test_class in get_integ_tests(args.specific_test, args.exclude_test):
+ run_result = _run_test(args, test_class, args.exclude_test, output_filters)
test.output.print_logging(logging_buffer)
skipped_tests += len(getattr(run_result, 'skipped', []))
@@ -382,7 +399,7 @@ def _print_static_issues(static_check_issues):
println()
-def _run_test(args, test_class, output_filters):
+def _run_test(args, test_class, exclude, output_filters):
# When logging to a file we don't have stdout's test delimiters to correlate
# logs with the test that generated them.
@@ -423,6 +440,18 @@ def _run_test(args, test_class, output_filters):
traceback.print_exc(exc)
return None
+ # check if we should skip any individual tests within this module
+
+ if exclude:
+ cropped_name = test.arguments.crop_module_name(test_class)
+ cropped_name = cropped_name.rsplit('.', 1)[0] # exclude the class name
+
+ for prefix in exclude:
+ if prefix.startswith(cropped_name):
+ test_name = prefix.split('.')[-1]
+
+ suite._tests = list(filter(lambda test: test.id().split('.')[-1] != test_name, suite._tests))
+
test_results = io.StringIO()
run_result = stem.util.test_tools.TimedTestRunner(test_results, verbosity = 2).run(suite)
diff --git a/test/arguments.py b/test/arguments.py
index 65434b8b..d0f0dc3f 100644
--- a/test/arguments.py
+++ b/test/arguments.py
@@ -27,6 +27,7 @@ DEFAULT_ARGS = {
'run_unit': False,
'run_integ': False,
'specific_test': [],
+ 'exclude_test': [],
'logging_runlevel': None,
'logging_path': None,
'tor_path': 'tor',
@@ -38,7 +39,7 @@ DEFAULT_ARGS = {
}
OPT = 'auit:l:qvh'
-OPT_EXPANDED = ['all', 'unit', 'integ', 'targets=', 'test=', 'log=', 'log-file=', 'tor=', 'quiet', 'verbose', 'help']
+OPT_EXPANDED = ['all', 'unit', 'integ', 'targets=', 'test=', 'exclude-test=', 'log=', 'log-file=', 'tor=', 'quiet', 'verbose', 'help']
def parse(argv):
@@ -105,6 +106,8 @@ def parse(argv):
args['attribute_targets'] = attribute_targets
elif opt == '--test':
args['specific_test'].append(crop_module_name(arg))
+ elif opt == '--exclude-test':
+ args['exclude_test'].append(crop_module_name(arg))
elif opt in ('-l', '--log'):
arg = arg.upper()
diff --git a/test/settings.cfg b/test/settings.cfg
index 9c746234..1ec8176e 100644
--- a/test/settings.cfg
+++ b/test/settings.cfg
@@ -68,7 +68,6 @@ msg.help
| -a, --all runs unit, integ, and style checks (same as '-ui')
| -u, --unit runs unit tests
| -i, --integ runs integration tests
-| --test TEST_NAME only run tests with this in the module name
|
| -t, --target TARGET comma separated list of integ targets (see below)
| --tor PATH custom tor binary to run testing against
@@ -77,6 +76,9 @@ msg.help
| TRACE, DEBUG, INFO, NOTICE, WARN, ERROR
| --log-file PATH logs to this file, otherwise logging is to stdout
|
+| --test TEST_NAME only run this test or or test module
+| --exclude-test TEST_NAME skip this test or test module
+|
| -q, --quiet only present failures
| -v, --verbose provides additional test output
| -h, --help presents this help
@@ -92,6 +94,9 @@ msg.help
| run_tests.py --integ --test test.integ.util
| Only run integration tests for the util modules.
|
+| run_tests.py --unit --test interpreter.commands --exclude-test interpreter.commands.test_events --verbose
+| Only run interpreter.commands tests, but skip test_events.
+|
| Integration targets:
|
More information about the tor-commits
mailing list