commit fae5b1db09ff925ac41d8bfab36621e29dbf0e69 Author: Damian Johnson atagar@torproject.org Date: Sun Apr 14 14:47:35 2013 -0700
Checking for unused tests
Well... shame on me. We've always had a hardcoded list of our tests, but somehow when I moved it to our settings.cfg I missed a few. I'm not so much troubled about that mistake, but more that this mistake went undetected.
Adding a check at the start of our tests for this kind of misconfiguration. --- run_tests.py | 1 + test/settings.cfg | 3 +++ test/util.py | 38 +++++++++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 1 deletions(-)
diff --git a/run_tests.py b/run_tests.py index 610f0e2..9929cca 100755 --- a/run_tests.py +++ b/run_tests.py @@ -110,6 +110,7 @@ def main(): Task("checking pyflakes version", test.util.check_pyflakes_version), Task("checking pep8 version", test.util.check_pep8_version), Task("checking for orphaned .pyc files", test.util.clean_orphaned_pyc, (SRC_PATHS,)), + Task("checking for unused tests", test.util.check_for_unused_tests, ((os.path.join(STEM_BASE, 'test'),),)), )
if args.run_python3 and sys.version_info[0] != 3: diff --git a/test/settings.cfg b/test/settings.cfg index a40e1b7..bd3e3cd 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -172,6 +172,9 @@ test.unit_tests |test.unit.response.getinfo.TestGetInfoResponse |test.unit.response.getconf.TestGetConfResponse |test.unit.response.singleline.TestSingleLineResponse +|test.unit.response.authchallenge.TestAuthChallengeResponse +|test.unit.response.protocolinfo.TestProtocolInfoResponse +|test.unit.response.mapaddress.TestMapAddressResponse |test.unit.connection.authentication.TestAuthenticate |test.unit.control.controller.TestControl
diff --git a/test/util.py b/test/util.py index 52a9989..623f0cd 100644 --- a/test/util.py +++ b/test/util.py @@ -25,7 +25,8 @@ Tasks are... |- check_python_version - checks our version of python |- check_pyflakes_version - checks our version of pyflakes |- check_pep8_version - checks our version of pep8 - +- clean_orphaned_pyc - removes any *.pyc without a corresponding *.py + |- clean_orphaned_pyc - removes any *.pyc without a corresponding *.py + +- check_for_unused_tests - checks to see if any tests are missing from our settings
Testing Python 3 |- python3_prereq - checks that we have python3 and 2to3 @@ -351,6 +352,41 @@ def clean_orphaned_pyc(paths): return ["removed %s" % path for path in orphaned_pyc]
+def check_for_unused_tests(paths): + """ + The 'test.unit_tests' and 'test.integ_tests' in our settings.cfg defines the + tests that we run. We do it this way so that we can control the order in + which our tests are run but there's a disadvantage: when we add new test + modules we can easily forget to add it there. + + Checking to see if we have any unittest.TestCase subclasses not covered by + our settings. + + :param list paths: paths to search for unused tests + """ + + unused_tests = [] + + for path in paths: + for py_path in _get_files_with_suffix(path, ".py"): + if _is_test_data(py_path): + continue + + with open(py_path) as f: + file_contents = f.read() + + test_match = re.search("^class (\S*)(unittest.TestCase):$", file_contents, re.MULTILINE) + + if test_match: + class_name = test_match.groups()[0] + module_name = py_path.replace(os.path.sep, '.')[len(STEM_BASE) + 1:-3] + '.' + class_name + + if not (module_name in CONFIG['test.unit_tests'] or module_name in CONFIG['test.integ_tests']): + unused_tests.append(module_name) + + if unused_tests: + raise ValueError("Test modules are missing from our test/settings.cfg:\n%s" % "\n".join(unused_tests)) + def python3_prereq(): for required_cmd in ("2to3", "python3"): if not stem.util.system.is_available(required_cmd):