[tor-commits] [stem/master] Checking for unused tests

[atagar at torproject.org]

commit fae5b1db09ff925ac41d8bfab36621e29dbf0e69
Author: Damian Johnson <atagar at torproject.org>
Date:   Sun Apr 14 14:47:35 2013 -0700

    Checking for unused tests
    
    Well... shame on me. We've always had a hardcoded list of our tests, but
    somehow when I moved it to our settings.cfg I missed a few. I'm not so much
    troubled about that mistake, but more that this mistake went undetected.
    
    Adding a check at the start of our tests for this kind of misconfiguration.
---
 run_tests.py      |    1 +
 test/settings.cfg |    3 +++
 test/util.py      |   38 +++++++++++++++++++++++++++++++++++++-
 3 files changed, 41 insertions(+), 1 deletions(-)

diff --git a/run_tests.py b/run_tests.py
index 610f0e2..9929cca 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -110,6 +110,7 @@ def main():
     Task("checking pyflakes version", test.util.check_pyflakes_version),
     Task("checking pep8 version", test.util.check_pep8_version),
     Task("checking for orphaned .pyc files", test.util.clean_orphaned_pyc, (SRC_PATHS,)),
+    Task("checking for unused tests", test.util.check_for_unused_tests, ((os.path.join(STEM_BASE, 'test'),),)),
   )
 
   if args.run_python3 and sys.version_info[0] != 3:
diff --git a/test/settings.cfg b/test/settings.cfg
index a40e1b7..bd3e3cd 100644
--- a/test/settings.cfg
+++ b/test/settings.cfg
@@ -172,6 +172,9 @@ test.unit_tests
 |test.unit.response.getinfo.TestGetInfoResponse
 |test.unit.response.getconf.TestGetConfResponse
 |test.unit.response.singleline.TestSingleLineResponse
+|test.unit.response.authchallenge.TestAuthChallengeResponse
+|test.unit.response.protocolinfo.TestProtocolInfoResponse
+|test.unit.response.mapaddress.TestMapAddressResponse
 |test.unit.connection.authentication.TestAuthenticate
 |test.unit.control.controller.TestControl
 
diff --git a/test/util.py b/test/util.py
index 52a9989..623f0cd 100644
--- a/test/util.py
+++ b/test/util.py
@@ -25,7 +25,8 @@ Tasks are...
   |- check_python_version - checks our version of python
   |- check_pyflakes_version - checks our version of pyflakes
   |- check_pep8_version - checks our version of pep8
-  +- clean_orphaned_pyc - removes any *.pyc without a corresponding *.py
+  |- clean_orphaned_pyc - removes any *.pyc without a corresponding *.py
+  +- check_for_unused_tests - checks to see if any tests are missing from our settings
 
   Testing Python 3
   |- python3_prereq - checks that we have python3 and 2to3
@@ -351,6 +352,41 @@ def clean_orphaned_pyc(paths):
   return ["removed %s" % path for path in orphaned_pyc]
 
 
+def check_for_unused_tests(paths):
+  """
+  The 'test.unit_tests' and 'test.integ_tests' in our settings.cfg defines the
+  tests that we run. We do it this way so that we can control the order in
+  which our tests are run but there's a disadvantage: when we add new test
+  modules we can easily forget to add it there.
+
+  Checking to see if we have any unittest.TestCase subclasses not covered by
+  our settings.
+
+  :param list paths: paths to search for unused tests
+  """
+
+  unused_tests = []
+
+  for path in paths:
+    for py_path in _get_files_with_suffix(path, ".py"):
+      if _is_test_data(py_path):
+        continue
+
+      with open(py_path) as f:
+        file_contents = f.read()
+
+      test_match = re.search("^class (\S*)\(unittest.TestCase\):$", file_contents, re.MULTILINE)
+
+      if test_match:
+        class_name = test_match.groups()[0]
+        module_name = py_path.replace(os.path.sep, '.')[len(STEM_BASE) + 1:-3] + '.' + class_name
+
+        if not (module_name in CONFIG['test.unit_tests'] or module_name in CONFIG['test.integ_tests']):
+          unused_tests.append(module_name)
+
+  if unused_tests:
+    raise ValueError("Test modules are missing from our test/settings.cfg:\n%s" % "\n".join(unused_tests))
+
 def python3_prereq():
   for required_cmd in ("2to3", "python3"):
     if not stem.util.system.is_available(required_cmd):