commit c03d05abdec29eebba2d3ed0f15c7e5f46aa658b Author: Damian Johnson atagar@torproject.org Date: Wed Aug 16 07:46:03 2017 -0700
Run static checks in the background
Stem got quite a substantial speedup in its tests by running long-running tasks in the background. Nyx is a much smaller test suite so not much to gain, but might as well improve things a tad here too.
Running pyflakes and pycodestyle in the background. On my laptop this drops the test runtime from 3.8s to 2.4s (37% faster). --- run_tests.py | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-)
diff --git a/run_tests.py b/run_tests.py index 2b76561..e115552 100755 --- a/run_tests.py +++ b/run_tests.py @@ -9,6 +9,7 @@ render to the screen. """
import os +import multiprocessing import unittest
import stem.util.conf @@ -27,6 +28,12 @@ SRC_PATHS = [os.path.join(NYX_BASE, path) for path in ( )]
+def _run_wrapper(conn, runner, args): + os.nice(15) + conn.send(runner(*args) if args else runner()) + conn.close() + + @nyx.uses_settings def main(): nyx.TESTING = True @@ -38,7 +45,20 @@ def main(): for path in orphaned_pyc: print('Deleted orphaned pyc file: %s' % path)
- tests = unittest.defaultTestLoader.discover('test', pattern='*.py') + pyflakes_task, pyflakes_pipe = None, None + pycodestyle_task, pycodestyle_pipe = None, None + + if stem.util.test_tools.is_pyflakes_available(): + pyflakes_pipe, child_pipe = multiprocessing.Pipe() + pyflakes_task = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, stem.util.test_tools.pyflakes_issues, (SRC_PATHS,))) + pyflakes_task.start() + + if stem.util.test_tools.is_pep8_available(): + pycodestyle_pipe, child_pipe = multiprocessing.Pipe() + pycodestyle_task = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, stem.util.test_tools.stylistic_issues, (SRC_PATHS, True, True, True))) + pycodestyle_task.start() + + tests = unittest.defaultTestLoader.discover('test', pattern = '*.py') test_runner = unittest.TextTestRunner() test_runner.run(tests)
@@ -46,22 +66,19 @@ def main():
static_check_issues = {}
- if stem.util.test_tools.is_pyflakes_available(): - pyflakes_issues = stem.util.test_tools.pyflakes_issues(SRC_PATHS) + if pyflakes_task: + pyflakes_issues = pyflakes_pipe.recv() + pyflakes_task.join()
for path, issues in pyflakes_issues.items(): for issue in issues: static_check_issues.setdefault(path, []).append(issue)
- if stem.util.test_tools.is_pep8_available(): - pep8_issues = stem.util.test_tools.stylistic_issues( - SRC_PATHS, - check_newlines = True, - check_exception_keyword = True, - prefer_single_quotes = True, - ) - - for path, issues in pep8_issues.items(): + if pycodestyle_task: + pycodestyle_issues = pycodestyle_pipe.recv() + pycodestyle_task.join() + + for path, issues in pycodestyle_issues.items(): for issue in issues: static_check_issues.setdefault(path, []).append(issue)