commit 3fc623445df21927930899ed9d78098da5c40b72 Author: Damian Johnson <atagar@torproject.org> Date: Mon May 29 15:00:45 2017 -0700 Parallelize test's static analysis checks This alone made attending PyCon well worth it. For months I've been struggling to speed up our tests but python's GIL keeps getting in the way. Turns out the builtins include a module *specifically* for this... https://docs.python.org/3/library/multiprocessing.html#module-multiprocessin... Running our static checks (pyflakes and pycodestyle) in the background. On my desktop this drops the runtime of our unit tests from 10.8s to 8.9s (18% faster). I was expecting to get more but this is a pretty old dual-core system so probably would get more elsewhere. --- run_tests.py | 4 +++- test/task.py | 30 +++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/run_tests.py b/run_tests.py index 8cb1404..7c894d2 100755 --- a/run_tests.py +++ b/run_tests.py @@ -327,7 +327,9 @@ def main(): static_check_issues = {} for task in (test.task.PYFLAKES_TASK, test.task.PYCODESTYLE_TASK): - if task and task.is_successful: + if task: + task.join() + for path, issues in task.result.items(): for issue in issues: static_check_issues.setdefault(path, []).append(issue) diff --git a/test/task.py b/test/task.py index cff4c72..1c1b34e 100644 --- a/test/task.py +++ b/test/task.py @@ -22,6 +22,7 @@ """ import importlib +import multiprocessing import os import re import sys @@ -141,7 +142,7 @@ class Task(object): message or list of strings for its results. """ - def __init__(self, label, runner, args = None, is_required = True, print_result = True, print_runtime = False): + def __init__(self, label, runner, args = None, is_required = True, print_result = True, print_runtime = False, background = False): super(Task, self).__init__() self.label = label @@ -155,6 +156,10 @@ class Task(object): self.is_successful = False self.result = None + self._is_background_task = background + self._background_process = None + self._background_pipe = None + def run(self): start_time = time.time() println(' %s...' % self.label, STATUS, NO_NL) @@ -163,13 +168,19 @@ class Task(object): println(' ' * padding, NO_NL) try: - if self.args: - self.result = self.runner(*self.args) + if self._is_background_task: + def _run_wrapper(conn, runner, args): + conn.send(runner(*args) if args else runner()) + conn.close() + + self._background_pipe, child_pipe = multiprocessing.Pipe() + self._background_process = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, self.runner, self.args)) + self._background_process.start() else: - self.result = self.runner() + self.result = self.runner(*self.args) if self.args else self.runner() self.is_successful = True - output_msg = 'done' + output_msg = 'running' if self._is_background_task else 'done' if self.print_result and isinstance(self.result, str): output_msg = self.result @@ -190,6 +201,11 @@ class Task(object): println(output_msg, ERROR) self.error = exc + def join(self): + if self._is_background_task: + self.result = self._background_pipe.recv() + self._background_process.join() + class ModuleVersion(Task): def __init__(self, label, modules, prereq_check = None): @@ -208,8 +224,8 @@ class ModuleVersion(Task): class StaticCheckTask(Task): - def __init__(self, label, runner, args = None, is_available = None, unavailable_msg = None): - super(StaticCheckTask, self).__init__(label, runner, args, is_required = False, print_result = False, print_runtime = True) + def __init__(self, label, runner, args = None, is_available = None, unavailable_msg = None, background = True): + super(StaticCheckTask, self).__init__(label, runner, args, is_required = False, print_result = False, print_runtime = not background, background = background) self.is_available = is_available self.unavailable_msg = unavailable_msg