[tor-commits] [stem/master] Parallelize test's static analysis checks

[atagar at torproject.org]

commit 3fc623445df21927930899ed9d78098da5c40b72
Author: Damian Johnson <atagar at torproject.org>
Date:   Mon May 29 15:00:45 2017 -0700

    Parallelize test's static analysis checks
    
    This alone made attending PyCon well worth it. For months I've been struggling
    to speed up our tests but python's GIL keeps getting in the way. Turns out the
    builtins include a module *specifically* for this...
    
      https://docs.python.org/3/library/multiprocessing.html#module-multiprocessing
    
    Running our static checks (pyflakes and pycodestyle) in the background. On my
    desktop this drops the runtime of our unit tests from 10.8s to 8.9s (18%
    faster). I was expecting to get more but this is a pretty old dual-core system
    so probably would get more elsewhere.
---
 run_tests.py |  4 +++-
 test/task.py | 30 +++++++++++++++++++++++-------
 2 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/run_tests.py b/run_tests.py
index 8cb1404..7c894d2 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -327,7 +327,9 @@ def main():
   static_check_issues = {}
 
   for task in (test.task.PYFLAKES_TASK, test.task.PYCODESTYLE_TASK):
-    if task and task.is_successful:
+    if task:
+      task.join()
+
       for path, issues in task.result.items():
         for issue in issues:
           static_check_issues.setdefault(path, []).append(issue)
diff --git a/test/task.py b/test/task.py
index cff4c72..1c1b34e 100644
--- a/test/task.py
+++ b/test/task.py
@@ -22,6 +22,7 @@
 """
 
 import importlib
+import multiprocessing
 import os
 import re
 import sys
@@ -141,7 +142,7 @@ class Task(object):
   message or list of strings for its results.
   """
 
-  def __init__(self, label, runner, args = None, is_required = True, print_result = True, print_runtime = False):
+  def __init__(self, label, runner, args = None, is_required = True, print_result = True, print_runtime = False, background = False):
     super(Task, self).__init__()
 
     self.label = label
@@ -155,6 +156,10 @@ class Task(object):
     self.is_successful = False
     self.result = None
 
+    self._is_background_task = background
+    self._background_process = None
+    self._background_pipe = None
+
   def run(self):
     start_time = time.time()
     println('  %s...' % self.label, STATUS, NO_NL)
@@ -163,13 +168,19 @@ class Task(object):
     println(' ' * padding, NO_NL)
 
     try:
-      if self.args:
-        self.result = self.runner(*self.args)
+      if self._is_background_task:
+        def _run_wrapper(conn, runner, args):
+          conn.send(runner(*args) if args else runner())
+          conn.close()
+
+        self._background_pipe, child_pipe = multiprocessing.Pipe()
+        self._background_process = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, self.runner, self.args))
+        self._background_process.start()
       else:
-        self.result = self.runner()
+        self.result = self.runner(*self.args) if self.args else self.runner()
 
       self.is_successful = True
-      output_msg = 'done'
+      output_msg = 'running' if self._is_background_task else 'done'
 
       if self.print_result and isinstance(self.result, str):
         output_msg = self.result
@@ -190,6 +201,11 @@ class Task(object):
       println(output_msg, ERROR)
       self.error = exc
 
+  def join(self):
+    if self._is_background_task:
+      self.result = self._background_pipe.recv()
+      self._background_process.join()
+
 
 class ModuleVersion(Task):
   def __init__(self, label, modules, prereq_check = None):
@@ -208,8 +224,8 @@ class ModuleVersion(Task):
 
 
 class StaticCheckTask(Task):
-  def __init__(self, label, runner, args = None, is_available = None, unavailable_msg = None):
-    super(StaticCheckTask, self).__init__(label, runner, args, is_required = False, print_result = False, print_runtime = True)
+  def __init__(self, label, runner, args = None, is_available = None, unavailable_msg = None, background = True):
+    super(StaticCheckTask, self).__init__(label, runner, args, is_required = False, print_result = False, print_runtime = not background, background = background)
     self.is_available = is_available
     self.unavailable_msg = unavailable_msg