commit b3a145677e12af8df22cdbf27d53d1babba1ac62 Author: Damian Johnson atagar@torproject.org Date: Wed Feb 8 17:17:30 2017 -0800
Speed up installation integ tests
Installing and creating source distributions take several seconds, making these slow tests. Thankfully this time is all disk io so we can simply kick off another thread to parallelize this with other tests. Shaves whole seconds off our integ test runtime (... which for our tests is quite a bit!). --- run_tests.py | 4 +++ test/integ/installation.py | 78 +++++++++++++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 28 deletions(-)
diff --git a/run_tests.py b/run_tests.py index abb5123..3db6280 100755 --- a/run_tests.py +++ b/run_tests.py @@ -26,6 +26,7 @@ import stem.util.system import stem.util.test_tools
import test.arguments +import test.integ.installation import test.output import test.runner import test.util @@ -213,6 +214,9 @@ def main(): our_version = stem.version.get_system_tor_version(args.tor_path) skipped_targets = []
+ if not args.specific_test or 'test.integ.installation'.startswith(args.specific_test): + test.integ.installation.setup() + for target in args.run_targets: # check if we meet this target's tor version prerequisites
diff --git a/test/integ/installation.py b/test/integ/installation.py index d1a1eb8..cfe771a 100644 --- a/test/integ/installation.py +++ b/test/integ/installation.py @@ -3,6 +3,7 @@ import os import shutil import sys import tarfile +import threading import unittest
import stem @@ -13,34 +14,50 @@ import test.util
INSTALL_MISMATCH_MSG = "Running 'python setup.py sdist' doesn't match our git contents in the following way. The manifest in our setup.py may need to be updated...\n\n"
+BASE_INSTALL_PATH = '/tmp/stem_test' +SETUP_THREAD, INSTALL_FAILURE, INSTALL_PATH, SDIST_FAILURE = None, None, None, None
-def _install(path): + +def setup(): + """ + Performs setup our tests will need. This mostly just needs disk iops so it + can happen asynchronously with other tests. """ - Installs to the given location. - - :param str path: location to install ourselves to
- :returns: **str** where our site packages are located + global SETUP_THREAD
- :raises: **AssertionError** if installation is unsuccessful - """ + def _setup(): + global INSTALL_FAILURE, INSTALL_PATH, SDIST_FAILURE + original_cwd = os.getcwd() + + try: + os.chdir(test.util.STEM_BASE) + + try: + os.chdir(test.util.STEM_BASE) + stem.util.system.call('%s setup.py install --prefix %s' % (sys.executable, BASE_INSTALL_PATH)) + stem.util.system.call('%s setup.py clean --all' % sys.executable) # tidy up the build directory + site_packages_paths = glob.glob('%s/lib*/*/site-packages' % BASE_INSTALL_PATH)
- original_cwd = os.getcwd() + if len(site_packages_paths) != 1: + raise AssertionError('We should only have a single site-packages directory, but instead had: %s' % site_packages_paths)
- try: - os.chdir(test.util.STEM_BASE) - stem.util.system.call('%s setup.py install --prefix %s' % (sys.executable, path)) - stem.util.system.call('%s setup.py clean --all' % sys.executable) # tidy up the build directory - site_packages_paths = glob.glob('%s/lib*/*/site-packages' % path) + INSTALL_PATH = site_packages_paths[0] + except Exception as exc: + INSTALL_FAILURE = AssertionError("Unable to install with 'python setup.py install': %s" % exc)
- if len(site_packages_paths) != 1: - raise AssertionError('We should only have a single site-packages directory, but instead had: %s' % site_packages_paths) + try: + stem.util.system.call('%s setup.py sdist' % sys.executable) + except Exception as exc: + SDIST_FAILURE = exc + finally: + os.chdir(original_cwd)
- return site_packages_paths[0] - except Exception as exc: - raise AssertionError("Unable to install with 'python setup.py install': %s" % exc) - finally: - os.chdir(original_cwd) + if SETUP_THREAD is None: + SETUP_THREAD = threading.Thread(target = _setup) + SETUP_THREAD.start() + + return SETUP_THREAD
def _assert_has_all_files(path): @@ -84,13 +101,16 @@ class TestInstallation(unittest.TestCase): """
try: - site_packages_path = _install('/tmp/stem_test') + setup().join() + + if INSTALL_FAILURE: + raise INSTALL_FAILURE
- self.assertEqual(stem.__version__, stem.util.system.call([sys.executable, '-c', "import sys;sys.path.insert(0, '%s');import stem;print(stem.__version__)" % site_packages_path])[0]) - _assert_has_all_files(site_packages_path) + self.assertEqual(stem.__version__, stem.util.system.call([sys.executable, '-c', "import sys;sys.path.insert(0, '%s');import stem;print(stem.__version__)" % INSTALL_PATH])[0]) + _assert_has_all_files(INSTALL_PATH) finally: - if os.path.exists('/tmp/stem_test'): - shutil.rmtree('/tmp/stem_test') + if os.path.exists(BASE_INSTALL_PATH): + shutil.rmtree(BASE_INSTALL_PATH)
@test.runner.only_run_once def test_sdist(self): @@ -104,14 +124,16 @@ class TestInstallation(unittest.TestCase): test.runner.skip(self, '(git unavailable)') return
+ setup().join() + + if SDIST_FAILURE: + raise SDIST_FAILURE + original_cwd = os.getcwd() dist_path = os.path.join(test.util.STEM_BASE, 'dist') git_contents = [line.split()[-1] for line in stem.util.system.call('git ls-tree --full-tree -r HEAD')]
try: - os.chdir(test.util.STEM_BASE) - stem.util.system.call(sys.executable + ' setup.py sdist') - # tarball has a prefix 'stem-[verion]' directory so stipping that out
dist_tar = tarfile.open(os.path.join(dist_path, 'stem-dry-run-%s.tar.gz' % stem.__version__))