[tor-commits] [stem/master] Rewrite of integration test runner

commit fbfa73a099d9645f18d9846420cbf0145065b11d Author: Damian Johnson atagar@torproject.org Date: Wed Oct 26 20:13:34 2011 -0700

Rewrite of integration test runner

Several imporvements for the integration tests, most notably including... - Test configurability via a 'test/settings.cfg' file

- Thread safety for runner usage

- Vastly better startup time for how integration tests run by default... - Reusing data directory so we don't need to request as much from authorities when starting (faster startup and less burden on them). Users can opt for a fresh temporary directory instead by setting 'test.integ.test_directory' to a blank value.

- Starting tests when bootstraping reaches 5%. This is enough for tests that don't require network activity to run, and we can explicitly run those tests by setting the 'test.integ.run.online' option. This change also means that we can now run integration tests while offline. --- .gitignore | 1 + run_tests.py | 16 ++- stem/process.py | 11 ++- stem/util/conf.py | 9 +- test/integ/message.py | 4 +- test/runner.py | 335 +++++++++++++++++++++++++++++++++++++------------ test/settings.cfg | 17 +++ 7 files changed, 303 insertions(+), 90 deletions(-)

diff --git a/.gitignore b/.gitignore index 0e16dd0..124fb16 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc *.swp +test/data/

diff --git a/run_tests.py b/run_tests.py index 1741892..ccdee42 100755 --- a/run_tests.py +++ b/run_tests.py @@ -4,6 +4,7 @@ Runs unit and integration tests. """

+import os import sys import time import getopt @@ -14,7 +15,7 @@ import test.unit.version import test.integ.message import test.integ.system

-from stem.util import enum, term +from stem.util import conf, enum, term

OPT = "uit:h" OPT_EXPANDED = ["unit", "integ", "targets=", "help"] @@ -29,6 +30,7 @@ INTEG_TESTS = (("stem.types.ControlMessage", test.integ.message.TestMessageFunct ("stem.util.system", test.integ.system.TestSystemFunctions), )

+# TODO: drop targets? # Configurations that the intergration tests can be ran with. Attributs are # tuples of the test runner and description. TARGETS = enum.Enum(*[(v, v) for v in ("NONE", "NO_CONTROL", "NO_AUTH", "COOKIE", "PASSWORD", "SOCKET")]) @@ -97,6 +99,15 @@ if __name__ == '__main__': print HELP_MSG % "\n ".join(target_lines) sys.exit()

+ test_config = conf.get_config("test") + + try: + config_path = os.path.dirname(__file__) + "/test/settings.cfg" + test_config.load(config_path) + except IOError, exc: + print term.format("Unable to load testing configuration: %s" % exc, term.Color.RED, term.Attr.BOLD) + sys.exit(1) + if not run_unit_tests and not run_integ_tests: print "Nothing to run (for usage provide --help)\n" sys.exit() @@ -118,8 +129,7 @@ if __name__ == '__main__': integ_runner = test.runner.get_runner()

try: - integ_runner.run_setup() - integ_runner.start() + integ_runner.start(user_config = test_config)

print term.format("Running tests...", term.Color.BLUE, term.Attr.BOLD) print diff --git a/stem/process.py b/stem/process.py index b2034c2..0baa5dd 100644 --- a/stem/process.py +++ b/stem/process.py @@ -3,6 +3,7 @@ Helper functions for working with tor as a process. These are mostly os dependent, only working on linux, osx, and bsd. """

+import re import os import signal import subprocess @@ -52,7 +53,7 @@ def get_version(tor_cmd = "tor"): else: raise IOError("'%s' didn't have any output" % version_cmd)

-def launch_tor(torrc_path, init_msg_handler = None, timeout = DEFAULT_INIT_TIMEOUT): +def launch_tor(torrc_path, completion_percent = 100, init_msg_handler = None, timeout = DEFAULT_INIT_TIMEOUT): """ Initializes a tor process. This blocks until initialization completes or we error out. @@ -64,6 +65,8 @@ def launch_tor(torrc_path, init_msg_handler = None, timeout = DEFAULT_INIT_TIMEO

Arguments: torrc_path (str) - location of the torrc for us to use + completion_percent (int) - percent of bootstrap completion at which + this'll return init_msg_handler (functor) - optional functor that will be provided with tor's initialization stdout as we get it timeout (int) - time after which the attempt to start tor is @@ -93,6 +96,8 @@ def launch_tor(torrc_path, init_msg_handler = None, timeout = DEFAULT_INIT_TIMEO signal.signal(signal.SIGALRM, timeout_handler) signal.alarm(timeout)

+ bootstrap_line = re.compile("Bootstrapped ([0-9]+)%: ") + while True: init_line = tor_process.stdout.readline().strip()

@@ -105,6 +110,8 @@ def launch_tor(torrc_path, init_msg_handler = None, timeout = DEFAULT_INIT_TIMEO if init_msg_handler: init_msg_handler(init_line)

# return the process if we're done with bootstrapping - if init_line.endswith("Bootstrapped 100%: Done."): + bootstrap_match = bootstrap_line.search(init_line) + + if bootstrap_match and int(bootstrap_match.groups()[0]) >= completion_percent: return tor_process

diff --git a/stem/util/conf.py b/stem/util/conf.py index bb587b3..c7575ad 100644 --- a/stem/util/conf.py +++ b/stem/util/conf.py @@ -420,9 +420,12 @@ class Config(): line = line.strip()

# parse the key/value pair - if line and " " in line: - key, value = line.split(" ", 1) - value = value.strip() + if line: + if " " in line: + key, value = line.split(" ", 1) + value = value.strip() + else: + key, value = line, ""

if key in self._contents: self._contents[key].append(value) else: self._contents[key] = [value] diff --git a/test/integ/message.py b/test/integ/message.py index ae92af6..ad5417a 100644 --- a/test/integ/message.py +++ b/test/integ/message.py @@ -199,9 +199,9 @@ class TestMessageFunctions(unittest.TestCase): self.assertEquals("250 OK\r\n", setevents_response.raw_content()) self.assertEquals([("250", " ", "OK")], setevents_response.content())

- # Tor will emit a BW event once per second. Parsing three of them. + # Tor will emit a BW event once per second. Parsing two of them.

- for _ in range(3): + for _ in range(2): bw_event = stem.types.read_message(control_socket_file) self.assertTrue(re.match("BW [0-9]+ [0-9]+", str(bw_event))) self.assertTrue(re.match("650 BW [0-9]+ [0-9]+\r\n", bw_event.raw_content())) diff --git a/test/runner.py b/test/runner.py index 246fd79..da65262 100644 --- a/test/runner.py +++ b/test/runner.py @@ -1,5 +1,7 @@ """ -Runtime context for the integration tests. +Runtime context for the integration tests. This is used both by the test runner +to start and stop tor, and by the integration tests themselves for information +about the tor test instance they're running against. """

import os @@ -7,13 +9,20 @@ import sys import time import shutil import tempfile +import threading

import stem.process

from stem.util import term

-# number of seconds before we time out our attempt to start a tor instance -TOR_INIT_TIMEOUT = 90 +DEFAULT_CONFIG = { + "test.integ.test_directory": "./test/data", + "test.integ.run.online": False, +} + +STATUS_ATTR = (term.Color.BLUE, term.Attr.BOLD) +SUBSTATUS_ATTR = (term.Color.BLUE, ) +ERROR_ATTR = (term.Color.RED, term.Attr.BOLD)

BASIC_TORRC = """# configuration for stem integration tests DataDirectory %s @@ -33,106 +42,145 @@ def get_runner(): if not INTEG_RUNNER: INTEG_RUNNER = Runner() return INTEG_RUNNER

+class RunnerStopped(Exception): + "Raised when we try to use a Runner that doesn't have an active tor instance" + pass + class Runner: def __init__(self): - self._test_dir = tempfile.mktemp("-stem-integ") - self._torrc_contents = BASIC_TORRC % self._test_dir + self._config = dict(DEFAULT_CONFIG) + self._runner_lock = threading.RLock() + + # runtime attributes, set by the start method + self._test_dir = "" + self._torrc_contents = "" self._tor_process = None

- def run_setup(self): + def start(self, quiet = False, user_config = None): """ - Makes a temporary directory for the runtime resources of our integ tests. + Makes temporary testing resources and starts tor, blocking until it + completes. + + Arguments: + quiet (bool) - if False then this prints status information as we start + up to stdout + user_config (stem.util.conf.Config) - custom test configuration

Raises: - OSError if unsuccessful + OSError if unable to run test preparations or start tor """

- print term.format("Setting up a test instance...", term.Color.BLUE, term.Attr.BOLD) + self._runner_lock.acquire()

- # makes a temporary directory for the runtime resources of our integ test - try: - sys.stdout.write(term.format(" making test directory (%s)... " % self._test_dir, term.Color.BLUE, term.Attr.BOLD)) - os.makedirs(self._test_dir) - sys.stdout.write(term.format("done\n", term.Color.BLUE, term.Attr.BOLD)) - except OSError, exc: - sys.stdout.write(term.format("failed (%s)\n" % exc, term.Color.RED, term.Attr.BOLD)) - raise exc + # if we're holding on to a tor process (running or not) then clean up after + # it so we can start a fresh instance + if self._tor_process: self.stop(quiet)

- # writes our testing torrc - torrc_dst = self.get_torrc_path() - try: - sys.stdout.write(term.format(" writing torrc (%s)... " % torrc_dst, term.Color.BLUE, term.Attr.BOLD)) - - torrc_file = open(torrc_dst, "w") - torrc_file.write(self._torrc_contents) - torrc_file.close() - - sys.stdout.write(term.format("done\n", term.Color.BLUE, term.Attr.BOLD)) - - for line in self._torrc_contents.strip().split("\n"): - print term.format(" %s" % line.strip(), term.Color.BLUE) - except Exception, exc: - sys.stdout.write(term.format("failed (%s)\n" % exc, term.Color.RED, term.Attr.BOLD)) - raise exc - finally: - print # extra newline - - def start(self): - """ - Initializes a tor process. This blocks until initialization completes or we - error out. + # apply any custom configuration attributes + if user_config: user_config.update(self._config)

- Raises: - OSError if we either fail to create the tor process or reached a timeout - without success - """ + # if 'test_directory' is unset then we make a new data directory in /tmp + # and clean it up when we're done

- def print_init_line(init_line): - """ - Prints output from tor's stdout while it starts up. - """ + config_test_dir = self._config["test.integ.test_directory"] + + if config_test_dir: + # makes paths relative of stem's base directory (the one above us) + if config_test_dir.startswith("./"): + stem_base = "/".join(__file__.split("/")[:-2]) + config_test_dir = stem_base + config_test_dir[1:]

- print term.format(" %s" % init_line, term.Color.BLUE) + self._test_dir = os.path.expanduser(config_test_dir) + else: + self._test_dir = tempfile.mktemp("-stem-integ")

- print term.format("Starting tor...", term.Color.BLUE, term.Attr.BOLD) - start_time = time.time() + self._torrc_contents = BASIC_TORRC % self._test_dir

try: - self._tor_process = stem.process.launch_tor(self.get_torrc_path(), print_init_line) - print term.format(" done (%i seconds)" % (time.time() - start_time), term.Color.BLUE, term.Attr.BOLD) - return - except KeyboardInterrupt: - sys.exit(1) # quietly terminate + self._run_setup(quiet) + self._start_tor(quiet) except OSError, exc: - print term.format(" failed to start tor: %s" % exc, term.Color.RED, term.Attr.BOLD) + self.stop(quiet) raise exc finally: - print # extra newline + self._runner_lock.release()

- def stop(self): + def stop(self, quiet = False): """ - Terminates our tor instance. + Stops our tor test instance and cleans up any temporary resources. + + Argument: + quiet (bool) - prints status information to stdout if False """

+ self._runner_lock.acquire() + _print_status("Shutting down tor... ", STATUS_ATTR, quiet) + if self._tor_process: - sys.stdout.write(term.format("Shutting down tor... ", term.Color.BLUE, term.Attr.BOLD)) self._tor_process.kill() self._tor_process.communicate() # blocks until the process is done - self._tor_process = None - shutil.rmtree(self._test_dir, ignore_errors=True) - sys.stdout.write(term.format("done\n", term.Color.BLUE, term.Attr.BOLD)) + + # if we've made a temporary data directory then clean it up + if self._test_dir and self._config["test.integ.test_directory"] == "": + shutil.rmtree(self._test_dir, ignore_errors = True) + + self._test_dir = "" + self._torrc_contents = "" + self._tor_process = None + + _print_status("done\n", STATUS_ATTR, quiet) + self._runner_lock.release()

- def get_pid(self): + def is_running(self): """ - Provides the process id of tor. + Checks if we're running a tor test instance and that it's alive.

Returns: - int pid for the tor process, or None if it isn't running + True if we have a running tor test instance, False otherwise """

- if self._tor_process: - return self._tor_process.pid - else: return None + # subprocess.Popen.poll() checks the return code, returning None if it's + # still going + + self._runner_lock.acquire() + is_running = self._tor_process and self._tor_process.poll() == None + + # If the tor process closed unexpectedly then this is probably the first + # place that we're realizing it. Clean up the temporary resources now since + # we might not end up calling stop() as normal. + + if not is_running: self.stop(True) + + self._runner_lock.release() + + return is_running + + def get_torrc_path(self): + """ + Provides the absolute path for where our testing torrc resides. + + Returns: + str with our torrc path + + Raises: + RunnerStopped if we aren't running + """ + + test_dir = self._get("_test_dir") + return os.path.join(test_dir, "torrc") + + def get_torrc_contents(self): + """ + Provides the contents of our torrc. + + Returns: + str with the contents of our torrc, lines are newline separated + + Raises: + RunnerStopped if we aren't running + """ + + return self._get("_torrc_contents")

def get_control_port(self): """ @@ -141,28 +189,155 @@ class Runner: Returns: int for the port tor's controller interface is bound to, None if it doesn't have one + + Raises: + RunnerStopped if we aren't running + ValueError if our torrc has a malformed ControlPort entry """

- # TODO: this will be fetched from torrc contents when we use custom configs - return 1111 + torrc_contents = self.get_torrc_contents() + + for line in torrc_contents.split("\n"): + line_comp = line.strip().split() + + if line_comp[0] == "ControlPort": + if len(line_comp) == 2 and line_comp[1].isdigit(): + return int(line_comp[1]) + else: + raise ValueError("Malformed ControlPort entry: %s" % line) + + # torrc doesn't have a ControlPort + return None

- def get_torrc_path(self): + def get_pid(self): """ - Provides the absolute path for where our testing torrc resides. + Provides the process id of the tor process.

Returns: - str with our torrc path + int pid for the tor process + + Raises: + RunnerStopped if we aren't running """

- return os.path.join(self._test_dir, "torrc") + tor_process = self._get("_tor_process") + return tor_process.pid

- def get_torrc_contents(self): + def _get(self, attr): """ - Provides the contents of our torrc. + Fetches one of our attributes in a thread safe manner, raising if we aren't + running. + + Arguments: + attr (str) - class variable that we want to fetch

Returns: - str with the contents of our torrc, lines are newline separated + value of the fetched variable + + Raises: + RunnerStopped if we aren't running """

- return self._torrc_contents + try: + self._runner_lock.acquire() + + if self.is_running(): + return self.__dict__[attr] + else: raise RunnerStopped() + finally: + self._runner_lock.release() + + def _run_setup(self, quiet): + """ + Makes a temporary runtime resources of our integration test instance. + + Arguments: + quiet (bool) - prints status information to stdout if False + + Raises: + OSError if unsuccessful + """ + + _print_status("Setting up a test instance...\n", STATUS_ATTR, quiet) + + # makes a temporary data directory if needed + try: + _print_status(" making test directory (%s)... " % self._test_dir, STATUS_ATTR, quiet) + + if os.path.exists(self._test_dir): + _print_status("skipped\n", STATUS_ATTR, quiet) + else: + os.makedirs(self._test_dir) + _print_status("done\n", STATUS_ATTR, quiet) + except OSError, exc: + _print_status("failed (%s)\n" % exc, ERROR_ATTR, quiet) + raise exc + + # writes our testing torrc + torrc_dst = os.path.join(self._test_dir, "torrc") + try: + _print_status(" writing torrc (%s)... " % torrc_dst, STATUS_ATTR, quiet) + + torrc_file = open(torrc_dst, "w") + torrc_file.write(self._torrc_contents) + torrc_file.close() + + _print_status("done\n", STATUS_ATTR, quiet) + + for line in self._torrc_contents.strip().split("\n"): + _print_status(" %s\n" % line.strip(), SUBSTATUS_ATTR, quiet) + + _print_status("\n", (), quiet) + except Exception, exc: + _print_status("failed (%s)\n\n" % exc, ERROR_ATTR, quiet) + raise exc + + def _start_tor(self, quiet): + """ + Initializes a tor process. This blocks until initialization completes or we + error out. + + Arguments: + quiet (bool) - prints status information to stdout if False + + Raises: + OSError if we either fail to create the tor process or reached a timeout + without success + """ + + _print_status("Starting tor...\n", STATUS_ATTR, quiet) + start_time = time.time() + + try: + # wait to fully complete if we're running tests with network activity, + # otherwise finish after local bootstraping + complete_percent = 100 if self._config["test.integ.run.online"] else 5 + + # prints output from tor's stdout while it starts up + print_init_line = lambda line: _print_status(" %s\n" % line, SUBSTATUS_ATTR, quiet) + + torrc_dst = os.path.join(self._test_dir, "torrc") + self._tor_process = stem.process.launch_tor(torrc_dst, complete_percent, print_init_line) + + runtime = time.time() - start_time + _print_status(" done (%i seconds)\n\n" % runtime, STATUS_ATTR, quiet) + except KeyboardInterrupt: + _print_status(" aborted starting tor: keyboard interrupt\n\n", ERROR_ATTR, quiet) + raise OSError("keyboard interrupt") + except OSError, exc: + _print_status(" failed to start tor: %s\n\n" % exc, ERROR_ATTR, quiet) + raise exc + +def _print_status(msg, attr = (), quiet = False): + """ + Short alias for printing status messages. + + Arguments: + msg (str) - text to be printed + attr (tuple) - list of term attributes to be applied to the text + quiet (bool) - no-op if true, prints otherwise + """ + + if not quiet: + sys.stdout.write(term.format(msg, *attr))

diff --git a/test/settings.cfg b/test/settings.cfg new file mode 100644 index 0000000..cff8d13 --- /dev/null +++ b/test/settings.cfg @@ -0,0 +1,17 @@ +# Integration Test Settings +# +# test.integ.test_directory +# Path used for our data directory and any temporary test resources. Relative +# paths are expanded in reference to the location of 'run_tests.py'. +# +# If set then the directory's contents are reused for future tests (so we +# have a faster startup and lower load on authorities). If set to an empty +# value then this makes a fresh data directory for each test run. +# +# test.integ.run.online +# Runs tests with network activity. If set then we'll wait for tor to fully +# bootstrap when starting, which won't happen without a network connection. + +test.integ.test_directory ./test/data +test.integ.run.online false +