commit 34da3669909c5e89957bcf77c92f2a00714f921c Author: Damian Johnson atagar@torproject.org Date: Fri Jan 20 10:20:07 2012 -0800
Overhaul of integ testing targets
The integration testing targets had grown organically and were partly defined in mulitple places. Now that I've played with them a bit I've realized that testing targets are purely a mapping between user input and runner attributes. Hence I'm removing all notion of testing targets from runner.py, and instead having the run_tests.py handle all conversion between test targets and runtime attributes.
All target attributes are now generalized and stored in a large dictionary. Good news is that this makes code readability and maintainability far easier. Need a new target? Just add an enum and the attributes to the hash. Down side is that this is more static configuration than code, and run_tests.py would be far nicer without it.
I'll next look into moving this to a test_settings.cfg to abstract code from testing configuration but references to constants (test.runner.OPT_* and stem.version.Requirements) will take a little thought. --- run_tests.py | 189 +++++++++++++++++++++++-------- test/integ/connection/authentication.py | 2 +- test/runner.py | 47 ++------ 3 files changed, 156 insertions(+), 82 deletions(-)
diff --git a/run_tests.py b/run_tests.py index 64bde44..770953d 100755 --- a/run_tests.py +++ b/run_tests.py @@ -64,20 +64,103 @@ INTEG_TESTS = ( # Integration tests above the basic suite. TARGETS = stem.util.enum.Enum(*[(v, v) for v in ("ONLINE", "RELATIVE", "CONN_NONE", "CONN_OPEN", "CONN_PASSWORD", "CONN_COOKIE", "CONN_MULTIPLE", "CONN_SOCKET", "CONN_SCOOKIE", "CONN_PTRACE", "CONN_ALL")])
+# Attributes that integ targets can have are... +# +# config +# Configuration option with which this is synced. If an option is set via +# both the config and '--target' argument then the argument takes precedence. +# +# description +# The '--help' description of the target. +# +# prereq +# Version that we need to run the target. +# +# torrc +# Configuration options for the test instance. For each of these targets that +# we have we make an integration test run. +# +# TODO: This is a very, very long block and it's only gonna get worse. Should +# this be moved to a 'settings.cfg'? It might be problematic due to constants. + TARGET_ATTR = { - TARGETS.ONLINE: ("test.integ.target.online", "Includes tests that require network activity."), - TARGETS.RELATIVE: ("test.integ.target.relative_data_dir", "Uses a relative path for tor's data directory."), - TARGETS.CONN_NONE: ("test.integ.target.connection.none", "Configuration without a way for controllers to connect."), - TARGETS.CONN_OPEN: ("test.integ.target.connection.open", "Configuration with an open control port (default)."), - TARGETS.CONN_PASSWORD: ("test.integ.target.connection.password", "Configuration with password authentication."), - TARGETS.CONN_COOKIE: ("test.integ.target.connection.cookie", "Configuration with an authentication cookie."), - TARGETS.CONN_MULTIPLE: ("test.integ.target.connection.multiple", "Configuration with both password and cookie authentication."), - TARGETS.CONN_SOCKET: ("test.integ.target.connection.socket", "Configuration with a control socket."), - TARGETS.CONN_SCOOKIE: ("test.integ.target.connection.scookie", "Configuration with a control socket and authentication cookie."), - TARGETS.CONN_PTRACE: ("test.integ.target.connection.ptrace", "Configuration with an open control port and 'DisableDebuggerAttachment 0'"), - TARGETS.CONN_ALL: ("test.integ.target.connection.all", "Runs integration tests for all connection configurations."), + TARGETS.ONLINE: { + "config": "test.integ.target.online", + "description": "Includes tests that require network activity.", + }, + TARGETS.RELATIVE: { + "config": "test.integ.target.relative_data_dir", + "description": "Uses a relative path for tor's data directory.", + }, + TARGETS.CONN_NONE: { + "config": "test.integ.target.connection.none", + "description": "Configuration without a way for controllers to connect.", + "torrc": (), + }, + TARGETS.CONN_OPEN: { + "config": "test.integ.target.connection.open", + "description": "Configuration with an open control port (default).", + "torrc": ( + test.runner.OPT_PORT, + ), + }, + TARGETS.CONN_PASSWORD: { + "config": "test.integ.target.connection.password", + "description": "Configuration with password authentication.", + "torrc": ( + test.runner.OPT_PORT, + test.runner.OPT_PASSWORD, + ), + }, + TARGETS.CONN_COOKIE: { + "config": "test.integ.target.connection.cookie", + "description": "Configuration with an authentication cookie.", + "torrc": ( + test.runner.OPT_PORT, + test.runner.OPT_COOKIE, + ), + }, + TARGETS.CONN_MULTIPLE: { + "config": "test.integ.target.connection.multiple", + "description": "Configuration with both password and cookie authentication.", + "torrc": ( + test.runner.OPT_PORT, + test.runner.OPT_PASSWORD, + test.runner.OPT_COOKIE, + ), + }, + TARGETS.CONN_SOCKET: { + "config": "test.integ.target.connection.socket", + "description": "Configuration with a control socket.", + "torrc": ( + test.runner.OPT_SOCKET, + ), + }, + TARGETS.CONN_SCOOKIE: { + "config": "test.integ.target.connection.scookie", + "description": "Configuration with a control socket and authentication cookie.", + "torrc": ( + test.runner.OPT_SOCKET, + test.runner.OPT_COOKIE, + ), + }, + TARGETS.CONN_PTRACE: { + "config": "test.integ.target.connection.ptrace", + "description": "Configuration with an open control port and 'DisableDebuggerAttachment 0'", + "prereq": stem.version.Requirement.DISABLE_DEBUGGER_ATTACHMENT, + "torrc": ( + test.runner.OPT_PORT, + test.runner.OPT_PTRACE, + ), + }, + TARGETS.CONN_ALL: { + "config": "test.integ.target.connection.all", + "description": "Runs integration tests for all connection configurations.", + }, }
+DEFAULT_RUN_TARGET = TARGETS.CONN_OPEN + HELP_MSG = """Usage runTests.py [OPTION] Runs tests for the stem library.
@@ -124,6 +207,7 @@ if __name__ == '__main__': run_integ_tests = False config_path = None test_config = stem.util.conf.get_config("test") + override_targets = [] logging_runlevel = None tor_cmd = "tor"
@@ -151,9 +235,7 @@ if __name__ == '__main__': print "Invalid integration target: %s" % target sys.exit(1) else: - # sets the configuration flag - config_flag = TARGET_ATTR[target][0] - test_config.set(config_flag, "true") + override_targets.append(target) elif opt in ("-l", "--log"): logging_runlevel = arg.upper()
@@ -177,7 +259,7 @@ if __name__ == '__main__':
target_lines = [] for target in TARGETS: - target_lines.append(description_format % (target, TARGET_ATTR[target][1])) + target_lines.append(description_format % (target, TARGET_ATTR[target]["description"]))
print HELP_MSG % "\n ".join(target_lines) sys.exit() @@ -208,6 +290,13 @@ if __name__ == '__main__':
+ # Set the configuration flag for our '--target' arguments. This is meant to + # override our configuration flags if both set a target. + + for target in override_targets: + target_config = TARGET_ATTR[target].get("config") + if target_config: test_config.set(target_config, "true") + error_tracker = test.output.ErrorTracker() output_filters = ( error_tracker.get_filter(), @@ -240,46 +329,46 @@ if __name__ == '__main__': print_divider("INTEGRATION TESTS", True) integ_runner = test.runner.get_runner()
- # queue up all of the tor configurations we want to run the integration - # tests on + # Queue up all the targets with torrc options we want to run against.
- connection_types = [] + integ_run_targets = [] + all_run_targets = [t for t in TARGETS if "torrc" in TARGET_ATTR[t]]
if test_config.get("test.integ.target.connection.all", False): - connection_types = list(test.runner.TorConnection) + # test against everything with torrc options + integ_run_targets = all_run_targets else: - # mapping of config type to (enum, default) tuple - conn_type_mappings = { - "none": test.runner.TorConnection.NONE, - "open": test.runner.TorConnection.OPEN, - "password": test.runner.TorConnection.PASSWORD, - "cookie": test.runner.TorConnection.COOKIE, - "multiple": test.runner.TorConnection.MULTIPLE, - "socket": test.runner.TorConnection.SOCKET, - "scookie": test.runner.TorConnection.SCOOKIE, - "ptrace": test.runner.TorConnection.PTRACE, - } - - for type_key in conn_type_mappings: - if test_config.get("test.integ.target.connection.%s" % type_key, False): - connection_types.append(conn_type_mappings[type_key]) + for target in all_run_targets: + target_config = TARGET_ATTR[target].get("config") + + if target_config and test_config.get(target_config, False): + integ_run_targets.append(target) + + # if we didn't specify any targets then use the default + if not integ_run_targets: + integ_run_targets.append(DEFAULT_RUN_TARGET) + + # Determine targets we don't meet the prereqs for. Warnings are given about + # these at the end of the test run so they're more noticeable.
- # TorConnection.OPEN is the default if we don't have any defined - if not connection_types: - connection_types = [test.runner.TorConnection.OPEN] + our_version, skip_targets = None, []
- for connection_type in connection_types: - if connection_type == test.runner.TorConnection.PTRACE: - our_version = stem.version.get_system_tor_version(tor_cmd) - req_version = stem.version.Requirement.DISABLE_DEBUGGER_ATTACHMENT + for target in integ_run_targets: + target_prereq = TARGET_ATTR[target].get("prereq") + + if target_prereq: + # lazy loaded to skip system call if we don't have any prereqs + if not our_version: + our_version = stem.version.get_system_tor_version(tor_cmd)
- if our_version < req_version: - print term.format("Unable to run CONN_PTRACE target: DisableDebuggerAttachment was added in %s" % req_version, term.Color.RED, term.Attr.BOLD) - print - continue + if our_version < target_prereq: + skip_targets.append(target) + + for target in integ_run_targets: + if target in skip_targets: continue
try: - integ_runner.start(tor_cmd, connection_type = connection_type) + integ_runner.start(tor_cmd, extra_torrc_opts = TARGET_ATTR[target].get("torrc", []))
print term.format("Running tests...", term.Color.BLUE, term.Attr.BOLD) print @@ -299,6 +388,14 @@ if __name__ == '__main__': finally: integ_runner.stop()
+ if skip_targets: + print + + for target in skip_targets: + print term.format("Unable to run target %s, this requires tor version %s" % (target, TARGET_ATTR[target]["prereq"]), term.Color.RED, term.Attr.BOLD) + + print + # TODO: note unused config options afterward?
runtime_label = "(%i seconds)" % (time.time() - start_time) diff --git a/test/integ/connection/authentication.py b/test/integ/connection/authentication.py index 519e40a..8c84490 100644 --- a/test/integ/connection/authentication.py +++ b/test/integ/connection/authentication.py @@ -198,7 +198,7 @@ class TestAuthenticate(unittest.TestCase): fake_cookie.write("0" * 32) fake_cookie.close()
- if self._can_authenticate(test.runner.TorConnection.NONE): + if self._can_authenticate(stem.connection.AuthMethod.NONE): # authentication will work anyway self._check_auth(auth_type, auth_value) else: diff --git a/test/runner.py b/test/runner.py index 50a1031..37697e7 100644 --- a/test/runner.py +++ b/test/runner.py @@ -25,11 +25,9 @@ import os import sys import time import stat -import socket import shutil import logging import tempfile -import binascii import threading
import stem.socket @@ -46,12 +44,6 @@ DEFAULT_CONFIG = { "test.integ.target.relative_data_dir": False, }
-# Methods for connecting to tor. General integration tests only run with the -# DEFAULT_TOR_CONNECTION, but expanded integ tests will run with all of them. - -TorConnection = stem.util.enum.Enum("NONE", "OPEN", "PASSWORD", "COOKIE", "MULTIPLE", "SOCKET", "SCOOKIE", "PTRACE") -DEFAULT_TOR_CONNECTION = TorConnection.OPEN - STATUS_ATTR = (term.Color.BLUE, term.Attr.BOLD) SUBSTATUS_ATTR = (term.Color.BLUE, ) ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) @@ -79,19 +71,6 @@ OPT_PASSWORD = "HashedControlPassword 16:8C423A41EF4A542C6078985270AE28A4E04D056 OPT_SOCKET = "ControlSocket %s" % CONTROL_SOCKET_PATH OPT_PTRACE = "DisableDebuggerAttachment 0"
-# mapping of TorConnection to their options - -CONNECTION_OPTS = { - TorConnection.NONE: [], - TorConnection.OPEN: [OPT_PORT], - TorConnection.PASSWORD: [OPT_PORT, OPT_PASSWORD], - TorConnection.COOKIE: [OPT_PORT, OPT_COOKIE], - TorConnection.MULTIPLE: [OPT_PORT, OPT_PASSWORD, OPT_COOKIE], - TorConnection.SOCKET: [OPT_SOCKET], - TorConnection.SCOOKIE: [OPT_SOCKET, OPT_COOKIE], - TorConnection.PTRACE: [OPT_PORT, OPT_PTRACE], -} - def get_runner(): """ Singleton for the runtime context of integration tests. @@ -101,17 +80,15 @@ def get_runner(): if not INTEG_RUNNER: INTEG_RUNNER = Runner() return INTEG_RUNNER
-def get_torrc(connection_type = DEFAULT_TOR_CONNECTION): +def get_torrc(extra_torrc_opts): """ Provides a basic torrc with the given connection method. Hashed passwords are for "pw". """
- connection_opt, torrc = CONNECTION_OPTS[connection_type], BASE_TORRC - - if connection_opt: - return torrc + "\n".join(connection_opt) + "\n" - else: return torrc + if extra_torrc_opts: + return BASE_TORRC + "\n".join(extra_torrc_opts) + "\n" + else: return BASE_TORRC
def exercise_socket(test_case, control_socket): """ @@ -145,18 +122,17 @@ class Runner: self._test_dir = "" self._tor_cwd = "" self._torrc_contents = "" - self._connection_type = None + self._extra_torrc_opts = None self._tor_process = None
- def start(self, tor_cmd, connection_type = DEFAULT_TOR_CONNECTION, quiet = False): + def start(self, tor_cmd, extra_torrc_opts, quiet = False): """ Makes temporary testing resources and starts tor, blocking until it completes.
Arguments: tor_cmd (str) - command to start tor with - connection_type (TorConnection) - method for controllers to authenticate - to tor + extra_torrc_opts (list) - additional torrc options for our test instance quiet (bool) - if False then this prints status information as we start up to stdout
@@ -194,8 +170,8 @@ class Runner: os.chdir(tor_cwd) data_dir_path = "./%s" % os.path.basename(self._test_dir)
- self._connection_type = connection_type - self._torrc_contents = get_torrc(connection_type) % data_dir_path + self._extra_torrc_opts = extra_torrc_opts + self._torrc_contents = get_torrc(extra_torrc_opts) % data_dir_path
try: self._tor_cwd = os.getcwd() @@ -233,7 +209,7 @@ class Runner: self._test_dir = "" self._tor_cwd = "" self._torrc_contents = "" - self._connection_type = None + self._extra_torrc_opts = None self._tor_process = None
_print_status("done\n", STATUS_ATTR, quiet) @@ -364,7 +340,8 @@ class Runner: list of connection contstants (test.runner.OPT_*) we're running with """
- return CONNECTION_OPTS[self._connection_type] + # TODO: rename function + return self._extra_torrc_opts
def get_pid(self): """