[tor-commits] [stem/master] Overhaul of integ testing targets
atagar at torproject.org
atagar at torproject.org
Fri Jan 20 18:25:19 UTC 2012
commit 34da3669909c5e89957bcf77c92f2a00714f921c
Author: Damian Johnson <atagar at torproject.org>
Date: Fri Jan 20 10:20:07 2012 -0800
Overhaul of integ testing targets
The integration testing targets had grown organically and were partly defined
in mulitple places. Now that I've played with them a bit I've realized that
testing targets are purely a mapping between user input and runner attributes.
Hence I'm removing all notion of testing targets from runner.py, and instead
having the run_tests.py handle all conversion between test targets and runtime
attributes.
All target attributes are now generalized and stored in a large dictionary.
Good news is that this makes code readability and maintainability far easier.
Need a new target? Just add an enum and the attributes to the hash. Down side
is that this is more static configuration than code, and run_tests.py would
be far nicer without it.
I'll next look into moving this to a test_settings.cfg to abstract code from
testing configuration but references to constants (test.runner.OPT_* and
stem.version.Requirements) will take a little thought.
---
run_tests.py | 189 +++++++++++++++++++++++--------
test/integ/connection/authentication.py | 2 +-
test/runner.py | 47 ++------
3 files changed, 156 insertions(+), 82 deletions(-)
diff --git a/run_tests.py b/run_tests.py
index 64bde44..770953d 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -64,20 +64,103 @@ INTEG_TESTS = (
# Integration tests above the basic suite.
TARGETS = stem.util.enum.Enum(*[(v, v) for v in ("ONLINE", "RELATIVE", "CONN_NONE", "CONN_OPEN", "CONN_PASSWORD", "CONN_COOKIE", "CONN_MULTIPLE", "CONN_SOCKET", "CONN_SCOOKIE", "CONN_PTRACE", "CONN_ALL")])
+# Attributes that integ targets can have are...
+#
+# config
+# Configuration option with which this is synced. If an option is set via
+# both the config and '--target' argument then the argument takes precedence.
+#
+# description
+# The '--help' description of the target.
+#
+# prereq
+# Version that we need to run the target.
+#
+# torrc
+# Configuration options for the test instance. For each of these targets that
+# we have we make an integration test run.
+#
+# TODO: This is a very, very long block and it's only gonna get worse. Should
+# this be moved to a 'settings.cfg'? It might be problematic due to constants.
+
TARGET_ATTR = {
- TARGETS.ONLINE: ("test.integ.target.online", "Includes tests that require network activity."),
- TARGETS.RELATIVE: ("test.integ.target.relative_data_dir", "Uses a relative path for tor's data directory."),
- TARGETS.CONN_NONE: ("test.integ.target.connection.none", "Configuration without a way for controllers to connect."),
- TARGETS.CONN_OPEN: ("test.integ.target.connection.open", "Configuration with an open control port (default)."),
- TARGETS.CONN_PASSWORD: ("test.integ.target.connection.password", "Configuration with password authentication."),
- TARGETS.CONN_COOKIE: ("test.integ.target.connection.cookie", "Configuration with an authentication cookie."),
- TARGETS.CONN_MULTIPLE: ("test.integ.target.connection.multiple", "Configuration with both password and cookie authentication."),
- TARGETS.CONN_SOCKET: ("test.integ.target.connection.socket", "Configuration with a control socket."),
- TARGETS.CONN_SCOOKIE: ("test.integ.target.connection.scookie", "Configuration with a control socket and authentication cookie."),
- TARGETS.CONN_PTRACE: ("test.integ.target.connection.ptrace", "Configuration with an open control port and 'DisableDebuggerAttachment 0'"),
- TARGETS.CONN_ALL: ("test.integ.target.connection.all", "Runs integration tests for all connection configurations."),
+ TARGETS.ONLINE: {
+ "config": "test.integ.target.online",
+ "description": "Includes tests that require network activity.",
+ },
+ TARGETS.RELATIVE: {
+ "config": "test.integ.target.relative_data_dir",
+ "description": "Uses a relative path for tor's data directory.",
+ },
+ TARGETS.CONN_NONE: {
+ "config": "test.integ.target.connection.none",
+ "description": "Configuration without a way for controllers to connect.",
+ "torrc": (),
+ },
+ TARGETS.CONN_OPEN: {
+ "config": "test.integ.target.connection.open",
+ "description": "Configuration with an open control port (default).",
+ "torrc": (
+ test.runner.OPT_PORT,
+ ),
+ },
+ TARGETS.CONN_PASSWORD: {
+ "config": "test.integ.target.connection.password",
+ "description": "Configuration with password authentication.",
+ "torrc": (
+ test.runner.OPT_PORT,
+ test.runner.OPT_PASSWORD,
+ ),
+ },
+ TARGETS.CONN_COOKIE: {
+ "config": "test.integ.target.connection.cookie",
+ "description": "Configuration with an authentication cookie.",
+ "torrc": (
+ test.runner.OPT_PORT,
+ test.runner.OPT_COOKIE,
+ ),
+ },
+ TARGETS.CONN_MULTIPLE: {
+ "config": "test.integ.target.connection.multiple",
+ "description": "Configuration with both password and cookie authentication.",
+ "torrc": (
+ test.runner.OPT_PORT,
+ test.runner.OPT_PASSWORD,
+ test.runner.OPT_COOKIE,
+ ),
+ },
+ TARGETS.CONN_SOCKET: {
+ "config": "test.integ.target.connection.socket",
+ "description": "Configuration with a control socket.",
+ "torrc": (
+ test.runner.OPT_SOCKET,
+ ),
+ },
+ TARGETS.CONN_SCOOKIE: {
+ "config": "test.integ.target.connection.scookie",
+ "description": "Configuration with a control socket and authentication cookie.",
+ "torrc": (
+ test.runner.OPT_SOCKET,
+ test.runner.OPT_COOKIE,
+ ),
+ },
+ TARGETS.CONN_PTRACE: {
+ "config": "test.integ.target.connection.ptrace",
+ "description": "Configuration with an open control port and 'DisableDebuggerAttachment 0'",
+ "prereq": stem.version.Requirement.DISABLE_DEBUGGER_ATTACHMENT,
+ "torrc": (
+ test.runner.OPT_PORT,
+ test.runner.OPT_PTRACE,
+ ),
+ },
+ TARGETS.CONN_ALL: {
+ "config": "test.integ.target.connection.all",
+ "description": "Runs integration tests for all connection configurations.",
+ },
}
+DEFAULT_RUN_TARGET = TARGETS.CONN_OPEN
+
HELP_MSG = """Usage runTests.py [OPTION]
Runs tests for the stem library.
@@ -124,6 +207,7 @@ if __name__ == '__main__':
run_integ_tests = False
config_path = None
test_config = stem.util.conf.get_config("test")
+ override_targets = []
logging_runlevel = None
tor_cmd = "tor"
@@ -151,9 +235,7 @@ if __name__ == '__main__':
print "Invalid integration target: %s" % target
sys.exit(1)
else:
- # sets the configuration flag
- config_flag = TARGET_ATTR[target][0]
- test_config.set(config_flag, "true")
+ override_targets.append(target)
elif opt in ("-l", "--log"):
logging_runlevel = arg.upper()
@@ -177,7 +259,7 @@ if __name__ == '__main__':
target_lines = []
for target in TARGETS:
- target_lines.append(description_format % (target, TARGET_ATTR[target][1]))
+ target_lines.append(description_format % (target, TARGET_ATTR[target]["description"]))
print HELP_MSG % "\n ".join(target_lines)
sys.exit()
@@ -208,6 +290,13 @@ if __name__ == '__main__':
print
+ # Set the configuration flag for our '--target' arguments. This is meant to
+ # override our configuration flags if both set a target.
+
+ for target in override_targets:
+ target_config = TARGET_ATTR[target].get("config")
+ if target_config: test_config.set(target_config, "true")
+
error_tracker = test.output.ErrorTracker()
output_filters = (
error_tracker.get_filter(),
@@ -240,46 +329,46 @@ if __name__ == '__main__':
print_divider("INTEGRATION TESTS", True)
integ_runner = test.runner.get_runner()
- # queue up all of the tor configurations we want to run the integration
- # tests on
+ # Queue up all the targets with torrc options we want to run against.
- connection_types = []
+ integ_run_targets = []
+ all_run_targets = [t for t in TARGETS if "torrc" in TARGET_ATTR[t]]
if test_config.get("test.integ.target.connection.all", False):
- connection_types = list(test.runner.TorConnection)
+ # test against everything with torrc options
+ integ_run_targets = all_run_targets
else:
- # mapping of config type to (enum, default) tuple
- conn_type_mappings = {
- "none": test.runner.TorConnection.NONE,
- "open": test.runner.TorConnection.OPEN,
- "password": test.runner.TorConnection.PASSWORD,
- "cookie": test.runner.TorConnection.COOKIE,
- "multiple": test.runner.TorConnection.MULTIPLE,
- "socket": test.runner.TorConnection.SOCKET,
- "scookie": test.runner.TorConnection.SCOOKIE,
- "ptrace": test.runner.TorConnection.PTRACE,
- }
-
- for type_key in conn_type_mappings:
- if test_config.get("test.integ.target.connection.%s" % type_key, False):
- connection_types.append(conn_type_mappings[type_key])
+ for target in all_run_targets:
+ target_config = TARGET_ATTR[target].get("config")
+
+ if target_config and test_config.get(target_config, False):
+ integ_run_targets.append(target)
+
+ # if we didn't specify any targets then use the default
+ if not integ_run_targets:
+ integ_run_targets.append(DEFAULT_RUN_TARGET)
+
+ # Determine targets we don't meet the prereqs for. Warnings are given about
+ # these at the end of the test run so they're more noticeable.
- # TorConnection.OPEN is the default if we don't have any defined
- if not connection_types:
- connection_types = [test.runner.TorConnection.OPEN]
+ our_version, skip_targets = None, []
- for connection_type in connection_types:
- if connection_type == test.runner.TorConnection.PTRACE:
- our_version = stem.version.get_system_tor_version(tor_cmd)
- req_version = stem.version.Requirement.DISABLE_DEBUGGER_ATTACHMENT
+ for target in integ_run_targets:
+ target_prereq = TARGET_ATTR[target].get("prereq")
+
+ if target_prereq:
+ # lazy loaded to skip system call if we don't have any prereqs
+ if not our_version:
+ our_version = stem.version.get_system_tor_version(tor_cmd)
- if our_version < req_version:
- print term.format("Unable to run CONN_PTRACE target: DisableDebuggerAttachment was added in %s" % req_version, term.Color.RED, term.Attr.BOLD)
- print
- continue
+ if our_version < target_prereq:
+ skip_targets.append(target)
+
+ for target in integ_run_targets:
+ if target in skip_targets: continue
try:
- integ_runner.start(tor_cmd, connection_type = connection_type)
+ integ_runner.start(tor_cmd, extra_torrc_opts = TARGET_ATTR[target].get("torrc", []))
print term.format("Running tests...", term.Color.BLUE, term.Attr.BOLD)
print
@@ -299,6 +388,14 @@ if __name__ == '__main__':
finally:
integ_runner.stop()
+ if skip_targets:
+ print
+
+ for target in skip_targets:
+ print term.format("Unable to run target %s, this requires tor version %s" % (target, TARGET_ATTR[target]["prereq"]), term.Color.RED, term.Attr.BOLD)
+
+ print
+
# TODO: note unused config options afterward?
runtime_label = "(%i seconds)" % (time.time() - start_time)
diff --git a/test/integ/connection/authentication.py b/test/integ/connection/authentication.py
index 519e40a..8c84490 100644
--- a/test/integ/connection/authentication.py
+++ b/test/integ/connection/authentication.py
@@ -198,7 +198,7 @@ class TestAuthenticate(unittest.TestCase):
fake_cookie.write("0" * 32)
fake_cookie.close()
- if self._can_authenticate(test.runner.TorConnection.NONE):
+ if self._can_authenticate(stem.connection.AuthMethod.NONE):
# authentication will work anyway
self._check_auth(auth_type, auth_value)
else:
diff --git a/test/runner.py b/test/runner.py
index 50a1031..37697e7 100644
--- a/test/runner.py
+++ b/test/runner.py
@@ -25,11 +25,9 @@ import os
import sys
import time
import stat
-import socket
import shutil
import logging
import tempfile
-import binascii
import threading
import stem.socket
@@ -46,12 +44,6 @@ DEFAULT_CONFIG = {
"test.integ.target.relative_data_dir": False,
}
-# Methods for connecting to tor. General integration tests only run with the
-# DEFAULT_TOR_CONNECTION, but expanded integ tests will run with all of them.
-
-TorConnection = stem.util.enum.Enum("NONE", "OPEN", "PASSWORD", "COOKIE", "MULTIPLE", "SOCKET", "SCOOKIE", "PTRACE")
-DEFAULT_TOR_CONNECTION = TorConnection.OPEN
-
STATUS_ATTR = (term.Color.BLUE, term.Attr.BOLD)
SUBSTATUS_ATTR = (term.Color.BLUE, )
ERROR_ATTR = (term.Color.RED, term.Attr.BOLD)
@@ -79,19 +71,6 @@ OPT_PASSWORD = "HashedControlPassword 16:8C423A41EF4A542C6078985270AE28A4E04D056
OPT_SOCKET = "ControlSocket %s" % CONTROL_SOCKET_PATH
OPT_PTRACE = "DisableDebuggerAttachment 0"
-# mapping of TorConnection to their options
-
-CONNECTION_OPTS = {
- TorConnection.NONE: [],
- TorConnection.OPEN: [OPT_PORT],
- TorConnection.PASSWORD: [OPT_PORT, OPT_PASSWORD],
- TorConnection.COOKIE: [OPT_PORT, OPT_COOKIE],
- TorConnection.MULTIPLE: [OPT_PORT, OPT_PASSWORD, OPT_COOKIE],
- TorConnection.SOCKET: [OPT_SOCKET],
- TorConnection.SCOOKIE: [OPT_SOCKET, OPT_COOKIE],
- TorConnection.PTRACE: [OPT_PORT, OPT_PTRACE],
-}
-
def get_runner():
"""
Singleton for the runtime context of integration tests.
@@ -101,17 +80,15 @@ def get_runner():
if not INTEG_RUNNER: INTEG_RUNNER = Runner()
return INTEG_RUNNER
-def get_torrc(connection_type = DEFAULT_TOR_CONNECTION):
+def get_torrc(extra_torrc_opts):
"""
Provides a basic torrc with the given connection method. Hashed passwords are
for "pw".
"""
- connection_opt, torrc = CONNECTION_OPTS[connection_type], BASE_TORRC
-
- if connection_opt:
- return torrc + "\n".join(connection_opt) + "\n"
- else: return torrc
+ if extra_torrc_opts:
+ return BASE_TORRC + "\n".join(extra_torrc_opts) + "\n"
+ else: return BASE_TORRC
def exercise_socket(test_case, control_socket):
"""
@@ -145,18 +122,17 @@ class Runner:
self._test_dir = ""
self._tor_cwd = ""
self._torrc_contents = ""
- self._connection_type = None
+ self._extra_torrc_opts = None
self._tor_process = None
- def start(self, tor_cmd, connection_type = DEFAULT_TOR_CONNECTION, quiet = False):
+ def start(self, tor_cmd, extra_torrc_opts, quiet = False):
"""
Makes temporary testing resources and starts tor, blocking until it
completes.
Arguments:
tor_cmd (str) - command to start tor with
- connection_type (TorConnection) - method for controllers to authenticate
- to tor
+ extra_torrc_opts (list) - additional torrc options for our test instance
quiet (bool) - if False then this prints status information as we start
up to stdout
@@ -194,8 +170,8 @@ class Runner:
os.chdir(tor_cwd)
data_dir_path = "./%s" % os.path.basename(self._test_dir)
- self._connection_type = connection_type
- self._torrc_contents = get_torrc(connection_type) % data_dir_path
+ self._extra_torrc_opts = extra_torrc_opts
+ self._torrc_contents = get_torrc(extra_torrc_opts) % data_dir_path
try:
self._tor_cwd = os.getcwd()
@@ -233,7 +209,7 @@ class Runner:
self._test_dir = ""
self._tor_cwd = ""
self._torrc_contents = ""
- self._connection_type = None
+ self._extra_torrc_opts = None
self._tor_process = None
_print_status("done\n", STATUS_ATTR, quiet)
@@ -364,7 +340,8 @@ class Runner:
list of connection contstants (test.runner.OPT_*) we're running with
"""
- return CONNECTION_OPTS[self._connection_type]
+ # TODO: rename function
+ return self._extra_torrc_opts
def get_pid(self):
"""
More information about the tor-commits
mailing list