commit 10d0bd80985c6dbdf8423890d0cd5f7f346ae559 Author: Damian Johnson atagar@torproject.org Date: Fri Apr 12 07:56:32 2013 -0700
Untangling integration targets from configuration
Even I was starting to get confused by 'em. We've always implicitely had two kinds of targets: run targets which constitute an integraion test run and attribute targets which have a non-torrc change on all test runs (such as being in a chroot).
Making this division more explicit and dropping the confusing configuration dance we were doing. --- run_tests.py | 87 +++++++++++++++++++++------------------------------- test/runner.py | 38 +++++++++++++++++----- test/settings.cfg | 41 ------------------------- 3 files changed, 64 insertions(+), 102 deletions(-)
diff --git a/run_tests.py b/run_tests.py index ca43539..66e7fc2 100755 --- a/run_tests.py +++ b/run_tests.py @@ -25,34 +25,20 @@ import test.output import test.runner import test.static_checks
+from test.runner import Target + OPT = "auist:l:h" OPT_EXPANDED = ["all", "unit", "integ", "style", "python3", "clean", "targets=", "test=", "log=", "tor=", "help"] DIVIDER = "=" * 70
CONFIG = stem.util.conf.config_dict("test", { "msg.help": "", - "target.config": {}, "target.description": {}, "target.prereq": {}, "target.torrc": {}, "integ.test_directory": "./test/data", })
-Target = stem.util.enum.UppercaseEnum( - "ONLINE", - "RELATIVE", - "CHROOT", - "RUN_NONE", - "RUN_OPEN", - "RUN_PASSWORD", - "RUN_COOKIE", - "RUN_MULTIPLE", - "RUN_SOCKET", - "RUN_SCOOKIE", - "RUN_PTRACE", - "RUN_ALL", -) - DEFAULT_RUN_TARGET = Target.RUN_OPEN
ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) @@ -214,6 +200,18 @@ if __name__ == '__main__': logging_runlevel = None tor_path = "tor"
+ # Integration testing targets fall into two categories: + # + # * Run Targets (like RUN_COOKIE and RUN_PTRACE) which customize our torrc. + # We do an integration test run for each run target we get. + # + # * Attribute Target (like CHROOT and ONLINE) which indicates + # non-configuration changes to ur test runs. These are applied to all + # integration runs that we perform. + + run_targets = [DEFAULT_RUN_TARGET] + attribute_targets = [] + for opt, arg in opts: if opt in ("-a", "--all"): run_unit = True @@ -232,7 +230,11 @@ if __name__ == '__main__': elif opt in ("-t", "--targets"): integ_targets = arg.split(",")
- # validates the targets + run_targets = [] + all_run_targets = [t for t in Target if CONFIG["target.torrc"].get(t) is not None] + + # validates the targets and split them into run and attribute targets + if not integ_targets: print "No targets provided" sys.exit(1) @@ -241,11 +243,16 @@ if __name__ == '__main__': if not target in Target: print "Invalid integration target: %s" % target sys.exit(1) + elif target in all_run_targets: + run_targets.append(target) else: - target_config = test_config.get("target.config", {}).get(target) + attribute_targets.append(target) + + # check if we were told to use all run targets
- if target_config: - test_config.set(target_config, "true") + if Target.RUN_ALL in attribute_targets: + attribute_targets.remove(Target.RUN_ALL) + run_targets = all_run_targets elif opt in ("-l", "--test"): test_prefix = arg elif opt in ("-l", "--log"): @@ -346,43 +353,19 @@ if __name__ == '__main__': test.output.print_divider("INTEGRATION TESTS", True) integ_runner = test.runner.get_runner()
- # Queue up all the targets with torrc options we want to run against. - - integ_run_targets = [] - all_run_targets = [t for t in Target if CONFIG["target.torrc"].get(t) is not None] - - if test_config.get("integ.target.run.all", False): - # test against everything with torrc options - integ_run_targets = all_run_targets - else: - for target in all_run_targets: - target_config = CONFIG["target.config"].get(target) - - if target_config and test_config.get(target_config, False): - integ_run_targets.append(target) - - # if we didn't specify any targets then use the default - if not integ_run_targets: - integ_run_targets.append(DEFAULT_RUN_TARGET) - # Determine targets we don't meet the prereqs for. Warnings are given about # these at the end of the test run so they're more noticeable.
- our_version, skip_targets = None, [] - - for target in integ_run_targets: - target_prereq = CONFIG["target.prereq"].get(target) + our_version = stem.version.get_system_tor_version(tor_path) + skip_targets = []
- if target_prereq: - # lazy loaded to skip system call if we don't have any prereqs - if not our_version: - our_version = stem.version.get_system_tor_version(tor_path) + for target in run_targets: + # check if we meet this target's tor version prerequisites
- if our_version < stem.version.Requirement[target_prereq]: - skip_targets.append(target) + target_prereq = CONFIG["target.prereq"].get(target)
- for target in integ_run_targets: - if target in skip_targets: + if target_prereq and our_version < stem.version.Requirement[target_prereq]: + skip_targets.append(target) continue
error_tracker.set_category(target) @@ -402,7 +385,7 @@ if __name__ == '__main__': test.output.print_line("'%s' isn't a test.runner.Torrc enumeration" % opt) sys.exit(1)
- integ_runner.start(tor_path, extra_torrc_opts = torrc_opts) + integ_runner.start(target, attribute_targets, tor_path, extra_torrc_opts = torrc_opts)
test.output.print_line("Running tests...", term.Color.BLUE, term.Attr.BOLD) print diff --git a/test/runner.py b/test/runner.py index 9e809fe..d869d6a 100644 --- a/test/runner.py +++ b/test/runner.py @@ -65,13 +65,25 @@ from stem.util import term CONFIG = stem.util.conf.config_dict("test", { "integ.test_directory": "./test/data", "integ.log": "./test/data/log", - "integ.target.online": False, - "integ.target.relative_data_dir": False, - "integ.target.chroot": False, "test.unit_tests": "", "test.integ_tests": "", })
+Target = stem.util.enum.UppercaseEnum( + "ONLINE", + "RELATIVE", + "CHROOT", + "RUN_NONE", + "RUN_OPEN", + "RUN_PASSWORD", + "RUN_COOKIE", + "RUN_MULTIPLE", + "RUN_SOCKET", + "RUN_SCOOKIE", + "RUN_PTRACE", + "RUN_ALL", +) + STATUS_ATTR = (term.Color.BLUE, term.Attr.BOLD) SUBSTATUS_ATTR = (term.Color.BLUE, ) ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) @@ -170,7 +182,7 @@ def require_online(test_case): :returns: True if test should be skipped, False otherwise """
- if not CONFIG["integ.target.online"]: + if not Target.ONLINE in test.runner.get_runner().attribute_targets: skip(test_case, "(requires online target)") return True
@@ -305,6 +317,9 @@ class _MockChrootFile(object):
class Runner(object): def __init__(self): + self.run_target = None + self.attribute_targets = [] + self._runner_lock = threading.RLock()
# runtime attributes, set by the start method @@ -320,11 +335,13 @@ class Runner(object):
self._original_recv_message = None
- def start(self, tor_cmd, extra_torrc_opts): + def start(self, run_target, attribute_targets, tor_cmd, extra_torrc_opts): """ Makes temporary testing resources and starts tor, blocking until it completes.
+ :param Target run_target: configuration we're running with + :param list attribute_targets: **Targets** for our non-configuration attributes :param str tor_cmd: command to start tor with :param list extra_torrc_opts: additional torrc options for our test instance
@@ -332,6 +349,9 @@ class Runner(object): """
with self._runner_lock: + self.run_target = run_target + self.attribute_targets = attribute_targets + # if we're holding on to a tor process (running or not) then clean up after # it so we can start a fresh instance
@@ -352,7 +372,7 @@ class Runner(object):
original_cwd, data_dir_path = os.getcwd(), self._test_dir
- if CONFIG["integ.target.relative_data_dir"]: + if Target.RELATIVE in self.attribute_targets: tor_cwd = os.path.dirname(self._test_dir)
if not os.path.exists(tor_cwd): @@ -376,7 +396,7 @@ class Runner(object): # strip the testing directory from recv_message responses if we're # simulating a chroot setup
- if CONFIG["integ.target.chroot"] and not self._original_recv_message: + if Target.CHROOT in self.attribute_targets and not self._original_recv_message: # TODO: when we have a function for telling stem the chroot we'll # need to set that too
@@ -389,7 +409,7 @@ class Runner(object): stem.socket.recv_message = _chroot_recv_message
# revert our cwd back to normal - if CONFIG["integ.target.relative_data_dir"]: + if Target.RELATIVE in self.attribute_targets: os.chdir(original_cwd) except OSError, exc: raise exc @@ -768,7 +788,7 @@ class Runner(object): try: # wait to fully complete if we're running tests with network activity, # otherwise finish after local bootstraping - complete_percent = 100 if CONFIG["integ.target.online"] else 5 + complete_percent = 100 if Target.ONLINE in self.attribute_targets else 5
# prints output from tor's stdout while it starts up print_init_line = lambda line: test.output.print_line(" %s" % line, *SUBSTATUS_ATTR) diff --git a/test/settings.cfg b/test/settings.cfg index 81bb567..625a482 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -14,35 +14,10 @@ # integ.log # Path runtime logs are placed. Relative paths are expanded in reference to # 'run_tests.py'. Logging is disabled if set ot an empty value. -# -# integ.target.online -# Runs tests with network activity. If set then we'll wait for tor to fully -# bootstrap when starting, which won't happen without a network connection. -# -# integ.target.relative_data_dir -# Uses a relative path for the tor data directory if set. -# -# integ.target.run.* -# Runs the integration test suite for all of the given connection and -# authentication configurations. If the 'all' option is set then the other -# flags are ignored.
integ.test_directory ./test/data integ.log ./test/data/log
-integ.target.online false -integ.target.relative_data_dir false -integ.target.chroot false -integ.target.run.none false -integ.target.run.open false -integ.target.run.password false -integ.target.run.cookie false -integ.target.run.muiltipe false -integ.target.run.socket false -integ.target.run.scookie false -integ.target.run.ptrace false -integ.target.run.all false - # The following are less testing framework attributes that aren't as commonly # reconfigured. # @@ -85,22 +60,6 @@ msg.help # CATEGORY: TARGET # ##################
-# Configuration option with which the target is synced. If an option is set via -# both the config and '--target' argument then the argument takes precedence. - -target.config ONLINE => integ.target.online -target.config RELATIVE => integ.target.relative_data_dir -target.config CHROOT => integ.target.chroot -target.config RUN_NONE => integ.target.run.none -target.config RUN_OPEN => integ.target.run.open -target.config RUN_PASSWORD => integ.target.run.password -target.config RUN_COOKIE => integ.target.run.cookie -target.config RUN_MULTIPLE => integ.target.run.multiple -target.config RUN_SOCKET => integ.target.run.socket -target.config RUN_SCOOKIE => integ.target.run.scookie -target.config RUN_PTRACE => integ.target.run.ptrace -target.config RUN_ALL => integ.target.run.all - # The '--help' description of the target.
target.description ONLINE => Includes tests that require network activity.