tor-commits April 2013

tor-commits@lists.torproject.org

19 participants
1496 discussions

[stem/master] Making check_requirements() require python 2.6
by atagar＠torproject.org 14 Apr '13

14 Apr '13

commit 00edd58211c9f67a0499aa6c04d5c4fa97580708 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Apr 13 10:00:37 2013 -0700 Making check_requirements() require python 2.6 Oops, the prereq module was checking for python 2.5. We're now only compatable with 2.6 and above. --- stem/prereq.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stem/prereq.py b/stem/prereq.py index 4c0ffcd..41ba4b9 100644 --- a/stem/prereq.py +++ b/stem/prereq.py @@ -39,8 +39,8 @@ def check_requirements(): major_version, minor_version = sys.version_info[0:2] - if major_version < 2 or (major_version == 2 and minor_version < 5): - raise ImportError("stem requires python version 2.5 or greater") + if major_version < 2 or (major_version == 2 and minor_version < 6): + raise ImportError("stem requires python version 2.6 or greater") def is_python_26():

1 0

[stem/master] Merging test.network and test.util
by atagar＠torproject.org 14 Apr '13

14 Apr '13

commit e28971e61ec41ec76be74d9b703cd728d7ec0d0f Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 12 08:40:16 2013 -0700 Merging test.network and test.util Both modules are to support tests for the ONLINE target. The main difference is that one module was made by Ravi and the other by Sean. At some point we should truely unify their functionality, but for now sticking them in the same place. --- test/integ/control/controller.py | 7 +-- test/network.py | 75 +++++++++++++++++++++++++++++++++++ test/util.py | 80 -------------------------------------- 3 files changed, 78 insertions(+), 84 deletions(-) diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index 3cf09d9..64cbaa4 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -19,7 +19,6 @@ import stem.socket import stem.version import test.network import test.runner -import test.util from stem import Flag, Signal from stem.control import EventType, State @@ -788,8 +787,8 @@ class TestController(unittest.TestCase): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(('127.0.0.1', int(controller.get_conf('SocksListenAddress').rsplit(':', 1)[1]))) - test.util.negotiate_socks(s, '1.2.1.2', 80) - s.sendall(test.util.ip_request) # make the http request for the ip address + test.network.negotiate_socks(s, '1.2.1.2', 80) + s.sendall(test.network.ip_request) # make the http request for the ip address response = s.recv(1000) # everything after the blank line is the 'data' in a HTTP response. @@ -974,7 +973,7 @@ class TestController(unittest.TestCase): circuit_id = controller.new_circuit(await_build = True) socksport = controller.get_socks_listeners()[0][1] - ip = test.util.external_ip('127.0.0.1', socksport) + ip = test.network.external_ip('127.0.0.1', socksport) exit_circuit = controller.get_circuit(circuit_id) self.assertTrue(exit_circuit) exit_ip = controller.get_network_status(exit_circuit.path[2][0]).address diff --git a/test/network.py b/test/network.py index 2d4d9ba..df9f018 100644 --- a/test/network.py +++ b/test/network.py @@ -21,6 +21,7 @@ import socket import struct import stem.util.connection +from stem import ProtocolError, SocketError # Store a reference to the original class so we can find it after # monkey patching. @@ -32,6 +33,20 @@ SOCKS5_CONN_BY_IPV4 = (0x05, 0x01, 0x00, 0x01) SOCKS5_CONN_BY_NAME = (0x05, 0x01, 0x00, 0x03) +error_msgs = { + 0x5a: "SOCKS4A request granted", + 0x5b: "SOCKS4A request rejected or failed", + 0x5c: "SOCKS4A request failed because client is not running identd (or not reachable from the server)", + 0x5d: "SOCKS4A request failed because client's identd could not confirm the user ID string in the request", +} + +ip_request = """GET /ip HTTP/1.0 +Host: ifconfig.me +Accept-Encoding: identity + +""" + + class ProxyError(Exception): "Base error for proxy issues." @@ -253,3 +268,63 @@ class SocksPatch(object): def __exit__(self, exit_type, value, traceback): socket.socket = _socket_socket + + +def external_ip(host, port): + """ + Returns the externally visible IP address when using a SOCKS4a proxy. + Negotiates the socks connection, connects to ipconfig.me and requests + http://ifconfig.me/ip to find out the externally visible IP. + + Supports only SOCKS4a proxies. + + :param str host: hostname/IP of the proxy server + :param int port: port on which the proxy server is listening + + :returns: externally visible IP address, or None if it isn't able to + + :raises: :class:`stem.socket.SocketError`: unable to connect a socket to the socks server + """ + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, int(port))) + except Exception, exc: + raise SocketError("Failed to connect to the socks server: " + str(exc)) + + try: + negotiate_socks(sock, "ifconfig.me", 80) + sock.sendall(ip_request) + response = sock.recv(1000) + + # everything after the blank line is the 'data' in a HTTP response + # The response data for our request for request should be an IP address + '\n' + return response[response.find("\r\n\r\n"):].strip() + except Exception, exc: + return None + + +def negotiate_socks(sock, host, port): + """ + Negotiate with a socks4a server. Closes the socket and raises an exception on + failure. + + :param socket sock: socket connected to socks4a server + :param str host: hostname/IP to connect to + :param int port: port to connect to + + :raises: :class:`stem.ProtocolError` if the socks server doesn't grant our request + + :returns: a list with the IP address and the port that the proxy connected to + """ + + # SOCKS4a request here - http://en.wikipedia.org/wiki/SOCKS#Protocol + request = "\x04\x01" + struct.pack("!H", port) + "\x00\x00\x00\x01" + "\x00" + host + "\x00" + sock.sendall(request) + response = sock.recv(8) + + if len(response) != 8 or response[0] != "\x00" or response[1] != "\x5a": + sock.close() + raise ProtocolError(error_msgs.get(response[1], "SOCKS server returned unrecognized error code")) + + return [socket.inet_ntoa(response[4:]), struct.unpack("!H", response[2:4])[0]] diff --git a/test/util.py b/test/util.py deleted file mode 100644 index 467ca91..0000000 --- a/test/util.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2012-2013, Damian Johnson -# See LICENSE for licensing information - -import socket -import struct - -from stem import ProtocolError, SocketError - -error_msgs = { - 0x5a: "SOCKS4A request granted", - 0x5b: "SOCKS4A request rejected or failed", - 0x5c: "SOCKS4A request failed because client is not running identd (or not reachable from the server)", - 0x5d: "SOCKS4A request failed because client's identd could not confirm the user ID string in the request", -} - -ip_request = """GET /ip HTTP/1.0 -Host: ifconfig.me -Accept-Encoding: identity - -""" - - -def external_ip(host, port): - """ - Returns the externally visible IP address when using a SOCKS4a proxy. - Negotiates the socks connection, connects to ipconfig.me and requests - http://ifconfig.me/ip to find out the externally visible IP. - - Supports only SOCKS4a proxies. - - :param str host: hostname/IP of the proxy server - :param int port: port on which the proxy server is listening - - :returns: externally visible IP address, or None if it isn't able to - - :raises: :class:`stem.socket.SocketError`: unable to connect a socket to the socks server - """ - - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, int(port))) - except Exception, exc: - raise SocketError("Failed to connect to the socks server: " + str(exc)) - - try: - negotiate_socks(sock, "ifconfig.me", 80) - sock.sendall(ip_request) - response = sock.recv(1000) - - # everything after the blank line is the 'data' in a HTTP response - # The response data for our request for request should be an IP address + '\n' - return response[response.find("\r\n\r\n"):].strip() - except Exception, exc: - return None - - -def negotiate_socks(sock, host, port): - """ - Negotiate with a socks4a server. Closes the socket and raises an exception on - failure. - - :param socket sock: socket connected to socks4a server - :param str host: hostname/IP to connect to - :param int port: port to connect to - - :raises: :class:`stem.ProtocolError` if the socks server doesn't grant our request - - :returns: a list with the IP address and the port that the proxy connected to - """ - - # SOCKS4a request here - http://en.wikipedia.org/wiki/SOCKS#Protocol - request = "\x04\x01" + struct.pack("!H", port) + "\x00\x00\x00\x01" + "\x00" + host + "\x00" - sock.sendall(request) - response = sock.recv(8) - - if len(response) != 8 or response[0] != "\x00" or response[1] != "\x5a": - sock.close() - raise ProtocolError(error_msgs.get(response[1], "SOCKS server returned unrecognized error code")) - - return [socket.inet_ntoa(response[4:]), struct.unpack("!H", response[2:4])[0]]

1 0

[stem/master] Supporting list of paths in static checker functions
by atagar＠torproject.org 14 Apr '13

14 Apr '13

commit f141a007655ae4d667942745b1acafa77391e0d0 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 12 08:31:49 2013 -0700 Supporting list of paths in static checker functions Reudcing a bit of redundancy by having the static_checks module support list of paths. --- run_tests.py | 18 +++++++----------- test/runner.py | 1 - test/static_checks.py | 42 +++++++++++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/run_tests.py b/run_tests.py index ab35592..65c3444 100755 --- a/run_tests.py +++ b/run_tests.py @@ -41,13 +41,16 @@ CONFIG = stem.util.conf.config_dict("test", { DEFAULT_RUN_TARGET = Target.RUN_OPEN +base = os.path.sep.join(__file__.split(os.path.sep)[:-1]).lstrip("./") +SOURCE_BASE_PATHS = [os.path.join(base, path) for path in ('stem', 'test', 'run_tests.py')] + def _clean_orphaned_pyc(): test.output.print_noline(" checking for orphaned .pyc files... ", *test.runner.STATUS_ATTR) orphaned_pyc = [] - for base_dir in ('stem', 'test', 'run_tests.py'): + for base_dir in SOURCE_BASE_PATHS: for pyc_path in test.static_checks._get_files_with_suffix(base_dir, ".pyc"): # If we're running python 3 then the *.pyc files are no longer bundled # with the *.py. Rather, they're in a __pycache__ directory. @@ -121,10 +124,7 @@ def _python3_setup(python3_destination, clean): def _print_style_issues(run_unit, run_integ, run_style): - base_path = os.path.sep.join(__file__.split(os.path.sep)[:-1]).lstrip("./") - style_issues = test.static_checks.get_issues(os.path.join(base_path, "stem")) - style_issues.update(test.static_checks.get_issues(os.path.join(base_path, "test"))) - style_issues.update(test.static_checks.get_issues(os.path.join(base_path, "run_tests.py"))) + style_issues = test.static_checks.get_issues(SOURCE_BASE_PATHS) # If we're doing some sort of testing (unit or integ) and pyflakes is # available then use it. Its static checks are pretty quick so there's not @@ -132,17 +132,13 @@ def _print_style_issues(run_unit, run_integ, run_style): if run_unit or run_integ: if system.is_available("pyflakes"): - style_issues.update(test.static_checks.pyflakes_issues(os.path.join(base_path, "stem"))) - style_issues.update(test.static_checks.pyflakes_issues(os.path.join(base_path, "test"))) - style_issues.update(test.static_checks.pyflakes_issues(os.path.join(base_path, "run_tests.py"))) + style_issues.update(test.static_checks.pyflakes_issues(SOURCE_BASE_PATHS)) else: test.output.print_error("Static error checking requires pyflakes. Please install it from ...\n http://pypi.python.org/pypi/pyflakes\n") if run_style: if system.is_available("pep8"): - style_issues.update(test.static_checks.pep8_issues(os.path.join(base_path, "stem"))) - style_issues.update(test.static_checks.pep8_issues(os.path.join(base_path, "test"))) - style_issues.update(test.static_checks.pep8_issues(os.path.join(base_path, "run_tests.py"))) + style_issues.update(test.static_checks.pep8_issues(SOURCE_BASE_PATHS)) else: test.output.print_error("Style checks require pep8. Please install it from...\n http://pypi.python.org/pypi/pep8\n") diff --git a/test/runner.py b/test/runner.py index 2787795..fb30c9f 100644 --- a/test/runner.py +++ b/test/runner.py @@ -227,7 +227,6 @@ def exercise_controller(test_case, controller): test_case.assertEquals("config-file=%s\nOK" % torrc_path, str(config_file_response)) - def get_unit_tests(prefix = None): """ Provides the classes for our unit tests. diff --git a/test/static_checks.py b/test/static_checks.py index b8cc664..6d0a508 100644 --- a/test/static_checks.py +++ b/test/static_checks.py @@ -27,16 +27,24 @@ CONFIG = conf.config_dict("test", { }) -def pep8_issues(base_path = DEFAULT_TARGET): +def pep8_issues(base_paths = DEFAULT_TARGET): """ Checks for stylistic issues that are an issue according to the parts of PEP8 we conform to. - :param str base_path: directory to be iterated over + :param str,list base_paths: directory to be iterated over :returns: dict of the form ``path => [(line_number, message)...]`` """ + if isinstance(base_paths, (tuple, list)): + results = {} + + for path in base_paths: + results.update(pep8_issues(path)) + + return results + # The pep8 command give output of the form... # # FILE:LINE:CHARACTER ISSUE @@ -76,7 +84,7 @@ def pep8_issues(base_path = DEFAULT_TARGET): ignored_issues = "E111,E121,E501,E251,E127" issues = {} - pep8_output = system.call("pep8 --ignore %s %s" % (ignored_issues, base_path)) + pep8_output = system.call("pep8 --ignore %s %s" % (ignored_issues, base_paths)) for line in pep8_output: line_match = re.match("^(.*):(\d+):(\d+): (.*)$", line) @@ -90,16 +98,24 @@ def pep8_issues(base_path = DEFAULT_TARGET): return issues -def pyflakes_issues(base_path = DEFAULT_TARGET): +def pyflakes_issues(base_paths = DEFAULT_TARGET): """ Checks for issues via pyflakes. False positives can be whitelisted via our test configuration. - :param str base_path: directory to be iterated over + :param str,list base_paths: directory to be iterated over :returns: dict of the form ``path => [(line_number, message)...]`` """ + if isinstance(base_paths, (tuple, list)): + results = {} + + for path in base_paths: + results.update(pyflakes_issues(path)) + + return results + global PYFLAKES_IGNORE if PYFLAKES_IGNORE is None: @@ -121,7 +137,7 @@ def pyflakes_issues(base_path = DEFAULT_TARGET): # stem/control.py:957: undefined name 'entry' issues = {} - pyflakes_output = system.call("pyflakes %s" % base_path) + pyflakes_output = system.call("pyflakes %s" % base_paths) for line in pyflakes_output: line_match = re.match("^(.*):(\d+): (.*)$", line) @@ -135,22 +151,30 @@ def pyflakes_issues(base_path = DEFAULT_TARGET): return issues -def get_issues(base_path = DEFAULT_TARGET): +def get_issues(base_paths = DEFAULT_TARGET): """ Checks python source code in the given directory for whitespace issues. - :param str base_path: directory to be iterated over + :param str,list base_paths: directory to be iterated over :returns: dict of the form ``path => [(line_number, message)...]`` """ + if isinstance(base_paths, (tuple, list)): + results = {} + + for path in base_paths: + results.update(get_issues(path)) + + return results + # TODO: This does not check that block indentations are two spaces because # differentiating source from string blocks ("""foo""") is more of a pita # than I want to deal with right now. issues = {} - for file_path in _get_files_with_suffix(base_path): + for file_path in _get_files_with_suffix(base_paths): if _is_test_data(file_path): continue

1 0

[stem/master] Dropping runability of static_checks.py
by atagar＠torproject.org 14 Apr '13

14 Apr '13

commit 02fb92e92892f58c18c4f2de1bcf831ee43f4ff6 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 12 08:33:34 2013 -0700 Dropping runability of static_checks.py Once upon a time static_checks.py had a single function that did a fair bit more validation. At that time it made sense for this to be runable as a standalone script. However, nowadays it's mostly just helpers for pyflakes and pep8. --- test/static_checks.py | 21 +++------------------ 1 files changed, 3 insertions(+), 18 deletions(-) diff --git a/test/static_checks.py b/test/static_checks.py index 6d0a508..307a858 100644 --- a/test/static_checks.py +++ b/test/static_checks.py @@ -15,9 +15,6 @@ import os from stem.util import conf, system -# if ran directly then run over everything one level up -DEFAULT_TARGET = os.path.sep.join(__file__.split(os.path.sep)[:-1]) - # mapping of files to the issues that should be ignored PYFLAKES_IGNORE = None @@ -27,7 +24,7 @@ CONFIG = conf.config_dict("test", { }) -def pep8_issues(base_paths = DEFAULT_TARGET): +def pep8_issues(base_paths): """ Checks for stylistic issues that are an issue according to the parts of PEP8 we conform to. @@ -98,7 +95,7 @@ def pep8_issues(base_paths = DEFAULT_TARGET): return issues -def pyflakes_issues(base_paths = DEFAULT_TARGET): +def pyflakes_issues(base_paths): """ Checks for issues via pyflakes. False positives can be whitelisted via our test configuration. @@ -151,7 +148,7 @@ def pyflakes_issues(base_paths = DEFAULT_TARGET): return issues -def get_issues(base_paths = DEFAULT_TARGET): +def get_issues(base_paths): """ Checks python source code in the given directory for whitespace issues. @@ -226,15 +223,3 @@ def _get_files_with_suffix(base_path, suffix = ".py"): for filename in files: if filename.endswith(suffix): yield os.path.join(root, filename) - -if __name__ == '__main__': - issues = get_issues() - - for file_path in issues: - print file_path - - for line_number, msg in issues[file_path]: - line_count = "%-4s" % line_number - print " line %s %s" % (line_count, msg) - - print

1 0

[stem/master] Dropping the argument.* config options
by atagar＠torproject.org 14 Apr '13

14 Apr '13

commit d46ecdfafd6883ac39bf24841af988d45d2fd64e Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Apr 11 09:28:18 2013 -0700 Dropping the argument.* config options Global state is icky, and in essence all stem.util.conf usage is global state. As mentioned in earlier commits the argument.* options were never used. Happily they were only consumed within run_tests.py so narrowing their scope is trivial. --- run_tests.py | 81 +++++++++++++++++++++++++++-------------------------- test/settings.cfg | 20 +------------ 2 files changed, 43 insertions(+), 58 deletions(-) diff --git a/run_tests.py b/run_tests.py index 45c3e00..ca43539 100755 --- a/run_tests.py +++ b/run_tests.py @@ -30,14 +30,6 @@ OPT_EXPANDED = ["all", "unit", "integ", "style", "python3", "clean", "targets=", DIVIDER = "=" * 70 CONFIG = stem.util.conf.config_dict("test", { - "argument.unit": False, - "argument.integ": False, - "argument.style": False, - "argument.python3": False, - "argument.python3_clean": False, - "argument.test": "", - "argument.log": None, - "argument.tor": "tor", "msg.help": "", "target.config": {}, "target.description": {}, @@ -96,7 +88,7 @@ def _clean_orphaned_pyc(): os.remove(pyc_file) -def _python3_setup(python3_destination): +def _python3_setup(python3_destination, clean): # Python 2.7.3 added some nice capabilities to 2to3, like '--output-dir'... # # http://docs.python.org/2/library/2to3.html @@ -106,7 +98,7 @@ def _python3_setup(python3_destination): test.output.print_divider("EXPORTING TO PYTHON 3", True) - if CONFIG["argument.python3_clean"]: + if clean: shutil.rmtree(python3_destination, ignore_errors = True) if os.path.exists(python3_destination): @@ -144,7 +136,7 @@ def _python3_setup(python3_destination): return True -def _print_style_issues(): +def _print_style_issues(run_unit, run_integ, run_style): base_path = os.path.sep.join(__file__.split(os.path.sep)[:-1]).lstrip("./") style_issues = test.static_checks.get_issues(os.path.join(base_path, "stem")) style_issues.update(test.static_checks.get_issues(os.path.join(base_path, "test"))) @@ -154,7 +146,7 @@ def _print_style_issues(): # available then use it. Its static checks are pretty quick so there's not # much overhead in including it with all tests. - if CONFIG["argument.unit"] or CONFIG["argument.integ"]: + if run_unit or run_integ: if system.is_available("pyflakes"): style_issues.update(test.static_checks.pyflakes_issues(os.path.join(base_path, "stem"))) style_issues.update(test.static_checks.pyflakes_issues(os.path.join(base_path, "test"))) @@ -162,7 +154,7 @@ def _print_style_issues(): else: test.output.print_line("Static error checking requires pyflakes. Please install it from ...\n http://pypi.python.org/pypi/pyflakes\n", *ERROR_ATTR) - if CONFIG["argument.style"]: + if run_style: if system.is_available("pep8"): style_issues.update(test.static_checks.pep8_issues(os.path.join(base_path, "stem"))) style_issues.update(test.static_checks.pep8_issues(os.path.join(base_path, "test"))) @@ -212,21 +204,31 @@ if __name__ == '__main__': print "%s (for usage provide --help)" % exc sys.exit(1) + run_unit = False + run_integ = False + run_style = False + run_python3 = False + run_python3_clean = False + + test_prefix = None + logging_runlevel = None + tor_path = "tor" + for opt, arg in opts: if opt in ("-a", "--all"): - test_config.set("argument.unit", "true") - test_config.set("argument.integ", "true") - test_config.set("argument.style", "true") + run_unit = True + run_integ = True + run_style = True elif opt in ("-u", "--unit"): - test_config.set("argument.unit", "true") + run_unit = True elif opt in ("-i", "--integ"): - test_config.set("argument.integ", "true") + run_integ = True elif opt in ("-s", "--style"): - test_config.set("argument.style", "true") + run_style = True elif opt == "--python3": - test_config.set("argument.python3", "true") + run_python3 = True elif opt == "--clean": - test_config.set("argument.python3_clean", "true") + run_python3_clean = True elif opt in ("-t", "--targets"): integ_targets = arg.split(",") @@ -245,11 +247,11 @@ if __name__ == '__main__': if target_config: test_config.set(target_config, "true") elif opt in ("-l", "--test"): - test_config.set("argument.test", arg) + test_prefix = arg elif opt in ("-l", "--log"): - test_config.set("argument.log", arg.upper()) + logging_runlevel = arg.upper() elif opt in ("--tor"): - test_config.set("argument.tor", arg) + tor_path = arg elif opt in ("-h", "--help"): # Prints usage information and quits. This includes a listing of the # valid integration targets. @@ -269,35 +271,34 @@ if __name__ == '__main__': # basic validation on user input - log_config = CONFIG["argument.log"] - if log_config and not log_config in log.LOG_VALUES: - print "'%s' isn't a logging runlevel, use one of the following instead:" % log_config + if logging_runlevel and not logging_runlevel in log.LOG_VALUES: + print "'%s' isn't a logging runlevel, use one of the following instead:" % logging_runlevel print " TRACE, DEBUG, INFO, NOTICE, WARN, ERROR" sys.exit(1) # check that we have 2to3 and python3 available in our PATH - if CONFIG["argument.python3"]: + if run_python3: for required_cmd in ("2to3", "python3"): if not system.is_available(required_cmd): test.output.print_line("Unable to test python 3 because %s isn't in your path" % required_cmd, *test.runner.ERROR_ATTR) sys.exit(1) - if CONFIG["argument.python3"] and sys.version_info[0] != 3: + if run_python3 and sys.version_info[0] != 3: python3_destination = os.path.join(CONFIG["integ.test_directory"], "python3") - if _python3_setup(python3_destination): + if _python3_setup(python3_destination, run_python3_clean): python3_runner = os.path.join(python3_destination, "run_tests.py") exit_status = os.system("python3 %s %s" % (python3_runner, " ".join(sys.argv[1:]))) sys.exit(exit_status) else: sys.exit(1) # failed to do python3 setup - if not CONFIG["argument.unit"] and not CONFIG["argument.integ"] and not CONFIG["argument.style"]: + if not run_unit and not run_integ and not run_style: test.output.print_line("Nothing to run (for usage provide --help)\n") sys.exit() # if we have verbose logging then provide the testing config - our_level = stem.util.log.logging_level(CONFIG["argument.log"]) + our_level = stem.util.log.logging_level(logging_runlevel) info_level = stem.util.log.logging_level(stem.util.log.INFO) if our_level <= info_level: @@ -312,7 +313,7 @@ if __name__ == '__main__': ) stem_logger = log.get_logger() - logging_buffer = log.LogBuffer(CONFIG["argument.log"]) + logging_buffer = log.LogBuffer(logging_runlevel) stem_logger.addHandler(logging_buffer) test.output.print_divider("INITIALISING", True) @@ -322,11 +323,11 @@ if __name__ == '__main__': print - if CONFIG["argument.unit"]: + if run_unit: test.output.print_divider("UNIT TESTS", True) error_tracker.set_category("UNIT TEST") - for test_class in test.runner.get_unit_tests(CONFIG["argument.test"]): + for test_class in test.runner.get_unit_tests(test_prefix): test.output.print_divider(test_class.__module__) suite = unittest.TestLoader().loadTestsFromTestCase(test_class) test_results = StringIO.StringIO() @@ -341,7 +342,7 @@ if __name__ == '__main__': print - if CONFIG["argument.integ"]: + if run_integ: test.output.print_divider("INTEGRATION TESTS", True) integ_runner = test.runner.get_runner() @@ -375,7 +376,7 @@ if __name__ == '__main__': if target_prereq: # lazy loaded to skip system call if we don't have any prereqs if not our_version: - our_version = stem.version.get_system_tor_version(CONFIG["argument.tor"]) + our_version = stem.version.get_system_tor_version(tor_path) if our_version < stem.version.Requirement[target_prereq]: skip_targets.append(target) @@ -401,12 +402,12 @@ if __name__ == '__main__': test.output.print_line("'%s' isn't a test.runner.Torrc enumeration" % opt) sys.exit(1) - integ_runner.start(CONFIG["argument.tor"], extra_torrc_opts = torrc_opts) + integ_runner.start(tor_path, extra_torrc_opts = torrc_opts) test.output.print_line("Running tests...", term.Color.BLUE, term.Attr.BOLD) print - for test_class in test.runner.get_integ_tests(CONFIG["argument.test"]): + for test_class in test.runner.get_integ_tests(test_prefix): test.output.print_divider(test_class.__module__) suite = unittest.TestLoader().loadTestsFromTestCase(test_class) test_results = StringIO.StringIO() @@ -452,7 +453,7 @@ if __name__ == '__main__': # TODO: note unused config options afterward? if not stem.prereq.is_python_3(): - _print_style_issues() + _print_style_issues(run_unit, run_integ, run_style) runtime = time.time() - start_time diff --git a/test/settings.cfg b/test/settings.cfg index 372ce33..81bb567 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -1,16 +1,7 @@ # Testing Configuration # -# The following are configurations used by stem's testing framework. You can -# overwrite any of these my making your own configuratino file and using the -# '--config' option of run_tests.py... -# -# run_tests.py --config my_testrc --unit -# -# The commonly customized options are... -# -# argument.* -# Default test runner arguments. You can set these to always run with -# particular arguments. +# The following contains customizable configuration information for stem's +# testing framework. # # integ.test_directory # Path used for our data directory and any temporary test resources. Relative @@ -36,13 +27,6 @@ # authentication configurations. If the 'all' option is set then the other # flags are ignored. -argument.unit false -argument.integ false -argument.style false -argument.test -argument.log -argument.tor tor - integ.test_directory ./test/data integ.log ./test/data/log

1 0

[stem/master] Untangling integration targets from configuration
by atagar＠torproject.org 14 Apr '13

14 Apr '13

commit 10d0bd80985c6dbdf8423890d0cd5f7f346ae559 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 12 07:56:32 2013 -0700 Untangling integration targets from configuration Even I was starting to get confused by 'em. We've always implicitely had two kinds of targets: run targets which constitute an integraion test run and attribute targets which have a non-torrc change on all test runs (such as being in a chroot). Making this division more explicit and dropping the confusing configuration dance we were doing. --- run_tests.py | 87 +++++++++++++++++++++------------------------------- test/runner.py | 38 +++++++++++++++++----- test/settings.cfg | 41 ------------------------- 3 files changed, 64 insertions(+), 102 deletions(-) diff --git a/run_tests.py b/run_tests.py index ca43539..66e7fc2 100755 --- a/run_tests.py +++ b/run_tests.py @@ -25,34 +25,20 @@ import test.output import test.runner import test.static_checks +from test.runner import Target + OPT = "auist:l:h" OPT_EXPANDED = ["all", "unit", "integ", "style", "python3", "clean", "targets=", "test=", "log=", "tor=", "help"] DIVIDER = "=" * 70 CONFIG = stem.util.conf.config_dict("test", { "msg.help": "", - "target.config": {}, "target.description": {}, "target.prereq": {}, "target.torrc": {}, "integ.test_directory": "./test/data", }) -Target = stem.util.enum.UppercaseEnum( - "ONLINE", - "RELATIVE", - "CHROOT", - "RUN_NONE", - "RUN_OPEN", - "RUN_PASSWORD", - "RUN_COOKIE", - "RUN_MULTIPLE", - "RUN_SOCKET", - "RUN_SCOOKIE", - "RUN_PTRACE", - "RUN_ALL", -) - DEFAULT_RUN_TARGET = Target.RUN_OPEN ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) @@ -214,6 +200,18 @@ if __name__ == '__main__': logging_runlevel = None tor_path = "tor" + # Integration testing targets fall into two categories: + # + # * Run Targets (like RUN_COOKIE and RUN_PTRACE) which customize our torrc. + # We do an integration test run for each run target we get. + # + # * Attribute Target (like CHROOT and ONLINE) which indicates + # non-configuration changes to ur test runs. These are applied to all + # integration runs that we perform. + + run_targets = [DEFAULT_RUN_TARGET] + attribute_targets = [] + for opt, arg in opts: if opt in ("-a", "--all"): run_unit = True @@ -232,7 +230,11 @@ if __name__ == '__main__': elif opt in ("-t", "--targets"): integ_targets = arg.split(",") - # validates the targets + run_targets = [] + all_run_targets = [t for t in Target if CONFIG["target.torrc"].get(t) is not None] + + # validates the targets and split them into run and attribute targets + if not integ_targets: print "No targets provided" sys.exit(1) @@ -241,11 +243,16 @@ if __name__ == '__main__': if not target in Target: print "Invalid integration target: %s" % target sys.exit(1) + elif target in all_run_targets: + run_targets.append(target) else: - target_config = test_config.get("target.config", {}).get(target) + attribute_targets.append(target) + + # check if we were told to use all run targets - if target_config: - test_config.set(target_config, "true") + if Target.RUN_ALL in attribute_targets: + attribute_targets.remove(Target.RUN_ALL) + run_targets = all_run_targets elif opt in ("-l", "--test"): test_prefix = arg elif opt in ("-l", "--log"): @@ -346,43 +353,19 @@ if __name__ == '__main__': test.output.print_divider("INTEGRATION TESTS", True) integ_runner = test.runner.get_runner() - # Queue up all the targets with torrc options we want to run against. - - integ_run_targets = [] - all_run_targets = [t for t in Target if CONFIG["target.torrc"].get(t) is not None] - - if test_config.get("integ.target.run.all", False): - # test against everything with torrc options - integ_run_targets = all_run_targets - else: - for target in all_run_targets: - target_config = CONFIG["target.config"].get(target) - - if target_config and test_config.get(target_config, False): - integ_run_targets.append(target) - - # if we didn't specify any targets then use the default - if not integ_run_targets: - integ_run_targets.append(DEFAULT_RUN_TARGET) - # Determine targets we don't meet the prereqs for. Warnings are given about # these at the end of the test run so they're more noticeable. - our_version, skip_targets = None, [] - - for target in integ_run_targets: - target_prereq = CONFIG["target.prereq"].get(target) + our_version = stem.version.get_system_tor_version(tor_path) + skip_targets = [] - if target_prereq: - # lazy loaded to skip system call if we don't have any prereqs - if not our_version: - our_version = stem.version.get_system_tor_version(tor_path) + for target in run_targets: + # check if we meet this target's tor version prerequisites - if our_version < stem.version.Requirement[target_prereq]: - skip_targets.append(target) + target_prereq = CONFIG["target.prereq"].get(target) - for target in integ_run_targets: - if target in skip_targets: + if target_prereq and our_version < stem.version.Requirement[target_prereq]: + skip_targets.append(target) continue error_tracker.set_category(target) @@ -402,7 +385,7 @@ if __name__ == '__main__': test.output.print_line("'%s' isn't a test.runner.Torrc enumeration" % opt) sys.exit(1) - integ_runner.start(tor_path, extra_torrc_opts = torrc_opts) + integ_runner.start(target, attribute_targets, tor_path, extra_torrc_opts = torrc_opts) test.output.print_line("Running tests...", term.Color.BLUE, term.Attr.BOLD) print diff --git a/test/runner.py b/test/runner.py index 9e809fe..d869d6a 100644 --- a/test/runner.py +++ b/test/runner.py @@ -65,13 +65,25 @@ from stem.util import term CONFIG = stem.util.conf.config_dict("test", { "integ.test_directory": "./test/data", "integ.log": "./test/data/log", - "integ.target.online": False, - "integ.target.relative_data_dir": False, - "integ.target.chroot": False, "test.unit_tests": "", "test.integ_tests": "", }) +Target = stem.util.enum.UppercaseEnum( + "ONLINE", + "RELATIVE", + "CHROOT", + "RUN_NONE", + "RUN_OPEN", + "RUN_PASSWORD", + "RUN_COOKIE", + "RUN_MULTIPLE", + "RUN_SOCKET", + "RUN_SCOOKIE", + "RUN_PTRACE", + "RUN_ALL", +) + STATUS_ATTR = (term.Color.BLUE, term.Attr.BOLD) SUBSTATUS_ATTR = (term.Color.BLUE, ) ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) @@ -170,7 +182,7 @@ def require_online(test_case): :returns: True if test should be skipped, False otherwise """ - if not CONFIG["integ.target.online"]: + if not Target.ONLINE in test.runner.get_runner().attribute_targets: skip(test_case, "(requires online target)") return True @@ -305,6 +317,9 @@ class _MockChrootFile(object): class Runner(object): def __init__(self): + self.run_target = None + self.attribute_targets = [] + self._runner_lock = threading.RLock() # runtime attributes, set by the start method @@ -320,11 +335,13 @@ class Runner(object): self._original_recv_message = None - def start(self, tor_cmd, extra_torrc_opts): + def start(self, run_target, attribute_targets, tor_cmd, extra_torrc_opts): """ Makes temporary testing resources and starts tor, blocking until it completes. + :param Target run_target: configuration we're running with + :param list attribute_targets: **Targets** for our non-configuration attributes :param str tor_cmd: command to start tor with :param list extra_torrc_opts: additional torrc options for our test instance @@ -332,6 +349,9 @@ class Runner(object): """ with self._runner_lock: + self.run_target = run_target + self.attribute_targets = attribute_targets + # if we're holding on to a tor process (running or not) then clean up after # it so we can start a fresh instance @@ -352,7 +372,7 @@ class Runner(object): original_cwd, data_dir_path = os.getcwd(), self._test_dir - if CONFIG["integ.target.relative_data_dir"]: + if Target.RELATIVE in self.attribute_targets: tor_cwd = os.path.dirname(self._test_dir) if not os.path.exists(tor_cwd): @@ -376,7 +396,7 @@ class Runner(object): # strip the testing directory from recv_message responses if we're # simulating a chroot setup - if CONFIG["integ.target.chroot"] and not self._original_recv_message: + if Target.CHROOT in self.attribute_targets and not self._original_recv_message: # TODO: when we have a function for telling stem the chroot we'll # need to set that too @@ -389,7 +409,7 @@ class Runner(object): stem.socket.recv_message = _chroot_recv_message # revert our cwd back to normal - if CONFIG["integ.target.relative_data_dir"]: + if Target.RELATIVE in self.attribute_targets: os.chdir(original_cwd) except OSError, exc: raise exc @@ -768,7 +788,7 @@ class Runner(object): try: # wait to fully complete if we're running tests with network activity, # otherwise finish after local bootstraping - complete_percent = 100 if CONFIG["integ.target.online"] else 5 + complete_percent = 100 if Target.ONLINE in self.attribute_targets else 5 # prints output from tor's stdout while it starts up print_init_line = lambda line: test.output.print_line(" %s" % line, *SUBSTATUS_ATTR) diff --git a/test/settings.cfg b/test/settings.cfg index 81bb567..625a482 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -14,35 +14,10 @@ # integ.log # Path runtime logs are placed. Relative paths are expanded in reference to # 'run_tests.py'. Logging is disabled if set ot an empty value. -# -# integ.target.online -# Runs tests with network activity. If set then we'll wait for tor to fully -# bootstrap when starting, which won't happen without a network connection. -# -# integ.target.relative_data_dir -# Uses a relative path for the tor data directory if set. -# -# integ.target.run.* -# Runs the integration test suite for all of the given connection and -# authentication configurations. If the 'all' option is set then the other -# flags are ignored. integ.test_directory ./test/data integ.log ./test/data/log -integ.target.online false -integ.target.relative_data_dir false -integ.target.chroot false -integ.target.run.none false -integ.target.run.open false -integ.target.run.password false -integ.target.run.cookie false -integ.target.run.muiltipe false -integ.target.run.socket false -integ.target.run.scookie false -integ.target.run.ptrace false -integ.target.run.all false - # The following are less testing framework attributes that aren't as commonly # reconfigured. # @@ -85,22 +60,6 @@ msg.help # CATEGORY: TARGET # ################## -# Configuration option with which the target is synced. If an option is set via -# both the config and '--target' argument then the argument takes precedence. - -target.config ONLINE => integ.target.online -target.config RELATIVE => integ.target.relative_data_dir -target.config CHROOT => integ.target.chroot -target.config RUN_NONE => integ.target.run.none -target.config RUN_OPEN => integ.target.run.open -target.config RUN_PASSWORD => integ.target.run.password -target.config RUN_COOKIE => integ.target.run.cookie -target.config RUN_MULTIPLE => integ.target.run.multiple -target.config RUN_SOCKET => integ.target.run.socket -target.config RUN_SCOOKIE => integ.target.run.scookie -target.config RUN_PTRACE => integ.target.run.ptrace -target.config RUN_ALL => integ.target.run.all - # The '--help' description of the target. target.description ONLINE => Includes tests that require network activity.

1 0

[stem/master] Moving ERROR_ATTR to output module
by atagar＠torproject.org 14 Apr '13

14 Apr '13

commit 90cb6e347fc02dd60e19011a13e9947e0eed7381 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Apr 12 08:10:56 2013 -0700 Moving ERROR_ATTR to output module The ERROR_ATTR was defined in both run_tests.py and the runner. Making both modules a little cleaner by moving it to the output module. --- run_tests.py | 26 ++++++++++++-------------- test/output.py | 5 +++++ test/runner.py | 13 ++++++------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/run_tests.py b/run_tests.py index 66e7fc2..ab35592 100755 --- a/run_tests.py +++ b/run_tests.py @@ -41,8 +41,6 @@ CONFIG = stem.util.conf.config_dict("test", { DEFAULT_RUN_TARGET = Target.RUN_OPEN -ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) - def _clean_orphaned_pyc(): test.output.print_noline(" checking for orphaned .pyc files... ", *test.runner.STATUS_ATTR) @@ -70,7 +68,7 @@ def _clean_orphaned_pyc(): else: print for pyc_file in orphaned_pyc: - test.output.print_line(" removing %s" % pyc_file, *test.runner.ERROR_ATTR) + test.output.print_error(" removing %s" % pyc_file) os.remove(pyc_file) @@ -88,7 +86,7 @@ def _python3_setup(python3_destination, clean): shutil.rmtree(python3_destination, ignore_errors = True) if os.path.exists(python3_destination): - test.output.print_line("Reusing '%s'. Run again with '--clean' if you want to recreate the python3 export." % python3_destination, *test.runner.ERROR_ATTR) + test.output.print_error("Reusing '%s'. Run again with '--clean' if you want to recreate the python3 export." % python3_destination) print return True @@ -108,7 +106,7 @@ def _python3_setup(python3_destination, clean): shutil.copy('run_tests.py', os.path.join(python3_destination, 'run_tests.py')) test.output.print_line("done", *test.runner.STATUS_ATTR) except OSError, exc: - test.output.print_line("failed\n%s" % exc, *test.runner.ERROR_ATTR) + test.output.print_error("failed\n%s" % exc) return False try: @@ -116,7 +114,7 @@ def _python3_setup(python3_destination, clean): system.call("2to3 --write --nobackups --no-diffs %s" % python3_destination) test.output.print_line("done", *test.runner.STATUS_ATTR) except OSError, exc: - test.output.print_line("failed\n%s" % exc, *test.runner.ERROR_ATTR) + test.output.print_error("failed\n%s" % exc) return False return True @@ -138,7 +136,7 @@ def _print_style_issues(run_unit, run_integ, run_style): style_issues.update(test.static_checks.pyflakes_issues(os.path.join(base_path, "test"))) style_issues.update(test.static_checks.pyflakes_issues(os.path.join(base_path, "run_tests.py"))) else: - test.output.print_line("Static error checking requires pyflakes. Please install it from ...\n http://pypi.python.org/pypi/pyflakes\n", *ERROR_ATTR) + test.output.print_error("Static error checking requires pyflakes. Please install it from ...\n http://pypi.python.org/pypi/pyflakes\n") if run_style: if system.is_available("pep8"): @@ -146,7 +144,7 @@ def _print_style_issues(run_unit, run_integ, run_style): style_issues.update(test.static_checks.pep8_issues(os.path.join(base_path, "test"))) style_issues.update(test.static_checks.pep8_issues(os.path.join(base_path, "run_tests.py"))) else: - test.output.print_line("Style checks require pep8. Please install it from...\n http://pypi.python.org/pypi/pep8\n", *ERROR_ATTR) + test.output.print_error("Style checks require pep8. Please install it from...\n http://pypi.python.org/pypi/pep8\n") if style_issues: test.output.print_line("STYLE ISSUES", term.Color.BLUE, term.Attr.BOLD) @@ -287,7 +285,7 @@ if __name__ == '__main__': if run_python3: for required_cmd in ("2to3", "python3"): if not system.is_available(required_cmd): - test.output.print_line("Unable to test python 3 because %s isn't in your path" % required_cmd, *test.runner.ERROR_ATTR) + test.output.print_error("Unable to test python 3 because %s isn't in your path" % required_cmd) sys.exit(1) if run_python3 and sys.version_info[0] != 3: @@ -409,15 +407,15 @@ if __name__ == '__main__': active_threads = threading.enumerate() if len(active_threads) > 1: - test.output.print_line("Threads lingering after test run:", *ERROR_ATTR) + test.output.print_error("Threads lingering after test run:") for lingering_thread in active_threads: - test.output.print_line(" %s" % lingering_thread, *ERROR_ATTR) + test.output.print_error(" %s" % lingering_thread) testing_failed = True break except KeyboardInterrupt: - test.output.print_line(" aborted starting tor: keyboard interrupt\n", *ERROR_ATTR) + test.output.print_error(" aborted starting tor: keyboard interrupt\n") break except OSError: testing_failed = True @@ -448,10 +446,10 @@ if __name__ == '__main__': has_error = testing_failed or error_tracker.has_error_occured() if has_error: - test.output.print_line("TESTING FAILED %s" % runtime_label, *ERROR_ATTR) + test.output.print_error("TESTING FAILED %s" % runtime_label) for line in error_tracker: - test.output.print_line(" %s" % line, *ERROR_ATTR) + test.output.print_error(" %s" % line) elif skipped_test_count > 0: test.output.print_line("%i TESTS WERE SKIPPED" % skipped_test_count, term.Color.BLUE, term.Attr.BOLD) test.output.print_line("ALL OTHER TESTS PASSED %s" % runtime_label, term.Color.GREEN, term.Attr.BOLD) diff --git a/test/output.py b/test/output.py index 66e19c6..22f3605 100644 --- a/test/output.py +++ b/test/output.py @@ -18,6 +18,7 @@ COLOR_SUPPORT = sys.stdout.isatty() and not system.is_windows() DIVIDER = "=" * 70 HEADER_ATTR = (term.Color.CYAN, term.Attr.BOLD) CATEGORY_ATTR = (term.Color.GREEN, term.Attr.BOLD) +ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) LineType = stem.util.enum.Enum("OK", "FAIL", "ERROR", "SKIPPED", "CONTENT") @@ -52,6 +53,10 @@ def print_noline(msg, *attr): sys.stdout.flush() +def print_error(msg): + print_line(msg, *ERROR_ATTR) + + def print_divider(msg, is_header = False): attr = HEADER_ATTR if is_header else CATEGORY_ATTR print_line("%s\n%s\n%s\n" % (DIVIDER, msg.center(70), DIVIDER), *attr) diff --git a/test/runner.py b/test/runner.py index d869d6a..2787795 100644 --- a/test/runner.py +++ b/test/runner.py @@ -86,7 +86,6 @@ Target = stem.util.enum.UppercaseEnum( STATUS_ATTR = (term.Color.BLUE, term.Attr.BOLD) SUBSTATUS_ATTR = (term.Color.BLUE, ) -ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) SOCKS_HOST = "127.0.0.1" SOCKS_PORT = 1112 @@ -432,7 +431,7 @@ class Runner(object): elif not stem.util.system.is_windows(): os.kill(self._tor_process.pid, signal.SIGTERM) else: - test.output.print_line("failed (unable to call kill() in python 2.5)", *ERROR_ATTR) + test.output.print_error("failed (unable to call kill() in python 2.5)") except OSError: pass @@ -470,7 +469,7 @@ class Runner(object): if self._tor_process and self._tor_process.poll() is not None: # clean up the temporary resources and note the unexpected shutdown self.stop() - test.output.print_line("tor shut down unexpectedly", *ERROR_ATTR) + test.output.print_error("tor shut down unexpectedly") return bool(self._tor_process) @@ -707,7 +706,7 @@ class Runner(object): os.makedirs(self._test_dir) test.output.print_line("done", *STATUS_ATTR) except OSError, exc: - test.output.print_line("failed (%s)" % exc, *ERROR_ATTR) + test.output.print_error("failed (%s)" % exc) raise exc # Tor checks during startup that the directory a control socket resides in @@ -728,7 +727,7 @@ class Runner(object): os.chmod(socket_dir, 0700) test.output.print_line("done", *STATUS_ATTR) except OSError, exc: - test.output.print_line("failed (%s)" % exc, *ERROR_ATTR) + test.output.print_error("failed (%s)" % exc) raise exc # configures logging @@ -769,7 +768,7 @@ class Runner(object): print except Exception, exc: - test.output.print_line("failed (%s)\n" % exc, *ERROR_ATTR) + test.output.print_error("failed (%s)\n" % exc) raise OSError(exc) def _start_tor(self, tor_cmd): @@ -799,5 +798,5 @@ class Runner(object): runtime = time.time() - start_time test.output.print_line(" done (%i seconds)\n" % runtime, *STATUS_ATTR) except OSError, exc: - test.output.print_line(" failed to start tor: %s\n" % exc, *ERROR_ATTR) + test.output.print_error(" failed to start tor: %s\n" % exc) raise exc

1 0

[stem/master] Moving test imports to the runner module
by atagar＠torproject.org 14 Apr '13

14 Apr '13

commit 76e751f9c8fd338c73752a8547cedc92a95363de Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Apr 10 10:16:35 2013 -0700 Moving test imports to the runner module Ideally the run_tests.py module should be as simple as possible, simply parsing the user input and calling helper functions to run the tests. To this end I'm moving the code for enumerating unit and integration tests into the runner module. --- run_tests.py | 33 ++------------------------------- test/runner.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/run_tests.py b/run_tests.py index 17de0ee..096de69 100755 --- a/run_tests.py +++ b/run_tests.py @@ -45,8 +45,6 @@ CONFIG = stem.util.conf.config_dict("test", { "target.prereq": {}, "target.torrc": {}, "integ.test_directory": "./test/data", - "test.unit_tests": "", - "test.integ_tests": "", }) Target = stem.util.enum.UppercaseEnum( @@ -284,23 +282,6 @@ def _print_style_issues(): print -def _import_test(import_name): - # Dynamically imports test modules. The __import__() call has a couple quirks - # that make this a little clunky... - # - # * it only accepts modules, not the actual class we want to import - # - # * it returns the top level module, so we need to transverse into it for - # the test class - - module_name = '.'.join(import_name.split('.')[:-1]) - module = __import__(module_name) - - for subcomponent in import_name.split(".")[1:]: - module = getattr(module, subcomponent) - - return module - if __name__ == '__main__': try: stem.prereq.check_requirements() @@ -377,12 +358,7 @@ if __name__ == '__main__': test.output.print_divider("UNIT TESTS", True) error_tracker.set_category("UNIT TEST") - for test_module in CONFIG["test.unit_tests"].splitlines(): - if not test_module: - continue - - test_class = _import_test(test_module) - + for test_class in test.runner.get_unit_tests(): if CONFIG["argument.test"] and \ not test_class.__module__.startswith(CONFIG["argument.test"]): continue @@ -466,12 +442,7 @@ if __name__ == '__main__': test.output.print_line("Running tests...", term.Color.BLUE, term.Attr.BOLD) print - for test_module in CONFIG["test.integ_tests"].splitlines(): - if not test_module: - continue - - test_class = _import_test(test_module) - + for test_class in test.runner.get_integ_tests(): if CONFIG["argument.test"] and \ not test_class.__module__.startswith(CONFIG["argument.test"]): continue diff --git a/test/runner.py b/test/runner.py index 5010224..592d367 100644 --- a/test/runner.py +++ b/test/runner.py @@ -18,6 +18,9 @@ about the tor test instance they're running against. require_online - skips unless targets allow for online tests exercise_controller - basic sanity check that a controller connection can be used + get_unit_tests - provides our unit tests + get_integ_tests - provides our integration tests + get_runner - Singleton for fetching our runtime context. Runner - Runtime context for our integration tests. |- start - prepares and starts a tor instance for our tests to run against @@ -65,6 +68,8 @@ CONFIG = stem.util.conf.config_dict("test", { "integ.target.online": False, "integ.target.relative_data_dir": False, "integ.target.chroot": False, + "test.unit_tests": "", + "test.integ_tests": "", }) STATUS_ATTR = (term.Color.BLUE, term.Attr.BOLD) @@ -211,6 +216,49 @@ def exercise_controller(test_case, controller): test_case.assertEquals("config-file=%s\nOK" % torrc_path, str(config_file_response)) + +def get_unit_tests(): + """ + Provides the classes for our unit tests. + + :returns: an **iterator** for our unit tests + """ + + for test_module in CONFIG["test.unit_tests"].splitlines(): + if test_module: + yield _import_test(test_module) + + +def get_integ_tests(): + """ + Provides the classes for our integration tests. + + :returns: an **iterator** for our integration tests + """ + + for test_module in CONFIG["test.unit_tests"].splitlines(): + if test_module: + yield _import_test(test_module) + + +def _import_test(import_name): + # Dynamically imports test modules. The __import__() call has a couple quirks + # that make this a little clunky... + # + # * it only accepts modules, not the actual class we want to import + # + # * it returns the top level module, so we need to transverse into it for + # the test class + + module_name = '.'.join(import_name.split('.')[:-1]) + module = __import__(module_name) + + for subcomponent in import_name.split(".")[1:]: + module = getattr(module, subcomponent) + + return module + + def get_runner(): """ Singleton for the runtime context of integration tests.

1 0

[stem/master] Moving test filter to helper function
by atagar＠torproject.org 14 Apr '13

14 Apr '13

commit 347d2859073672e9c0354727901769245fb3b758 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Apr 11 08:35:31 2013 -0700 Moving test filter to helper function Shifting support for the '--test' argument to the helper. Oh, and the integ helper was running the unit tests instead. ;) --- run_tests.py | 12 ++---------- test/runner.py | 30 ++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/run_tests.py b/run_tests.py index 096de69..3de1fbf 100755 --- a/run_tests.py +++ b/run_tests.py @@ -358,11 +358,7 @@ if __name__ == '__main__': test.output.print_divider("UNIT TESTS", True) error_tracker.set_category("UNIT TEST") - for test_class in test.runner.get_unit_tests(): - if CONFIG["argument.test"] and \ - not test_class.__module__.startswith(CONFIG["argument.test"]): - continue - + for test_class in test.runner.get_unit_tests(CONFIG["argument.test"]): test.output.print_divider(test_class.__module__) suite = unittest.TestLoader().loadTestsFromTestCase(test_class) test_results = StringIO.StringIO() @@ -442,11 +438,7 @@ if __name__ == '__main__': test.output.print_line("Running tests...", term.Color.BLUE, term.Attr.BOLD) print - for test_class in test.runner.get_integ_tests(): - if CONFIG["argument.test"] and \ - not test_class.__module__.startswith(CONFIG["argument.test"]): - continue - + for test_class in test.runner.get_integ_tests(CONFIG["argument.test"]): test.output.print_divider(test_class.__module__) suite = unittest.TestLoader().loadTestsFromTestCase(test_class) test_results = StringIO.StringIO() diff --git a/test/runner.py b/test/runner.py index 592d367..9e809fe 100644 --- a/test/runner.py +++ b/test/runner.py @@ -217,28 +217,42 @@ def exercise_controller(test_case, controller): -def get_unit_tests(): +def get_unit_tests(prefix = None): """ Provides the classes for our unit tests. + :param str prefix: only provide the test if the module starts with this prefix + :returns: an **iterator** for our unit tests """ - for test_module in CONFIG["test.unit_tests"].splitlines(): - if test_module: - yield _import_test(test_module) + for line in CONFIG["test.unit_tests"].splitlines(): + if line: + test_class = _import_test(line) + + if prefix and not test_class.__module__.startswith(prefix): + continue + + yield test_class -def get_integ_tests(): +def get_integ_tests(prefix = None): """ Provides the classes for our integration tests. + :param str prefix: only provide the test if the module starts with this prefix + :returns: an **iterator** for our integration tests """ - for test_module in CONFIG["test.unit_tests"].splitlines(): - if test_module: - yield _import_test(test_module) + for line in CONFIG["test.integ_tests"].splitlines(): + if line: + test_class = _import_test(line) + + if prefix and not test_class.__module__.startswith(prefix): + continue + + yield test_class def _import_test(import_name):

1 0

[stem/master] Dropping load_user_configuration() helper
by atagar＠torproject.org 14 Apr '13

14 Apr '13

commit a8d1eb06768335ac2d95af78195608ba0f091f7b Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Apr 11 08:58:05 2013 -0700 Dropping load_user_configuration() helper One step closer to simplifying our arg parsing. The load_user_configuration() was mostly there to unify our configuration and commandline arguments. Now that the new-world-order will only have the later we should drop the helper to simplify the code. --- run_tests.py | 152 ++++++++++++++++++++++++++-------------------------------- 1 files changed, 68 insertions(+), 84 deletions(-) diff --git a/run_tests.py b/run_tests.py index 92c40ab..45c3e00 100755 --- a/run_tests.py +++ b/run_tests.py @@ -66,89 +66,6 @@ DEFAULT_RUN_TARGET = Target.RUN_OPEN ERROR_ATTR = (term.Color.RED, term.Attr.BOLD) -def load_user_configuration(test_config): - """ - Parses our commandline arguments, loading our custom test configuration if - '--config' was provided and then appending arguments to that. This does some - sanity checking on the input, printing an error and quitting if validation - fails. - """ - - arg_overrides = {} - - try: - opts = getopt.getopt(sys.argv[1:], OPT, OPT_EXPANDED)[0] - except getopt.GetoptError, exc: - print "%s (for usage provide --help)" % exc - sys.exit(1) - - for opt, arg in opts: - if opt in ("-a", "--all"): - arg_overrides["argument.unit"] = "true" - arg_overrides["argument.integ"] = "true" - arg_overrides["argument.style"] = "true" - elif opt in ("-u", "--unit"): - arg_overrides["argument.unit"] = "true" - elif opt in ("-i", "--integ"): - arg_overrides["argument.integ"] = "true" - elif opt in ("-s", "--style"): - arg_overrides["argument.style"] = "true" - elif opt == "--python3": - arg_overrides["argument.python3"] = "true" - elif opt == "--clean": - arg_overrides["argument.python3_clean"] = "true" - elif opt in ("-t", "--targets"): - integ_targets = arg.split(",") - - # validates the targets - if not integ_targets: - print "No targets provided" - sys.exit(1) - - for target in integ_targets: - if not target in Target: - print "Invalid integration target: %s" % target - sys.exit(1) - else: - target_config = test_config.get("target.config", {}).get(target) - - if target_config: - arg_overrides[target_config] = "true" - elif opt in ("-l", "--test"): - arg_overrides["argument.test"] = arg - elif opt in ("-l", "--log"): - arg_overrides["argument.log"] = arg.upper() - elif opt in ("--tor"): - arg_overrides["argument.tor"] = arg - elif opt in ("-h", "--help"): - # Prints usage information and quits. This includes a listing of the - # valid integration targets. - - print CONFIG["msg.help"] - - # gets the longest target length so we can show the entries in columns - target_name_length = max(map(len, Target)) - description_format = " %%-%is - %%s" % target_name_length - - for target in Target: - print description_format % (target, CONFIG["target.description"].get(target, "")) - - print - - sys.exit() - - for key, value in arg_overrides.items(): - test_config.set(key, value) - - # basic validation on user input - - log_config = CONFIG["argument.log"] - if log_config and not log_config in log.LOG_VALUES: - print "'%s' isn't a logging runlevel, use one of the following instead:" % log_config - print " TRACE, DEBUG, INFO, NOTICE, WARN, ERROR" - sys.exit(1) - - def _clean_orphaned_pyc(): test.output.print_noline(" checking for orphaned .pyc files... ", *test.runner.STATUS_ATTR) @@ -289,7 +206,74 @@ if __name__ == '__main__': settings_path = os.path.join(test.runner.STEM_BASE, "test", "settings.cfg") test_config.load(settings_path) - load_user_configuration(test_config) + try: + opts = getopt.getopt(sys.argv[1:], OPT, OPT_EXPANDED)[0] + except getopt.GetoptError, exc: + print "%s (for usage provide --help)" % exc + sys.exit(1) + + for opt, arg in opts: + if opt in ("-a", "--all"): + test_config.set("argument.unit", "true") + test_config.set("argument.integ", "true") + test_config.set("argument.style", "true") + elif opt in ("-u", "--unit"): + test_config.set("argument.unit", "true") + elif opt in ("-i", "--integ"): + test_config.set("argument.integ", "true") + elif opt in ("-s", "--style"): + test_config.set("argument.style", "true") + elif opt == "--python3": + test_config.set("argument.python3", "true") + elif opt == "--clean": + test_config.set("argument.python3_clean", "true") + elif opt in ("-t", "--targets"): + integ_targets = arg.split(",") + + # validates the targets + if not integ_targets: + print "No targets provided" + sys.exit(1) + + for target in integ_targets: + if not target in Target: + print "Invalid integration target: %s" % target + sys.exit(1) + else: + target_config = test_config.get("target.config", {}).get(target) + + if target_config: + test_config.set(target_config, "true") + elif opt in ("-l", "--test"): + test_config.set("argument.test", arg) + elif opt in ("-l", "--log"): + test_config.set("argument.log", arg.upper()) + elif opt in ("--tor"): + test_config.set("argument.tor", arg) + elif opt in ("-h", "--help"): + # Prints usage information and quits. This includes a listing of the + # valid integration targets. + + print CONFIG["msg.help"] + + # gets the longest target length so we can show the entries in columns + target_name_length = max(map(len, Target)) + description_format = " %%-%is - %%s" % target_name_length + + for target in Target: + print description_format % (target, CONFIG["target.description"].get(target, "")) + + print + + sys.exit() + + # basic validation on user input + + log_config = CONFIG["argument.log"] + if log_config and not log_config in log.LOG_VALUES: + print "'%s' isn't a logging runlevel, use one of the following instead:" % log_config + print " TRACE, DEBUG, INFO, NOTICE, WARN, ERROR" + sys.exit(1) # check that we have 2to3 and python3 available in our PATH if CONFIG["argument.python3"]:

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits April 2013