commit 1b44b967e75b08d6f702ffa1507d8ad8c4980bda Author: Damian Johnson <atagar@torproject.org> Date: Mon Oct 17 09:45:43 2011 -0700 Integration tests / fixes for types.ControlMessage Adding integration tests for basic control port communication, exercising... - connection failure - bad commands - bad getinfo queries - general getinfo queries - setevent/basic event parsing This also includes fixes for a variety of issues found while testing. --- run_tests.py | 4 +- stem/types.py | 18 ++++- stem/util/system.py | 53 ++++++++---- test/integ/message.py | 234 +++++++++++++++++++++++++++++++++++++++++++++++++ test/integ/system.py | 1 - test/runner.py | 29 ++++++- 6 files changed, 314 insertions(+), 25 deletions(-) diff --git a/run_tests.py b/run_tests.py index 6d62ab3..1741892 100755 --- a/run_tests.py +++ b/run_tests.py @@ -11,6 +11,7 @@ import unittest import test.runner import test.unit.message import test.unit.version +import test.integ.message import test.integ.system from stem.util import enum, term @@ -24,7 +25,8 @@ UNIT_TESTS = (("stem.types.ControlMessage", test.unit.message.TestMessageFunctio ("stem.types.Version", test.unit.version.TestVerionFunctions), ) -INTEG_TESTS = (("stem.util.system", test.integ.system.TestSystemFunctions), +INTEG_TESTS = (("stem.types.ControlMessage", test.integ.message.TestMessageFunctions), + ("stem.util.system", test.integ.system.TestSystemFunctions), ) # Configurations that the intergration tests can be ran with. Attributs are diff --git a/stem/types.py b/stem/types.py index 2339007..818b6d3 100644 --- a/stem/types.py +++ b/stem/types.py @@ -53,14 +53,28 @@ def read_message(control_file): while True: try: line = control_file.readline() - except socket.error, exc: raise ControlSocketClosed(exc) + except AttributeError, exc: + # if the control_file has been closed then we will receive: + # AttributeError: 'NoneType' object has no attribute 'recv' + + log.log(log.WARN, "ControlSocketClosed: socket file has been closed") + raise ControlSocketClosed("socket file has been closed") + except socket.error, exc: + log.log(log.WARN, "ControlSocketClosed: received an exception (%s)" % exc) + raise ControlSocketClosed(exc) raw_content += line # Parses the tor control lines. These are of the form... # <status code><divider><content>\r\n - if len(line) < 4: + if len(line) == 0: + # if the socket is disconnected then the readline() method will provide + # empty content + + log.log(log.WARN, "ControlSocketClosed: empty socket content") + raise ControlSocketClosed("Received empty socket content.") + elif len(line) < 4: log.log(log.WARN, "ProtocolError: line too short (%s)" % line) raise ProtocolError("Badly formatted reply line: too short") elif not re.match(r'^[a-zA-Z0-9]{3}[-+ ]', line): diff --git a/stem/util/system.py b/stem/util/system.py index 746eed4..5f7e9ca 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -3,6 +3,7 @@ Helper functions for working with the underlying system. These are mostly os dependent, only working on linux, osx, and bsd. """ +import re import os import time import subprocess @@ -123,7 +124,7 @@ def get_pid(process_name, process_port = None): try: results = call("pgrep -x %s" % process_name) - if len(results) == 1 and len(results[0].split()) == 1: + if results and len(results) == 1 and len(results[0].split()) == 1: pid = results[0].strip() if pid.isdigit(): return int(pid) except IOError: pass @@ -135,7 +136,7 @@ def get_pid(process_name, process_port = None): try: results = call("pidof %s" % process_name) - if len(results) == 1 and len(results[0].split()) == 1: + if results and len(results) == 1 and len(results[0].split()) == 1: pid = results[0].strip() if pid.isdigit(): return int(pid) except IOError: pass @@ -145,12 +146,16 @@ def get_pid(process_name, process_port = None): if process_port: try: - results = call("netstat -npl | grep 127.0.0.1:%i" % process_port) + results = call("netstat -npl") - if len(results) == 1: - results = results[0].split()[6] # process field (ex. "7184/tor") - pid = results[:results.find("/")] - if pid.isdigit(): return int(pid) + # filters to results with our port (same as "grep 127.0.0.1:<port>") + if results: + results = [r for r in results if "127.0.0.1:%i" % process_port in r] + + if len(results) == 1: + results = results[0].split()[6] # process field (ex. "7184/tor") + pid = results[:results.find("/")] + if pid.isdigit(): return int(pid) except IOError: pass # attempts to resolve using ps, failing if: @@ -160,7 +165,7 @@ def get_pid(process_name, process_port = None): try: results = call("ps -o pid -C %s" % process_name) - if len(results) == 2: + if results and len(results) == 2: pid = results[1].strip() if pid.isdigit(): return int(pid) except IOError: pass @@ -175,11 +180,15 @@ def get_pid(process_name, process_port = None): if process_port: try: - results = call("sockstat -4l -P tcp -p %i | grep %s" % (process_port, process_name)) + results = call("sockstat -4l -P tcp -p %i" % process_port) - if len(results) == 1 and len(results[0].split()) == 7: - pid = results[0].split()[2] - if pid.isdigit(): return int(pid) + # filters to results with our port (same as "grep <name>") + if results: + results = [r for r in results if process_name in r] + + if len(results) == 1 and len(results[0].split()) == 7: + pid = results[0].split()[2] + if pid.isdigit(): return int(pid) except IOError: pass # attempts to resolve via a ps command that works on mac/bsd (this and lsof @@ -188,11 +197,15 @@ def get_pid(process_name, process_port = None): # - there are multiple instances try: - results = call("ps axc | egrep \" %s$\"" % process_name) + results = call("ps axc") - if len(results) == 1 and len(results[0].split()) > 0: - pid = results[0].split()[0] - if pid.isdigit(): return int(pid) + # filters to results with our port (same as "egrep ' <name>$'") + if results: + results = [r for r in results if r.endswith(" %s" % process_name)] + + if len(results) == 1 and len(results[0].split()) > 0: + pid = results[0].split()[0] + if pid.isdigit(): return int(pid) except IOError: pass # attempts to resolve via lsof, this should work on linux, mac, and bsd @@ -202,8 +215,12 @@ def get_pid(process_name, process_port = None): # - there are multiple instances using the same port on different addresses try: - port_comp = str(process_port) if process_port else "" - results = call("lsof -wnPi | egrep \"^%s.*:%s\"" % (process_name, port_comp)) + results = call("lsof -wnPi") + + # filters to results with our port (same as "egrep '^<name>.*:<port>'") + if results: + port_comp = str(process_port) if process_port else "" + results = [r for r in results if re.match("^%s.*:%s" % (process_name, port_comp), r)] # This can result in multiple entries with the same pid (from the query # itself). Checking all lines to see if they're in agreement about the pid. diff --git a/test/integ/message.py b/test/integ/message.py new file mode 100644 index 0000000..46f20d2 --- /dev/null +++ b/test/integ/message.py @@ -0,0 +1,234 @@ +""" +Integration tests for the types.ControlMessage class. +""" + +import re +import socket +import unittest + +import stem.types +import test.runner + +class TestMessageFunctions(unittest.TestCase): + """ + Exercises the 'types.ControlMessage' class with an actual tor instance. + """ + + def test_unestablished_socket(self): + """ + Checks message parsing when we have a valid but unauthenticated socket. + """ + + control_socket, control_socket_file = self._get_control_socket(False) + + # If an unauthenticated connection gets a message besides AUTHENTICATE or + # PROTOCOLINFO then tor will give an 'Authentication required.' message and + # hang up. + + control_socket_file.write("GETINFO version\r\n") + control_socket_file.flush() + + auth_required_response = stem.types.read_message(control_socket_file) + self.assertEquals("Authentication required.", str(auth_required_response)) + self.assertEquals(["Authentication required."], list(auth_required_response)) + self.assertEquals("514 Authentication required.\r\n", auth_required_response.raw_content()) + self.assertEquals([("514", " ", "Authentication required.")], auth_required_response.content()) + + # The socket's broken but doesn't realize it yet. Send another message and + # it should fail with a closed exception. + + control_socket_file.write("GETINFO version\r\n") + control_socket_file.flush() + + self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) + + # Additional socket usage should fail, and pulling more responses will fail + # with more closed exceptions. + + control_socket_file.write("GETINFO version\r\n") + self.assertRaises(socket.error, control_socket_file.flush) + self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) + self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) + self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) + + # The socket connection is already broken so calling close shouldn't have + # an impact. + + control_socket.close() + control_socket_file.write("GETINFO version\r\n") + self.assertRaises(socket.error, control_socket_file.flush) + self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) + + # Closing the file handler, however, will cause a different type of error. + + control_socket_file.close() + control_socket_file.write("GETINFO version\r\n") + + # receives: AttributeError: 'NoneType' object has no attribute 'sendall' + self.assertRaises(AttributeError, control_socket_file.flush) + + # receives: stem.types.ControlSocketClosed: socket file has been closed + self.assertRaises(stem.types.ControlSocketClosed, stem.types.read_message, control_socket_file) + + def test_invalid_command(self): + """ + Parses the response for a command which doesn't exist. + """ + + control_socket, control_socket_file = self._get_control_socket() + + control_socket_file.write("blarg\r\n") + control_socket_file.flush() + + unrecognized_command_response = stem.types.read_message(control_socket_file) + self.assertEquals('Unrecognized command "blarg"', str(unrecognized_command_response)) + self.assertEquals(['Unrecognized command "blarg"'], list(unrecognized_command_response)) + self.assertEquals('510 Unrecognized command "blarg"\r\n', unrecognized_command_response.raw_content()) + self.assertEquals([('510', ' ', 'Unrecognized command "blarg"')], unrecognized_command_response.content()) + + control_socket.close() + control_socket_file.close() + + def test_invalid_getinfo(self): + """ + Parses the response for a GETINFO query which doesn't exist. + """ + + control_socket, control_socket_file = self._get_control_socket() + + control_socket_file.write("GETINFO blarg\r\n") + control_socket_file.flush() + + unrecognized_key_response = stem.types.read_message(control_socket_file) + self.assertEquals('Unrecognized key "blarg"', str(unrecognized_key_response)) + self.assertEquals(['Unrecognized key "blarg"'], list(unrecognized_key_response)) + self.assertEquals('552 Unrecognized key "blarg"\r\n', unrecognized_key_response.raw_content()) + self.assertEquals([('552', ' ', 'Unrecognized key "blarg"')], unrecognized_key_response.content()) + + control_socket.close() + control_socket_file.close() + + def test_getinfo_config_file(self): + """ + Parses the 'GETINFO config-file' response. + """ + + runner = test.runner.get_runner() + torrc_dst = runner.get_torrc_path() + + control_socket, control_socket_file = self._get_control_socket() + + control_socket_file.write("GETINFO config-file\r\n") + control_socket_file.flush() + + config_file_response = stem.types.read_message(control_socket_file) + self.assertEquals("config-file=%s\nOK" % torrc_dst, str(config_file_response)) + self.assertEquals(["config-file=%s" % torrc_dst, "OK"], list(config_file_response)) + self.assertEquals("250-config-file=%s\r\n250 OK\r\n" % torrc_dst, config_file_response.raw_content()) + self.assertEquals([("250", "-", "config-file=%s" % torrc_dst), ("250", " ", "OK")], config_file_response.content()) + + control_socket.close() + control_socket_file.close() + + def test_getinfo_config_text(self): + """ + Parses the 'GETINFO config-text' response. + """ + + # We can't be certain of the order, and there may be extra config-text + # entries as per... + # https://trac.torproject.org/projects/tor/ticket/2362 + # + # so we'll just check that the response is a superset of our config + + runner = test.runner.get_runner() + torrc_contents = [] + + for line in runner.get_torrc_contents().split("\n"): + line = line.strip() + + if line and not line.startswith("#"): + torrc_contents.append(line) + + control_socket, control_socket_file = self._get_control_socket() + + control_socket_file.write("GETINFO config-text\r\n") + control_socket_file.flush() + + config_text_response = stem.types.read_message(control_socket_file) + + # the response should contain two entries, the first being a data response + self.assertEqual(2, len(list(config_text_response))) + self.assertEqual("OK", list(config_text_response)[1]) + self.assertEqual(("250", " ", "OK"), config_text_response.content()[1]) + self.assertTrue(config_text_response.raw_content().startswith("250+config-text=\r\n")) + self.assertTrue(config_text_response.raw_content().endswith("\r\n.\r\n250 OK\r\n")) + self.assertTrue(str(config_text_response).startswith("config-text=\n")) + self.assertTrue(str(config_text_response).endswith("\nOK")) + + for torrc_entry in torrc_contents: + self.assertTrue("\n%s\n" % torrc_entry in str(config_text_response)) + self.assertTrue(torrc_entry in list(config_text_response)[0]) + self.assertTrue("%s\r\n" % torrc_entry in config_text_response.raw_content()) + self.assertTrue("%s" % torrc_entry in config_text_response.content()[0][2]) + + control_socket.close() + control_socket_file.close() + + def test_bw_event(self): + """ + Issues 'SETEVENTS BW' and parses a few events. + """ + + control_socket, control_socket_file = self._get_control_socket() + + control_socket_file.write("SETEVENTS BW\r\n") + control_socket_file.flush() + + setevents_response = stem.types.read_message(control_socket_file) + self.assertEquals("OK", str(setevents_response)) + self.assertEquals(["OK"], list(setevents_response)) + self.assertEquals("250 OK\r\n", setevents_response.raw_content()) + self.assertEquals([("250", " ", "OK")], setevents_response.content()) + + # Tor will emit a BW event once per second. Parsing three of them. + + for _ in range(3): + bw_event = stem.types.read_message(control_socket_file) + self.assertTrue(re.match("BW [0-9]+ [0-9]+", str(bw_event))) + self.assertTrue(re.match("650 BW [0-9]+ [0-9]+\r\n", bw_event.raw_content())) + self.assertEquals(("650", " "), bw_event.content()[0][:2]) + + control_socket.close() + control_socket_file.close() + + def _get_control_socket(self, authenticate = True): + """ + Provides a socket connected to the tor test instance's control port. + + Arguments: + authenticate (bool) - if True then the socket is authenticated + + Returns: + (socket.socket, file) tuple with the control socket and its file + """ + + runner = test.runner.get_runner() + + control_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + control_socket.connect(("127.0.0.1", runner.get_control_port())) + control_socket_file = control_socket.makefile() + + if authenticate: + control_socket_file.write("AUTHENTICATE\r\n") + control_socket_file.flush() + + authenticate_response = stem.types.read_message(control_socket_file) + + self.assertEquals("OK", str(authenticate_response)) + self.assertEquals(["OK"], list(authenticate_response)) + self.assertEquals("250 OK\r\n", authenticate_response.raw_content()) + self.assertEquals([("250", " ", "OK")], authenticate_response.content()) + + return (control_socket, control_socket_file) + diff --git a/test/integ/system.py b/test/integ/system.py index 8661660..1fcb3d2 100644 --- a/test/integ/system.py +++ b/test/integ/system.py @@ -37,7 +37,6 @@ class TestSystemFunctions(unittest.TestCase): """ runner = test.runner.get_runner() - self.assertEquals(runner.get_pid(), system.get_pid("tor")) self.assertEquals(runner.get_pid(), system.get_pid("tor", runner.get_control_port())) self.assertEquals(None, system.get_pid("blarg_and_stuff")) diff --git a/test/runner.py b/test/runner.py index b3abdfa..6f76d01 100644 --- a/test/runner.py +++ b/test/runner.py @@ -5,6 +5,7 @@ Runtime context for the integration tests. import os import sys import time +import shutil import signal import tempfile import subprocess @@ -12,10 +13,11 @@ import subprocess from stem.util import term # number of seconds before we time out our attempt to start a tor instance -TOR_INIT_TIMEOUT = 60 +TOR_INIT_TIMEOUT = 90 BASIC_TORRC = """# configuration for stem integration tests DataDirectory %s +SocksPort 0 ControlPort 1111 """ @@ -57,7 +59,7 @@ class Runner: raise exc # writes our testing torrc - torrc_dst = os.path.join(self._test_dir, "torrc") + torrc_dst = self.get_torrc_path() try: sys.stdout.write(term.format(" writing torrc (%s)... " % torrc_dst, term.Color.BLUE, term.Attr.BOLD)) @@ -93,7 +95,7 @@ class Runner: if self._tor_process: self._tor_process.kill() # double check that we have a torrc to work with - torrc_dst = os.path.join(self._test_dir, "torrc") + torrc_dst = self.get_torrc_path() if not os.path.exists(torrc_dst): raise OSError("torrc doesn't exist (%s)" % torrc_dst) @@ -139,6 +141,7 @@ class Runner: self._tor_process.kill() self._tor_process.communicate() # blocks until the process is done self._tor_process = None + shutil.rmtree(self._test_dir, ignore_errors=True) sys.stdout.write(term.format("done\n", term.Color.BLUE, term.Attr.BOLD)) def get_pid(self): @@ -164,4 +167,24 @@ class Runner: # TODO: this will be fetched from torrc contents when we use custom configs return 1111 + + def get_torrc_path(self): + """ + Provides the absolute path for where our testing torrc resides. + + Returns: + str with our torrc path + """ + + return os.path.join(self._test_dir, "torrc") + + def get_torrc_contents(self): + """ + Provides the contents of our torrc. + + Returns: + str with the contents of our torrc, lines are newline separated + """ + + return self._torrc_contents