- tor-commits - lists.torproject.org

[stem/master] get_pid_by_port() unavailable on Mac OSX
by atagar＠torproject.org 25 Jun '12

25 Jun '12

commit 2fc1c420818bff19437a2b74713223a410d9898e Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jun 23 11:47:24 2012 -0700 get_pid_by_port() unavailable on Mac OSX Tests for get_pid_by_port() are failing on Mac OSX 10.5.8 because sockstat is unavailable and lsof neither provides ports nor accepts the flags we need. Noting this in the pydocs/comments, and skipping its integ tests when on a mac. --- stem/util/system.py | 16 +++++++++++++++- test/integ/util/system.py | 6 ++++++ 2 files changed, 21 insertions(+), 1 deletions(-) diff --git a/stem/util/system.py b/stem/util/system.py index 73fd47e..b4c326f 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -8,6 +8,7 @@ best-effort, providing None if the lookup fails. :: is_windows - checks if we're running on windows + is_mac - checks if we're running on a mac is_bsd - checks if we're running on the bsd family of operating systems is_available - determines if a command is availabe on this system is_running - determines if a given process is running @@ -58,6 +59,15 @@ def is_windows(): return platform.system() == "Windows" +def is_mac(): + """ + Checks if we are running on Mac OSX. + + :returns: bool to indicate if we're on a Mac + """ + + return platform.system() == "Darwin" + def is_bsd(): """ Checks if we are within the BSD family of operating systems. This presently @@ -260,7 +270,8 @@ def get_pid_by_port(port): 2. sockstat -4l -P tcp -p <port> 3. lsof -wnP -iTCP -sTCP:LISTEN | grep ":<port>" - Most queries limit results to listening TCP connections. + Most queries limit results to listening TCP connections. This function likely + won't work on Mac OSX. :param int port: port where the process we're looking for is listening @@ -301,6 +312,7 @@ def get_pid_by_port(port): # attempts to resolve using sockstat, failing if: # - sockstat doesn't accept the -4 flag (BSD only) + # - sockstat isn't available (encountered with OSX 10.5.8) # - there are multiple instances using the same port on different addresses # # flags: @@ -334,6 +346,8 @@ def get_pid_by_port(port): # resolves using lsof which works on both Linux and BSD, only failing if: # - lsof is unavailable (not included by default on OpenBSD) + # - lsof doesn't provide the port ip/port, nor accept the -i and -s args + # (encountered with OSX 10.5.8) # - the process being run as a different user due to permissions # - there are multiple instances using the same port on different addresses # diff --git a/test/integ/util/system.py b/test/integ/util/system.py index 174357e..7999cbb 100644 --- a/test/integ/util/system.py +++ b/test/integ/util/system.py @@ -201,6 +201,9 @@ class TestSystem(unittest.TestCase): if not _has_port(): test.runner.skip(self, "(test instance has no port)") return + elif stem.util.system.is_mac(): + test.runner.skip(self, "(resolvers unavailable)") + return elif not runner.is_ptraceable(): test.runner.skip(self, "(DisableDebuggerAttachment is set)") return @@ -271,6 +274,9 @@ class TestSystem(unittest.TestCase): elif not stem.util.system.is_available("lsof"): test.runner.skip(self, "(lsof unavailable)") return + elif stem.util.system.is_mac(): + test.runner.skip(self, "(resolvers unavailable)") + return elif not runner.is_ptraceable(): test.runner.skip(self, "(DisableDebuggerAttachment is set)") return

1 0

[stem/master] Respecting custom tor commands for process integ tests
by atagar＠torproject.org 25 Jun '12

25 Jun '12

commit a5f6006b3e1d072e1cd9524aa57408d707ff3ba0 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Jun 22 09:27:03 2012 -0700 Respecting custom tor commands for process integ tests The process integ tests were attempting to run 'tor', which naturally doesn't work if it isn't in your path. Using the command that we're using for our main test instance instead since that is sure to work. --- test/integ/process.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integ/process.py b/test/integ/process.py index 2037d93..0573a1c 100644 --- a/test/integ/process.py +++ b/test/integ/process.py @@ -28,7 +28,9 @@ class TestProcess(unittest.TestCase): # Launch tor without a torrc, but with a control port. Confirms that this # works by checking that we're still able to access the new instance. + runner = test.runner.get_runner() tor_process = stem.process.launch_tor_with_config( + tor_cmd = runner.get_tor_command(), config = {'SocksPort': '2777', 'ControlPort': '2778'}, completion_percent = 5 ) @@ -36,7 +38,6 @@ class TestProcess(unittest.TestCase): control_socket = None try: control_socket = stem.socket.ControlPort(control_port = 2778) - runner = test.runner.get_runner() stem.connection.authenticate(control_socket, chroot_path = runner.get_chroot()) # exercises the socket @@ -62,8 +63,9 @@ class TestProcess(unittest.TestCase): if test.runner.only_run_once(self, "test_launch_tor_with_timeout"): return + runner = test.runner.get_runner() start_time = time.time() - self.assertRaises(OSError, stem.process.launch_tor_with_config, {'SocksPort': '2777'}, "tor", 100, None, 2) + self.assertRaises(OSError, stem.process.launch_tor_with_config, {'SocksPort': '2777'}, runner.get_tor_command(), 100, None, 2) runtime = time.time() - start_time if not (runtime > 2 and runtime < 3):

1 0

[stem/master] Missing kill() fallback when ending integ tests
by atagar＠torproject.org 25 Jun '12

25 Jun '12

commit 4e02850db89006f256290e8bc7bea7169c6cd57c Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jun 23 11:55:57 2012 -0700 Missing kill() fallback when ending integ tests As an earlier commit mentioned, Popen.kill() is unavailable on python 2.5, and its fallback (os.kill()) is unavailable on Windows. Accounting for this in the integ test runner. --- test/runner.py | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diff --git a/test/runner.py b/test/runner.py index 1bf1860..3b696b3 100644 --- a/test/runner.py +++ b/test/runner.py @@ -41,6 +41,7 @@ import sys import time import stat import shutil +import signal import logging import tempfile import threading @@ -305,7 +306,13 @@ class Runner: # if the tor process has stopped on its own then the following raises # an OSError ([Errno 3] No such process) - try: self._tor_process.kill() + try: + if stem.prereq.is_python_26(): + self._tor_process.kill() + elif not stem.util.system.is_windows(): + os.kill(self._tor_process.pid, signal.SIGTERM) + else: + test.output.print_line("failed (unable to call kill() in python 2.5)", *ERROR_ATTR) except OSError: pass self._tor_process.communicate() # blocks until the process is done

1 0

[stem/master] Inverted check when falling back to kill a process
by atagar＠torproject.org 25 Jun '12

25 Jun '12

commit 04a6225ffdccc81a431044d05f4f411c7e5a88f5 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jun 23 14:18:32 2012 -0700 Inverted check when falling back to kill a process Oops, we're supposed to use os.kill() as a fallback if we *aren't* on windows. --- stem/process.py | 4 ++-- test/integ/process.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stem/process.py b/stem/process.py index e9d453e..337699d 100644 --- a/stem/process.py +++ b/stem/process.py @@ -87,7 +87,7 @@ def launch_tor(tor_cmd = "tor", args = None, torrc_path = None, completion_perce if stem.prereq.is_python_26(): tor_process.kill() - elif stem.util.system.is_windows(): + elif not stem.util.system.is_windows(): os.kill(tor_process.pid, signal.SIGTERM) raise OSError("reached a %i second timeout without success" % timeout) @@ -107,7 +107,7 @@ def launch_tor(tor_cmd = "tor", args = None, torrc_path = None, completion_perce # ... but best make sure if stem.prereq.is_python_26(): tor_process.kill() - elif stem.util.system.is_windows(): + elif not stem.util.system.is_windows(): os.kill(tor_process.pid, signal.SIGTERM) raise OSError("Process terminated: %s" % last_problem) diff --git a/test/integ/process.py b/test/integ/process.py index 0573a1c..423b2c3 100644 --- a/test/integ/process.py +++ b/test/integ/process.py @@ -49,7 +49,7 @@ class TestProcess(unittest.TestCase): if stem.prereq.is_python_26(): tor_process.kill() - elif stem.util.system.is_windows(): + elif not stem.util.system.is_windows(): os.kill(tor_process.pid, signal.SIGTERM) def test_launch_tor_with_timeout(self):

1 0

[stem/master] Respecting reader stop() calls in mid-directory
by atagar＠torproject.org 25 Jun '12

25 Jun '12

commit e625e84b003420908d410ab10f51fdd78a1ccf17 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jun 23 12:16:53 2012 -0700 Respecting reader stop() calls in mid-directory If we, say, attempted to read a directory with a million files then the reader's stop() method won't be respected until we've finished with that directory. That's stupid. Checking the _is_stopped Event after processing each file. Python lacks targeted breaks, so I needed to refactor the code a bit to accomidate this. --- stem/descriptor/reader.py | 16 +++++++++------- 1 files changed, 9 insertions(+), 7 deletions(-) diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 716b58c..8d4a462 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -354,13 +354,7 @@ class DescriptorReader: else: walker = os.walk(target) - # adds all of the files that it contains - for root, _, files in walker: - for filename in files: - self._handle_file(os.path.join(root, filename), new_processed_files) - - # this can take a while if, say, we're including the root directory - if self._is_stopped.isSet(): break + self._handle_walker(walker, new_processed_files) else: self._handle_file(target, new_processed_files) @@ -383,6 +377,14 @@ class DescriptorReader: self._iter_notice.wait() self._iter_notice.clear() + def _handle_walker(self, walker, new_processed_files): + for root, _, files in walker: + for filename in files: + self._handle_file(os.path.join(root, filename), new_processed_files) + + # this can take a while if, say, we're including the root directory + if self._is_stopped.isSet(): return + def _handle_file(self, target, new_processed_files): # This is a file. Register its last modified timestamp and check if # it's a file that we should skip.

1 0

[stem/master] Python 2.5 and OSX compatability fixes
by atagar＠torproject.org 25 Jun '12

25 Jun '12

commit 4b79530c47abdeebb33bc7c06a6a6bda4e2815b6 Merge: 09acb5e 1c01521 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jun 24 18:20:59 2012 -0700 Python 2.5 and OSX compatability fixes stem/control.py | 10 ++++-- stem/descriptor/reader.py | 60 ++++++++++++++++++++++++-------------- stem/process.py | 18 ++++++++++- stem/util/system.py | 19 +++++++++++- test/integ/descriptor/reader.py | 9 +++++- test/integ/process.py | 30 +++++++++++++++++-- test/integ/util/system.py | 13 +++++++- test/runner.py | 9 +++++- 8 files changed, 133 insertions(+), 35 deletions(-)

1 0

[stem/master] OSX sucks at terminating processes
by atagar＠torproject.org 25 Jun '12

25 Jun '12

commit 1c0152185a9ff27a49f6491bcd2ea06f64158820 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jun 23 14:36:08 2012 -0700 OSX sucks at terminating processes Our os.kill() call evidently doesn't kill tor right away on OSX, causing tests to disrupt each other (port conflicts). --- test/integ/process.py | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/test/integ/process.py b/test/integ/process.py index 423b2c3..58440e0 100644 --- a/test/integ/process.py +++ b/test/integ/process.py @@ -51,6 +51,12 @@ class TestProcess(unittest.TestCase): tor_process.kill() elif not stem.util.system.is_windows(): os.kill(tor_process.pid, signal.SIGTERM) + + # On OSX, python 2.5 this kill call doesn't seem to block, causing our + # tor instance to linger and cause a port conflict with the following + # test. Giving it a moment to kill for realz. + + time.sleep(0.5) def test_launch_tor_with_timeout(self): """

1 0

[stem/master] Stopping tor initialization timeout when done
by atagar＠torproject.org 25 Jun '12

25 Jun '12

commit 54a013552266540d5029af2ff08db2fed1c1dc92 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jun 24 14:14:49 2012 -0700 Stopping tor initialization timeout when done Ok, I'm kinda surprised that we haven't tripped over this before. We never terminate the tor initialization timeout on either or success or failure, causing us to arbitrarily kill our process after 90 seconds. This bug probably survived so long becuase our integ tests take less time than that to complete, and then spawns a new tor instance resetting the timer again. --- stem/process.py | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/stem/process.py b/stem/process.py index 337699d..04f787a 100644 --- a/stem/process.py +++ b/stem/process.py @@ -104,6 +104,8 @@ def launch_tor(tor_cmd = "tor", args = None, torrc_path = None, completion_perce # this will provide empty results if the process is terminated if not init_line: + if timeout: signal.alarm(0) # stop alarm + # ... but best make sure if stem.prereq.is_python_26(): tor_process.kill() @@ -120,6 +122,8 @@ def launch_tor(tor_cmd = "tor", args = None, torrc_path = None, completion_perce problem_match = problem_line.search(init_line) if bootstrap_match and int(bootstrap_match.groups()[0]) >= completion_percent: + if timeout: signal.alarm(0) # stop alarm + if temp_file: try: os.remove(temp_file) except: pass

1 0

[stem/master] Remove single letter variable
by atagar＠torproject.org 25 Jun '12

25 Jun '12

commit 86baf8e562c0486d2ac2e2be893480eb149742b4 Author: Sathyanarayanan Gunasekaran <gsathya.ceg(a)gmail.com> Date: Sat Jun 23 19:16:38 2012 +0530 Remove single letter variable According to the guidelines of our benevolent dictator we must abolish the use of single letter variables. The sole infiltrator has been abolished. Once again, there is peace in the land of stem. --- stem/version.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stem/version.py b/stem/version.py index cfb2e12..c63a92a 100644 --- a/stem/version.py +++ b/stem/version.py @@ -100,10 +100,10 @@ class Version: def __init__(self, version_str): self.version_str = version_str - m = re.match(r'^([0-9]+)\.([0-9]+)\.([0-9]+)(\.[0-9]+)?(-\S*)?$', version_str) + version_parts = re.match(r'^([0-9]+)\.([0-9]+)\.([0-9]+)(\.[0-9]+)?(-\S*)?$', version_str) - if m: - major, minor, micro, patch, status = m.groups() + if version_parts: + major, minor, micro, patch, status = version_parts.groups() # The patch and status matches are optional (may be None) and have an extra # proceeding period or dash if they exist. Stripping those off.

1 0

[stem/master] Integ test relied on tor being in path
by atagar＠torproject.org 25 Jun '12

25 Jun '12

commit cec8ecfcf175af55d9503d45e169cd26f9664279 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jun 19 09:47:33 2012 -0700 Integ test relied on tor being in path I thought that it was a safe assumption that tor was in our path since the integ tests will only run if it can start tor. However, the tests have a '--tor <path to tor>' argument which means that the tests may be running without our path including the tor binary (or even having the binary called 'tor'). Changing the test to look for 'ls' or 'dir' instead since every platform that I can think of has that. Caught by Karsten. --- stem/util/system.py | 3 +++ test/integ/util/system.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/stem/util/system.py b/stem/util/system.py index d6a3ce2..73fd47e 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -74,6 +74,9 @@ def is_available(command, cached=True): than one command is present (for instance "ls -a | grep foo") then this just checks the first. + Note that many builtin common commands (like cd and ulimit) aren't in the + PATH so this lookup will fail for them. + :param str command: command to search for :param bool cached: makes use of available cached results if True diff --git a/test/integ/util/system.py b/test/integ/util/system.py index 6cdf0ec..174357e 100644 --- a/test/integ/util/system.py +++ b/test/integ/util/system.py @@ -61,8 +61,11 @@ class TestSystem(unittest.TestCase): Checks the stem.util.system.is_available function. """ - # since we're running tor it would be kinda sad if this didn't detect it - self.assertTrue(stem.util.system.is_available("tor")) + # I have yet to see a platform without 'ls' + if stem.util.system.is_windows(): + self.assertTrue(stem.util.system.is_available("dir")) + else: + self.assertTrue(stem.util.system.is_available("ls")) # but it would be kinda weird if this did... self.assertFalse(stem.util.system.is_available("blarg_and_stuff"))

1 0