January 2014 - tor-commits - lists.torproject.org

[arm/master] Testing the ResourceTracker class
by atagar＠torproject.org 05 Jan '14

05 Jan '14

commit 0cede13371c0796a5a819ef65466b9d2aaf0b806 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 5 16:15:20 2014 -0800 Testing the ResourceTracker class Unit tests for our daemon that tracks the resource usage of tor. --- arm/util/tracker.py | 20 ++--- test/util/tracker/resource_tracker.py | 142 +++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 10 deletions(-) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index 1a716d3..b0f99b2 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -31,7 +31,7 @@ Background tasks for gathering information about the tor process. :var float cpu_average: average cpu usage since we first started tracking the process :var float cpu_total: total cpu time the process has used since starting :var int memory_bytes: memory usage of the process in bytes - :var float memory_precent: percentage of our memory used by this process + :var float memory_percent: percentage of our memory used by this process :var float timestamp: unix timestamp for when this information was fetched """ @@ -70,7 +70,7 @@ def get_connection_tracker(): global CONNECTION_TRACKER if CONNECTION_TRACKER is None: - CONNECTION_TRACKER = ConnectionTracker() + CONNECTION_TRACKER = ConnectionTracker(CONFIG['queries.connections.rate']) return CONNECTION_TRACKER @@ -83,7 +83,7 @@ def get_resource_tracker(): global RESOURCE_TRACKER if RESOURCE_TRACKER is None: - RESOURCE_TRACKER = ResourceTracker() + RESOURCE_TRACKER = ResourceTracker(CONFIG['queries.resources.rate']) return RESOURCE_TRACKER @@ -147,9 +147,9 @@ def _resources_via_ps(pid): total_cpu_time = str_tools.parse_short_time_label(stats[0]) uptime = str_tools.parse_short_time_label(stats[1]) memory_bytes = int(stats[2]) * 1024 # ps size is in kb - memory_precent = float(stats[3]) / 100.0 + memory_percent = float(stats[3]) / 100.0 - return (total_cpu_time, uptime, memory_bytes, memory_precent) + return (total_cpu_time, uptime, memory_bytes, memory_percent) except ValueError: pass @@ -320,8 +320,8 @@ class ConnectionTracker(Daemon): Periodically retrieves the connections established by tor. """ - def __init__(self): - super(ConnectionTracker, self).__init__(CONFIG['queries.connections.rate']) + def __init__(self, rate): + super(ConnectionTracker, self).__init__(rate) self._connections = [] self._resolvers = connection.get_system_resolvers() @@ -437,8 +437,8 @@ class ResourceTracker(Daemon): Periodically retrieves the resource usage of tor. """ - def __init__(self): - super(ResourceTracker, self).__init__(CONFIG['queries.resources.rate']) + def __init__(self, rate): + super(ResourceTracker, self).__init__(rate) self._resources = None self._use_proc = proc.is_available() # determines if we use proc or ps for lookups @@ -469,7 +469,7 @@ class ResourceTracker(Daemon): cpu_average = total_cpu_time / uptime, cpu_total = total_cpu_time, memory_bytes = memory_in_bytes, - memory_precent = memory_in_percent, + memory_percent = memory_in_percent, timestamp = time.time(), ) diff --git a/test/util/tracker/resource_tracker.py b/test/util/tracker/resource_tracker.py new file mode 100644 index 0000000..70cdf27 --- /dev/null +++ b/test/util/tracker/resource_tracker.py @@ -0,0 +1,142 @@ +import time +import unittest + +from arm.util.tracker import ResourceTracker, _resources_via_ps, _resources_via_proc + +from mock import Mock, patch + +PS_OUTPUT = """\ + TIME ELAPSED RSS %MEM +00:00:02 00:18 18848 0.4 +""" + + +class TestResourceTracker(unittest.TestCase): + @patch('arm.util.tracker.tor_controller') + @patch('arm.util.tracker._resources_via_proc') + @patch('arm.util.tracker.system', Mock(return_value = Mock())) + @patch('arm.util.tracker.proc.is_available', Mock(return_value = True)) + def test_fetching_samplings(self, resources_via_proc_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + resources_via_proc_mock.return_value = (105.3, 2.4, 8072, 0.3) + + with ResourceTracker(0.04) as daemon: + time.sleep(0.01) + + resources = daemon.get_resource_usage() + + self.assertEqual(1, daemon.run_counter()) + self.assertEqual(0.0, resources.cpu_sample) + self.assertEqual(43.875, resources.cpu_average) + self.assertEqual(105.3, resources.cpu_total) + self.assertEqual(8072, resources.memory_bytes) + self.assertEqual(0.3, resources.memory_percent) + self.assertTrue((time.time() - resources.timestamp) < 0.5) + + resources_via_proc_mock.return_value = (800.3, 3.2, 6020, 0.26) + time.sleep(0.05) + resources = daemon.get_resource_usage() + + self.assertEqual(2, daemon.run_counter()) + self.assertEqual(6.600189933523267, resources.cpu_sample) + self.assertEqual(250.09374999999997, resources.cpu_average) + self.assertEqual(800.3, resources.cpu_total) + self.assertEqual(6020, resources.memory_bytes) + self.assertEqual(0.26, resources.memory_percent) + self.assertTrue((time.time() - resources.timestamp) < 0.5) + + resources_via_proc_mock.assert_called_with(12345) + + @patch('arm.util.tracker.tor_controller') + @patch('arm.util.tracker.proc.is_available') + @patch('arm.util.tracker._resources_via_ps', Mock(return_value = (105.3, 2.4, 8072, 0.3))) + @patch('arm.util.tracker._resources_via_proc', Mock(return_value = (340.3, 3.2, 6020, 0.26))) + @patch('arm.util.tracker.system', Mock(return_value = Mock())) + def test_picking_proc_or_ps(self, is_proc_available_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + + is_proc_available_mock.return_value = True + + with ResourceTracker(0.04) as daemon: + time.sleep(0.01) + + resources = daemon.get_resource_usage() + + self.assertEqual(1, daemon.run_counter()) + self.assertEqual(0.0, resources.cpu_sample) + self.assertEqual(106.34375, resources.cpu_average) + self.assertEqual(340.3, resources.cpu_total) + self.assertEqual(6020, resources.memory_bytes) + self.assertEqual(0.26, resources.memory_percent) + self.assertTrue((time.time() - resources.timestamp) < 0.5) + + is_proc_available_mock.return_value = False + + with ResourceTracker(0.04) as daemon: + time.sleep(0.01) + + resources = daemon.get_resource_usage() + + self.assertEqual(1, daemon.run_counter()) + self.assertEqual(0.0, resources.cpu_sample) + self.assertEqual(43.875, resources.cpu_average) + self.assertEqual(105.3, resources.cpu_total) + self.assertEqual(8072, resources.memory_bytes) + self.assertEqual(0.3, resources.memory_percent) + self.assertTrue((time.time() - resources.timestamp) < 0.5) + + @patch('arm.util.tracker.tor_controller') + @patch('arm.util.tracker._resources_via_ps', Mock(return_value = (105.3, 2.4, 8072, 0.3))) + @patch('arm.util.tracker._resources_via_proc', Mock(side_effect = IOError())) + @patch('arm.util.tracker.system', Mock(return_value = Mock())) + @patch('arm.util.tracker.proc.is_available', Mock(return_value = True)) + def test_failing_over_to_ps(self, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + + with ResourceTracker(0.01) as daemon: + time.sleep(0.03) + + self.assertEqual(True, daemon._use_proc) + resources = daemon.get_resource_usage() + + self.assertEqual(0, daemon.run_counter()) + self.assertEqual(0.0, resources.cpu_sample) + self.assertEqual(0.0, resources.cpu_average) + self.assertEqual(0, resources.cpu_total) + self.assertEqual(0, resources.memory_bytes) + self.assertEqual(0.0, resources.memory_percent) + self.assertEqual(0.0, resources.timestamp) + + time.sleep(0.05) + self.assertEqual(False, daemon._use_proc) + + resources = daemon.get_resource_usage() + + self.assertEqual(1, daemon.run_counter()) + self.assertEqual(0.0, resources.cpu_sample) + self.assertEqual(43.875, resources.cpu_average) + self.assertEqual(105.3, resources.cpu_total) + self.assertEqual(8072, resources.memory_bytes) + self.assertEqual(0.3, resources.memory_percent) + self.assertTrue((time.time() - resources.timestamp) < 0.5) + + @patch('arm.util.tracker.system.call', Mock(return_value = PS_OUTPUT.split('\n'))) + def test_resources_via_ps(self): + total_cpu_time, uptime, memory_in_bytes, memory_in_percent = _resources_via_ps(12345) + + self.assertEqual(2.0, total_cpu_time) + self.assertEqual(18, uptime) + self.assertEqual(19300352, memory_in_bytes) + self.assertEqual(0.004, memory_in_percent) + + @patch('time.time', Mock(return_value = 1388967218.973117)) + @patch('arm.util.tracker.proc.get_stats', Mock(return_value = (1.5, 0.5, 1388967200.9))) + @patch('arm.util.tracker.proc.get_memory_usage', Mock(return_value = (19300352, 6432))) + @patch('arm.util.tracker.proc.get_physical_memory', Mock(return_value = 4825088000)) + def test_resources_via_proc(self): + total_cpu_time, uptime, memory_in_bytes, memory_in_percent = _resources_via_proc(12345) + + self.assertEqual(2.0, total_cpu_time) + self.assertEqual(18, int(uptime)) + self.assertEqual(19300352, memory_in_bytes) + self.assertEqual(0.004, memory_in_percent)

1 0

[arm/master] Testing the ConnectionTracker class
by atagar＠torproject.org 05 Jan '14

05 Jan '14

commit 6f2ce6c6d1004fd69d8ba1038f08c1376cdf2625 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 5 17:55:17 2014 -0800 Testing the ConnectionTracker class Unit tests for our connection resolver daemon. There's a lot more tests we could add for the tracker module, but this gets us pretty good coverage for now. --- arm/util/tracker.py | 2 +- test/util/tracker/connection_tracker.py | 76 +++++++++++++++++++++++++++++++ test/util/tracker/resource_tracker.py | 5 +- 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index b0f99b2..6c5b1f4 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -384,7 +384,7 @@ class ConnectionTracker(Daemon): self._failure_count += 1 if self._failure_count >= 3: - self._resolvers.pop() + self._resolvers.pop(0) self._failure_count = 0 if self._resolvers: diff --git a/test/util/tracker/connection_tracker.py b/test/util/tracker/connection_tracker.py new file mode 100644 index 0000000..bdb1d69 --- /dev/null +++ b/test/util/tracker/connection_tracker.py @@ -0,0 +1,76 @@ +import time +import unittest + +from arm.util.tracker import ConnectionTracker + +from stem.util import connection + +from mock import Mock, patch + +CONNECTION_1 = connection.Connection('127.0.0.1', 3531, '75.119.206.243', 22, 'tcp') +CONNECTION_2 = connection.Connection('127.0.0.1', 1766, '86.59.30.40', 443, 'tcp') +CONNECTION_3 = connection.Connection('127.0.0.1', 1059, '74.125.28.106', 80, 'tcp') + + +class TestConnectionTracker(unittest.TestCase): + @patch('arm.util.tracker.tor_controller') + @patch('arm.util.tracker.connection.get_connections') + @patch('arm.util.tracker.system', Mock(return_value = Mock())) + @patch('arm.util.tracker.connection.get_system_resolvers', Mock(return_value = [connection.Resolver.NETSTAT])) + def test_fetching_connections(self, get_connections_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + get_connections_mock.return_value = [CONNECTION_1, CONNECTION_2, CONNECTION_3] + + with ConnectionTracker(0.04) as daemon: + time.sleep(0.01) + + connections = daemon.get_connections() + + self.assertEqual(1, daemon.run_counter()) + self.assertEqual([CONNECTION_1, CONNECTION_2, CONNECTION_3], connections) + + get_connections_mock.return_value = [] # no connection results + time.sleep(0.05) + connections = daemon.get_connections() + + self.assertEqual(2, daemon.run_counter()) + self.assertEqual([], connections) + + @patch('arm.util.tracker.tor_controller') + @patch('arm.util.tracker.connection.get_connections') + @patch('arm.util.tracker.system', Mock(return_value = Mock())) + @patch('arm.util.tracker.connection.get_system_resolvers', Mock(return_value = [connection.Resolver.NETSTAT, connection.Resolver.LSOF])) + def test_resolver_failover(self, get_connections_mock, tor_controller_mock): + tor_controller_mock().get_pid.return_value = 12345 + get_connections_mock.side_effect = IOError() + + with ConnectionTracker(0.01) as daemon: + time.sleep(0.03) + + self.assertEqual([connection.Resolver.NETSTAT, connection.Resolver.LSOF], daemon._resolvers) + self.assertEqual([], daemon.get_connections()) + + time.sleep(0.05) + + self.assertEqual([connection.Resolver.LSOF], daemon._resolvers) + self.assertEqual([], daemon.get_connections()) + + time.sleep(0.05) + + self.assertEqual([], daemon._resolvers) + self.assertEqual([], daemon.get_connections()) + + # Now make connection resolution work. We still shouldn't provide any + # results since we stopped looking. + + get_connections_mock.return_value = [CONNECTION_1, CONNECTION_2] + get_connections_mock.side_effect = None + time.sleep(0.05) + self.assertEqual([], daemon.get_connections()) + + # Finally, select a custom resolver. This should cause us to query again + # reguardless of our prior failures. + + daemon.set_custom_resolver(connection.Resolver.NETSTAT) + time.sleep(0.05) + self.assertEqual([CONNECTION_1, CONNECTION_2], daemon.get_connections()) diff --git a/test/util/tracker/resource_tracker.py b/test/util/tracker/resource_tracker.py index 70cdf27..41ef80e 100644 --- a/test/util/tracker/resource_tracker.py +++ b/test/util/tracker/resource_tracker.py @@ -107,12 +107,13 @@ class TestResourceTracker(unittest.TestCase): self.assertEqual(0.0, resources.memory_percent) self.assertEqual(0.0, resources.timestamp) - time.sleep(0.05) + while daemon.run_counter() < 1: + time.sleep(0.01) + self.assertEqual(False, daemon._use_proc) resources = daemon.get_resource_usage() - self.assertEqual(1, daemon.run_counter()) self.assertEqual(0.0, resources.cpu_sample) self.assertEqual(43.875, resources.cpu_average) self.assertEqual(105.3, resources.cpu_total)

1 0

[arm/master] Test Daemon's general functionality
by atagar＠torproject.org 05 Jan '14

05 Jan '14

commit 2269d007c5ba9ba1207a49d1440e33284e57581d Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 5 15:09:27 2014 -0800 Test Daemon's general functionality Check that the Daemon's pause method and that it executes the given task at a set rate. --- arm/util/tracker.py | 12 ++++++++++-- test/util/tracker/daemon.py | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index 8febb12..1a716d3 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -218,7 +218,7 @@ class Daemon(threading.Thread): time_since_last_ran = time.time() - self._last_ran if self._is_paused or time_since_last_ran < self._rate: - sleep_duration = max(0.2, self._rate - time_since_last_ran) + sleep_duration = max(0.02, self._rate - time_since_last_ran) with self._pause_condition: if not self._halt: @@ -248,7 +248,7 @@ class Daemon(threading.Thread): :returns: **bool** indicating if our run was successful or not """ - pass + return True def run_counter(self): """ @@ -306,6 +306,14 @@ class Daemon(threading.Thread): self._process_pid = tor_pid self._process_name = tor_cmd if tor_cmd else 'tor' + def __enter__(self): + self.start() + return self + + def __exit__(self, exit_type, value, traceback): + self.stop() + self.join() + class ConnectionTracker(Daemon): """ diff --git a/test/util/tracker/daemon.py b/test/util/tracker/daemon.py index 9bde71f..9d409e2 100644 --- a/test/util/tracker/daemon.py +++ b/test/util/tracker/daemon.py @@ -1,8 +1,9 @@ +import time import unittest from arm.util.tracker import Daemon -from mock import patch +from mock import Mock, patch class TestDaemon(unittest.TestCase): @@ -17,7 +18,7 @@ class TestDaemon(unittest.TestCase): daemon = Daemon(0.05) - self.assertEqual(0.05, daemon._rate) + self.assertEqual(0.05, daemon.get_rate()) self.assertEqual(12345, daemon._process_pid) self.assertEqual('local_tor', daemon._process_name) @@ -45,3 +46,31 @@ class TestDaemon(unittest.TestCase): daemon = Daemon(0.05) self.assertEqual(None, daemon._process_pid) self.assertEqual('tor', daemon._process_name) + self.assertEqual(0, system_mock.call_count) + + @patch('arm.util.tracker.tor_controller', Mock(return_value = Mock())) + @patch('arm.util.tracker.system', Mock(return_value = Mock())) + def test_daemon_calls_task(self): + # Check that our Daemon calls the task method at the given rate. + + with Daemon(0.01) as daemon: + time.sleep(0.05) + self.assertTrue(2 < daemon.run_counter()) + + @patch('arm.util.tracker.tor_controller', Mock(return_value = Mock())) + @patch('arm.util.tracker.system', Mock(return_value = Mock())) + def test_pausing_daemon(self): + # Check that we can pause and unpause daemon. + + with Daemon(0.01) as daemon: + time.sleep(0.05) + self.assertTrue(2 < daemon.run_counter()) + + daemon.set_paused(True) + daemon._run_counter = 0 + time.sleep(0.05) + self.assertEqual(0, daemon.run_counter()) + + daemon.set_paused(False) + time.sleep(0.05) + self.assertTrue(2 < daemon.run_counter())

1 0

[ooni-probe/master] Case sensitivity is actually important.
by art＠torproject.org 05 Jan '14

05 Jan '14

commit 0fb6b6c020c9bd2739073e23dc0fddfe796fa38f Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jan 6 00:51:51 2014 +0100 Case sensitivity is actually important. --- data/ooniprobe.conf.sample | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index 8a4e0f7..412923a 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -21,7 +21,7 @@ reports: pcap: null collector: 'httpo://nkvphnp3p6agi5qq.onion' advanced: - geoip_data_dir: /usr/share/geoip + geoip_data_dir: /usr/share/GeoIP debug: false # enable if auto detection fails #tor_binary: /usr/sbin/tor diff --git a/setup.py b/setup.py index f761f30..bd9ebcf 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ if hasattr(sys, 'real_prefix'): )] else: data_files = [( - '/usr/share/geoip/', + '/usr/share/GeoIP/', [ 'data/GeoIP.dat', 'data/GeoIPASNum.dat',

1 0

[ooni-probe/master] Refactor naming of nettest related code
by art＠torproject.org 05 Jan '14

05 Jan '14

commit cc9fbb70666cf5322956332415dcdbf8cd3592d1 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jan 6 00:44:44 2014 +0100 Refactor naming of nettest related code --- ooni/nettest.py | 12 ++++++------ ooni/tasks.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index a1a46d6..513c11f 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -514,7 +514,7 @@ class NetTest(object): return report_results - def makeMeasurement(self, test_class, test_method, test_input=None): + def makeMeasurement(self, test_instance, test_method, test_input=None): """ Creates a new instance of :class:ooni.tasks.Measurement and add's it's callbacks and errbacks. @@ -531,7 +531,7 @@ class NetTest(object): NetTestCase """ - measurement = Measurement(test_class, test_method, test_input) + measurement = Measurement(test_instance, test_method, test_input) measurement.netTest = self if self.director: @@ -558,10 +558,10 @@ class NetTest(object): # load the input processor as late as possible for input in test_class.inputs: measurements = [] + test_instance = test_class() for method in test_methods: - klass = test_class() log.debug("Running %s %s" % (test_class, method)) - measurement = self.makeMeasurement(klass, method, input) + measurement = self.makeMeasurement(test_instance, method, input) measurements.append(measurement.done) self.state.taskCreated() yield measurement @@ -573,11 +573,11 @@ class NetTest(object): # Call the postProcessor, which must return a single report # or a deferred - post.addCallback(klass.postProcessor) + post.addCallback(test_instance.postProcessor) def noPostProcessor(failure, report): failure.trap(e.NoPostProcessor) return report - post.addErrback(noPostProcessor, klass.report) + post.addErrback(noPostProcessor, test_instance.report) post.addCallback(self.report.write) if self.report and self.director: diff --git a/ooni/tasks.py b/ooni/tasks.py index 9e74dd5..3e378e9 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -91,7 +91,7 @@ class TaskWithTimeout(BaseTask): return BaseTask.start(self) class Measurement(TaskWithTimeout): - def __init__(self, test_class, test_method, test_input): + def __init__(self, test_instance, test_method, test_input): """ test_class: is the class, subclass of NetTestCase, of the test to be run @@ -106,7 +106,7 @@ class Measurement(TaskWithTimeout): net_test: a reference to the net_test object such measurement belongs to. """ - self.testInstance = test_class + self.testInstance = test_instance self.testInstance.input = test_input if 'input' not in self.testInstance.report.keys(): self.testInstance.report = {'input': test_input}

1 0

[ooni-probe/master] request_line_capitalization is a lie, remove it.
by art＠torproject.org 05 Jan '14

05 Jan '14

commit 9b89f699208cb19be726519bc79b4132c3fa44c1 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jan 6 00:45:38 2014 +0100 request_line_capitalization is a lie, remove it. --- ooni/nettests/manipulation/http_header_field_manipulation.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/ooni/nettests/manipulation/http_header_field_manipulation.py b/ooni/nettests/manipulation/http_header_field_manipulation.py index e059c42..3fdcc5f 100644 --- a/ooni/nettests/manipulation/http_header_field_manipulation.py +++ b/ooni/nettests/manipulation/http_header_field_manipulation.py @@ -100,9 +100,6 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): * **total** when the response is not a json object and therefore we were not able to reach the ooniprobe test backend - * **request_line_capitalization** when the HTTP Request line (e.x. GET / - HTTP/1.1) does not match the capitalization we set. - * **header_field_number** when the number of headers we sent does not match with the ones the backend received @@ -116,7 +113,6 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): self.report['tampering'] = { 'total': False, - 'request_line_capitalization': False, 'header_name_capitalization': False, 'header_field_value': False, 'header_field_number': False @@ -136,9 +132,6 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): self.report['tampering']['total'] = True return - if request_request_line != response_request_line: - self.report['tampering']['request_line_capitalization'] = True - request_headers = TrueHeaders(self.request_headers) diff = request_headers.getDiff(TrueHeaders(response_headers_dict), ignore=['Connection']) @@ -148,7 +141,6 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): self.report['tampering']['header_field_name'] = False self.report['tampering']['header_name_diff'] = list(diff) log.msg(" total: %(total)s" % self.report['tampering']) - log.msg(" request_line_capitalization: %(request_line_capitalization)s" % self.report['tampering']) log.msg(" header_name_capitalization: %(header_name_capitalization)s" % self.report['tampering']) log.msg(" header_field_value: %(header_field_value)s" % self.report['tampering']) log.msg(" header_field_number: %(header_field_number)s" % self.report['tampering']) @@ -188,4 +180,3 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): self.request_headers = self.get_random_caps_headers() return self.doRequest(self.url, self.request_method, headers=self.request_headers) -

1 0

[ooni-probe/master] Fix a bug inside of HTTP test template
by art＠torproject.org 05 Jan '14

05 Jan '14

commit 98bb2cf5b556d54fd68859928c7e5bc4882a205d Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jan 6 00:22:20 2014 +0100 Fix a bug inside of HTTP test template --- ooni/templates/httpt.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index fdcbc3e..7c54782 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -232,13 +232,9 @@ class HTTPTest(NetTestCase): try: content_length = int(response.headers.getRawHeaders('content-length')[0]) - except IndexError: + except Exception: content_length = None - if not content_length: - self._processResponseBody(None, request, response, None) - return defer.succeed(None) - finished = defer.Deferred() response.deliverBody(BodyReceiver(finished, content_length)) finished.addCallback(self._processResponseBody, request,

1 0

[ooni-probe/master] Create a new class instance per test method
by art＠torproject.org 05 Jan '14

05 Jan '14

commit bff27aba551e64d429ef552fd0559ac859c5302c Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jan 6 00:21:36 2014 +0100 Create a new class instance per test method * This makes sure the class attributes are not shared across test methods --- ooni/nettest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 7e1ef04..a1a46d6 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -557,9 +557,9 @@ class NetTest(object): for test_class, test_methods in self.testCases: # load the input processor as late as possible for input in test_class.inputs: - klass = test_class() measurements = [] for method in test_methods: + klass = test_class() log.debug("Running %s %s" % (test_class, method)) measurement = self.makeMeasurement(klass, method, input) measurements.append(measurement.done)

1 0

[stem/master] Style issues suppressed pyflakes results
by atagar＠torproject.org 05 Jan '14

05 Jan '14

commit 46bfcad588b7e41cff109d72ea7a88717beb2c00 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 5 11:05:27 2014 -0800 Style issues suppressed pyflakes results When both pyflakes and pep8 reported issues about a file the pep8 results clobbered the pyflakes output. Properly combining the two instead. --- run_tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/run_tests.py b/run_tests.py index 664df35..90493da 100755 --- a/run_tests.py +++ b/run_tests.py @@ -293,12 +293,16 @@ def main(): static_check_issues = {} if pyflakes_task and pyflakes_task.is_successful: - static_check_issues.update(pyflakes_task.result) + for path, issues in pyflakes_task.result.items(): + for issue in issues: + static_check_issues.setdefault(path, []).append(issue) elif not test.util.is_pyflakes_available(): println("Static error checking requires pyflakes version 0.7.3 or later. Please install it from ...\n http://pypi.python.org/pypi/pyflakes\n", ERROR) if pep8_task and pep8_task.is_successful: - static_check_issues.update(pep8_task.result) + for path, issues in pep8_task.result.items(): + for issue in issues: + static_check_issues.setdefault(path, []).append(issue) elif not test.util.is_pep8_available(): println("Style checks require pep8 version 1.4.2 or later. Please install it from...\n http://pypi.python.org/pypi/pep8\n", ERROR)

1 0

[arm/master] Initial Daemon unit tests
by atagar＠torproject.org 05 Jan '14

05 Jan '14

commit 511c7ac98f13bf7b0817ab759d331a65424472f6 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 5 14:38:28 2014 -0800 Initial Daemon unit tests Few simple tests for its initialization. Further testing for the Daemon and subclasses in the tracker module coming soon. As for the load_test() test, it's both for a helper that no longer exists and the test wasn't being run due to not having an __init__.py. --- arm/util/tracker.py | 3 ++- run_tests.py | 8 +++++-- test/starter/load_settings.py | 31 --------------------------- test/util/tracker/daemon.py | 47 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 34 deletions(-) diff --git a/arm/util/tracker.py b/arm/util/tracker.py index 476b11e..8febb12 100644 --- a/arm/util/tracker.py +++ b/arm/util/tracker.py @@ -1,5 +1,5 @@ """ -Background tasks for gathering informatino about the tor process. +Background tasks for gathering information about the tor process. :: @@ -108,6 +108,7 @@ def stop_trackers(): tracker.join() halt_thread = threading.Thread(target = halt_trackers) + halt_thread.setDaemon(True) halt_thread.start() return halt_thread diff --git a/run_tests.py b/run_tests.py index 324ed02..1553da8 100755 --- a/run_tests.py +++ b/run_tests.py @@ -48,10 +48,14 @@ def main(): static_check_issues = {} if is_pyflakes_available(): - static_check_issues.update(get_pyflakes_issues(SRC_PATHS)) + for path, issues in get_pyflakes_issues(SRC_PATHS).items(): + for issue in issues: + static_check_issues.setdefault(path, []).append(issue) if is_pep8_available(): - static_check_issues.update(get_stylistic_issues(SRC_PATHS)) + for path, issues in get_stylistic_issues(SRC_PATHS).items(): + for issue in issues: + static_check_issues.setdefault(path, []).append(issue) if static_check_issues: print "STATIC CHECKS" diff --git a/test/starter/load_settings.py b/test/starter/load_settings.py deleted file mode 100644 index 5c9cfd8..0000000 --- a/test/starter/load_settings.py +++ /dev/null @@ -1,31 +0,0 @@ -import io -import unittest - -from mock import patch - -from arm.starter import _load_settings - - -class TestLoadSettings(unittest.TestCase): - def test_we_can_load_the_settings(self): - config = _load_settings(self.id()) - self.assertEqual(config.get('settings_loaded'), 'true') - - @patch('stem.util.conf.open', create = True) - def test_when_file_doesnt_exist(self, open_mock): - open_mock.side_effect = IOError("No such file or directory") - - try: - _load_settings(self.id()) - self.fail("We didn't raise an exception for a missing settings.cfg") - except ValueError as exc: - self.assertTrue("Unable to load arm's internal configuration" in str(exc)) - - @patch('stem.util.conf.open', create = True) - def test_that_repeated_calls_are_ignored(self, open_mock): - open_mock.return_value = io.BytesIO("settings_loaded true") - - _load_settings(self.id()) - _load_settings(self.id()) - _load_settings(self.id()) - self.assertEqual(1, open_mock.call_count) diff --git a/test/util/tracker/__init__.py b/test/util/tracker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/util/tracker/daemon.py b/test/util/tracker/daemon.py new file mode 100644 index 0000000..9bde71f --- /dev/null +++ b/test/util/tracker/daemon.py @@ -0,0 +1,47 @@ +import unittest + +from arm.util.tracker import Daemon + +from mock import patch + + +class TestDaemon(unittest.TestCase): + @patch('arm.util.tracker.tor_controller') + @patch('arm.util.tracker.system') + def test_init(self, system_mock, tor_controller_mock): + # Check that we register ourselves to listen for status changes, and + # properly retrieve the process' pid and name. + + tor_controller_mock().get_pid.return_value = 12345 + system_mock.get_name_by_pid.return_value = 'local_tor' + + daemon = Daemon(0.05) + + self.assertEqual(0.05, daemon._rate) + self.assertEqual(12345, daemon._process_pid) + self.assertEqual('local_tor', daemon._process_name) + + tor_controller_mock().add_status_listener.assert_called_with(daemon._tor_status_listener) + system_mock.get_name_by_pid.assert_called_with(12345) + + @patch('arm.util.tracker.tor_controller') + @patch('arm.util.tracker.system') + def test_init_without_name(self, system_mock, tor_controller_mock): + # Check when we default to 'tor' if unable to determine the process' name. + + tor_controller_mock().get_pid.return_value = 12345 + system_mock.get_name_by_pid.return_value = None + + daemon = Daemon(0.05) + self.assertEqual('tor', daemon._process_name) + + @patch('arm.util.tracker.tor_controller') + @patch('arm.util.tracker.system') + def test_init_without_pid(self, system_mock, tor_controller_mock): + # Check when we can't determine tor's pid. + + tor_controller_mock().get_pid.return_value = None + + daemon = Daemon(0.05) + self.assertEqual(None, daemon._process_pid) + self.assertEqual('tor', daemon._process_name)

1 0