commit 8b776c49a77e5ec73eaddf2572cea893859f3100 Author: Arturo Filastò art@fuffa.org Date: Thu Dec 27 14:53:05 2012 +0100

Clean up output of ooniprobe * Fix bug in http_requests body length comparison --- nettests/blocking/http_requests.py | 23 ++++++++++++++++--- .../manipulation/http_header_field_manipulation.py | 7 ++++++ ooni/oonicli.py | 4 +- ooni/runner.py | 4 +- ooni/templates/httpt.py | 2 +- ooni/utils/txagentwithsocks.py | 3 ++ 6 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/nettests/blocking/http_requests.py b/nettests/blocking/http_requests.py index bc2803b..34f5d04 100644 --- a/nettests/blocking/http_requests.py +++ b/nettests/blocking/http_requests.py @@ -5,6 +5,7 @@

from twisted.internet import defer from twisted.python import usage +from ooni.utils import log from ooni.templates import httpt

class UsageOptions(usage.Options): @@ -53,22 +54,34 @@ class HTTPRequestsTest(httpt.HTTPTest): body_length_a = self.control_body_length body_length_b = self.experiment_body_length

- rel = float(body_length_a)/float(body_length_b) + if body_length_b == 0 and body_length_a != 0: + rel = float(body_length_b)/float(body_length_a) + elif body_length_b == 0 and body_length_a == 0: + rel = float(1) + else: + rel = float(body_length_a)/float(body_length_b) + if rel > 1: rel = 1/rel

self.report['body_proportion'] = rel self.report['factor'] = self.factor - if rel < self.factor: + if rel > self.factor: + log.msg("The two body lengths appear to match") + log.msg("censorship is probably not happening") self.report['body_length_match'] = True else: + log.msg("The two body lengths appear to not match") + log.msg("censorship could be happening") self.report['body_length_match'] = False

def compare_headers(self): diff = TrueHeaders(self.control_headers).getDiff(self.experiment_headers) if diff: + log.msg("Headers appear to match") self.report['headers_match'] = False else: + log.msg("Headers appear to *not* match") self.report['headers_match'] = True

def test_get(self): @@ -81,7 +94,7 @@ class HTTPRequestsTest(httpt.HTTPTest): Callback for processing the control HTTP body response. """ self.control_body_length = len(result) - if self.experiment_body_length: + if self.experiment_body_length is not None: self.compare_body_lengths()

def experiment_body(result): @@ -89,7 +102,7 @@ class HTTPRequestsTest(httpt.HTTPTest): Callback for processing the experiment HTTP body response. """ self.experiment_body_length = len(result) - if self.control_body_length: + if self.control_body_length is not None: self.compare_body_lengths()

def control_headers(headers_dict): @@ -105,10 +118,12 @@ class HTTPRequestsTest(httpt.HTTPTest): self.experiment_headers = headers_dict

dl = [] + log.msg("Performing GET request to %s" % self.url) experiment_request = self.doRequest(self.url, method="GET", body_processor=experiment_body, headers_processor=control_headers)

+ log.msg("Performing GET request to %s via Tor" % self.url) control_request = self.doRequest(self.url, method="GET", use_tor=True, body_processor=control_body, headers_processor=control_headers) diff --git a/nettests/manipulation/http_header_field_manipulation.py b/nettests/manipulation/http_header_field_manipulation.py index 0140638..e6e3fde 100644 --- a/nettests/manipulation/http_header_field_manipulation.py +++ b/nettests/manipulation/http_header_field_manipulation.py @@ -110,6 +110,8 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): * **header_field_value** when the header field value does not match with the one we transmitted. """ + log.msg("Checking for tampering on %s" % self.url) + self.report['tampering'] = { 'total': False, 'request_line_capitalization': False, @@ -143,6 +145,11 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): else: self.report['tampering']['header_field_name'] = False self.report['tampering']['header_name_diff'] = list(diff) + log.msg(" total: %(total)s" % self.report['tampering']) + log.msg(" request_line_capitalization: %(request_line_capitalization)s" % self.report['tampering']) + log.msg(" header_name_capitalization: %(header_name_capitalization)s" % self.report['tampering']) + log.msg(" header_field_value: %(header_field_value)s" % self.report['tampering']) + log.msg(" header_field_number: %(header_field_number)s" % self.report['tampering'])

def test_get(self): self.request_method = "GET" diff --git a/ooni/oonicli.py b/ooni/oonicli.py index c333b27..c18f3bd 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -78,7 +78,7 @@ def updateStatusBar(): eta = config.state[test_filename].eta() progress = config.state[test_filename].progress() progress_bar_frmt = "[%s] %s%%" % (test_filename, progress) - print progress_bar_frmt + log.debug(progress_bar_frmt)

def testsEnded(*arg, **kw): """ @@ -158,7 +158,7 @@ def run(): cmd_line_options['resume'] = False test_list.append(runner.loadTest(cmd_line_options)) else: - log.msg("No test deck detected") + log.debug("No test deck detected") del cmd_line_options['testdeck'] test_list.append(runner.loadTest(cmd_line_options))

diff --git a/ooni/runner.py b/ooni/runner.py index 30a627c..be8c968 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -218,7 +218,7 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, test_class = test_case[0] test_method = test_case[1]

- log.msg("Running %s with %s..." % (test_method, test_input)) + log.debug("Running %s with %s..." % (test_method, test_input))

test_instance = test_class() test_instance.input = test_input @@ -511,7 +511,7 @@ def startTor(): return state.post_bootstrap

def updates(prog, tag, summary): - log.msg("%d%%: %s" % (prog, summary)) + log.debug("%d%%: %s" % (prog, summary))

tor_config = TorConfig() if config.tor.control_port: diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index aecae41..c61b5ff 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -276,7 +276,7 @@ class HTTPTest(NetTestCase): log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy'])) url = 's'+url

- log.msg("Performing request %s %s %s" % (url, method, headers)) + log.debug("Performing request %s %s %s" % (url, method, headers))

request = {} request['method'] = method diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py index af46b0f..830734e 100644 --- a/ooni/utils/txagentwithsocks.py +++ b/ooni/utils/txagentwithsocks.py @@ -17,6 +17,8 @@ from twisted.internet import interfaces, defer from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred

from txsocksx.client import SOCKS5ClientEndpoint +from txsocksx.client import SOCKS5ClientFactory +SOCKS5ClientFactory.noisy = False

from ooni.utils import log

@@ -138,6 +140,7 @@ class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol): return self._finishedRequest

class _HTTP11ClientFactory(client._HTTP11ClientFactory): + noisy = False def buildProtocol(self, addr): return HTTP11ClientProtocol(self._quiescentCallback)