February 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Refactor HTTP Requests test
by art＠torproject.org 26 Feb '13

26 Feb '13

commit becee5fe855d13fe46f72fe77648d68641bca4fc Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Feb 8 17:38:54 2013 +0100 Refactor HTTP Requests test Fix bug where response headers where not getting compared --- nettests/blocking/http_requests.py | 64 +++++++++++++---------------------- ooni/templates/httpt.py | 2 + 2 files changed, 26 insertions(+), 40 deletions(-) diff --git a/nettests/blocking/http_requests.py b/nettests/blocking/http_requests.py index aedb1b8..56e5944 100644 --- a/nettests/blocking/http_requests.py +++ b/nettests/blocking/http_requests.py @@ -25,7 +25,7 @@ class HTTPRequestsTest(httpt.HTTPTest): """ name = "HTTP Requests Test" author = "Arturo Filastò" - version = "0.1" + version = "0.2" usageOptions = UsageOptions @@ -50,9 +50,7 @@ class HTTPRequestsTest(httpt.HTTPTest): self.factor = self.localOptions['factor'] - def compare_body_lengths(self): - body_length_a = self.control_body_length - body_length_b = self.experiment_body_length + def compare_body_lengths(self, body_length_a, body_length_b): if body_length_b == 0 and body_length_a != 0: rel = float(body_length_b)/float(body_length_a) @@ -75,13 +73,15 @@ class HTTPRequestsTest(httpt.HTTPTest): log.msg("censorship could be happening") self.report['body_length_match'] = False - def compare_headers(self): - diff = TrueHeaders(self.control_headers).getDiff(self.experiment_headers) + def compare_headers(self, headers_a, headers_b): + diff = headers_a.getDiff(headers_b) if diff: - log.msg("Headers appear to match") + log.msg("Headers appear to *not* match") + self.report['headers_diff'] = diff self.report['headers_match'] = False else: - log.msg("Headers appear to *not* match") + log.msg("Headers appear to match") + self.report['headers_diff'] = diff self.report['headers_match'] = True def test_get(self): @@ -89,47 +89,31 @@ class HTTPRequestsTest(httpt.HTTPTest): log.err("There was an error while testing %s" % self.url) log.exception(failure) - def control_body(result): - """ - Callback for processing the control HTTP body response. - """ - self.control_body_length = len(result) - if self.experiment_body_length is not None: - self.compare_body_lengths() - - def experiment_body(result): - """ - Callback for processing the experiment HTTP body response. - """ - self.experiment_body_length = len(result) - if self.control_body_length is not None: - self.compare_body_lengths() - - def control_headers(headers_dict): - """ - Callback for processing the control HTTP headers response. - """ - self.control_headers = headers_dict - - def experiment_headers(headers_dict): - """ - Callback for processing the experiment HTTP headers response. - """ - self.experiment_headers = headers_dict + def callback(res): + experiment, control = res + experiment_success, experiment_response = experiment + control_success, control_response = control + + self.compare_body_lengths(len(experiment_response.body), + len(control_response.body)) + + self.compare_headers(control_response.headers, + experiment_response.headers) l = [] log.msg("Performing GET request to %s" % self.url) - experiment_request = self.doRequest(self.url, method="GET", - body_processor=experiment_body, - headers_processor=control_headers) + experiment_request = self.doRequest(self.url, method="GET") log.msg("Performing GET request to %s via Tor" % self.url) control_request = self.doRequest(self.url, method="GET", - use_tor=True, body_processor=control_body, - headers_processor=control_headers) + use_tor=True) l.append(experiment_request) l.append(control_request) + dl = defer.DeferredList(l) + dl.addCallback(callback) + dl.addErrback(errback) + return dl diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index b506d77..f7d5441 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -139,6 +139,8 @@ class HTTPTest(NetTestCase): body_processor(response_body) else: self.processResponseBody(response_body) + response.body = response_body + return response def processResponseBody(self, body): """

1 0

[ooni-probe/master] In case of failing requests write control_failure and experiment_failure report keys
by art＠torproject.org 26 Feb '13

26 Feb '13

commit f8b13b0b9cb1fd453c77a887726318b04ffe148e Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Feb 8 20:21:49 2013 +0100 In case of failing requests write control_failure and experiment_failure report keys These keys will contain the error strings for errors that have occurred while performing the HTTP request. --- nettests/blocking/http_requests.py | 30 +++++++++++++++++------------- 1 files changed, 17 insertions(+), 13 deletions(-) diff --git a/nettests/blocking/http_requests.py b/nettests/blocking/http_requests.py index 56e5944..b6bab00 100644 --- a/nettests/blocking/http_requests.py +++ b/nettests/blocking/http_requests.py @@ -5,8 +5,10 @@ from twisted.internet import defer from twisted.python import usage + from ooni.utils import log from ooni.templates import httpt +from ooni.nettest import failureToString, handleAllFailures class UsageOptions(usage.Options): optParameters = [ @@ -25,7 +27,7 @@ class HTTPRequestsTest(httpt.HTTPTest): """ name = "HTTP Requests Test" author = "Arturo Filastò" - version = "0.2" + version = "0.2.2" usageOptions = UsageOptions @@ -85,20 +87,23 @@ class HTTPRequestsTest(httpt.HTTPTest): self.report['headers_match'] = True def test_get(self): - def errback(failure): - log.err("There was an error while testing %s" % self.url) - log.exception(failure) - def callback(res): experiment, control = res - experiment_success, experiment_response = experiment - control_success, control_response = control + experiment_succeeded, experiment_result = experiment + control_succeeded, control_result = control + + if control_succeeded and experiment_succeeded: + self.compare_body_lengths(len(experiment_result.body), + len(control_result.body)) + + self.compare_headers(control_result.headers, + experiment_result.headers) - self.compare_body_lengths(len(experiment_response.body), - len(control_response.body)) + if not control_succeeded: + self.report['control_failure'] = failureToString(control_result) - self.compare_headers(control_response.headers, - experiment_response.headers) + if not experiment_succeeded: + self.report['experiment_failure'] = failureToString(experiment_result) l = [] log.msg("Performing GET request to %s" % self.url) @@ -111,9 +116,8 @@ class HTTPRequestsTest(httpt.HTTPTest): l.append(experiment_request) l.append(control_request) - dl = defer.DeferredList(l) + dl = defer.DeferredList(l, consumeErrors=True) dl.addCallback(callback) - dl.addErrback(errback) return dl

1 0

[ooni-probe/master] Make the user agent selection in the HTTP Requests test be static
by art＠torproject.org 26 Feb '13

26 Feb '13

commit 185867db9f33446d6c6f0e2cc3acffa4a4a1f7d4 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Feb 8 22:43:29 2013 +0100 Make the user agent selection in the HTTP Requests test be static This is to avoid false positives due to user agent based targeted content --- nettests/blocking/http_requests.py | 13 +++++++++---- 1 files changed, 9 insertions(+), 4 deletions(-) diff --git a/nettests/blocking/http_requests.py b/nettests/blocking/http_requests.py index ef2cece..4d99fc3 100644 --- a/nettests/blocking/http_requests.py +++ b/nettests/blocking/http_requests.py @@ -3,10 +3,12 @@ # :authors: Arturo Filastò # :licence: see LICENSE +import random from twisted.internet import defer from twisted.python import usage from ooni.utils import log +from ooni.utils.net import userAgents from ooni.templates import httpt from ooni.nettest import failureToString, handleAllFailures @@ -27,11 +29,11 @@ class HTTPRequestsTest(httpt.HTTPTest): """ name = "HTTP Requests Test" author = "Arturo Filastò" - version = "0.2.2" + version = "0.2.3" usageOptions = UsageOptions - inputFile = ['file', 'f', None, + inputFile = ['file', 'f', None, 'List of URLS to perform GET and POST requests to'] # These values are used for determining censorship based on response body @@ -107,13 +109,16 @@ class HTTPRequestsTest(httpt.HTTPTest): if not experiment_succeeded: self.report['experiment_failure'] = failureToString(experiment_result) + headers = {'User-Agent': [random.choice(userAgents)]} + l = [] log.msg("Performing GET request to %s" % self.url) - experiment_request = self.doRequest(self.url, method="GET") + experiment_request = self.doRequest(self.url, method="GET", + headers=headers) log.msg("Performing GET request to %s via Tor" % self.url) control_request = self.doRequest(self.url, method="GET", - use_tor=True) + use_tor=True, headers=headers) l.append(experiment_request) l.append(control_request)

1 0

[ooni-probe/master] Merge pull request #37 from hellais/http_requests_refactor
by art＠torproject.org 26 Feb '13

26 Feb '13

commit e746f6263dd88fdde408809dfa93af5b131ca558 Merge: 5f568cb 66cbb71 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Feb 26 07:13:47 2013 -0800 Merge pull request #37 from hellais/http_requests_refactor Refactor HTTP Requests test nettests/blocking/http_requests.py | 85 +++++++++---------- .../manipulation/http_header_field_manipulation.py | 2 +- ooni/templates/httpt.py | 6 +- ooni/utils/net.py | 27 +++--- 4 files changed, 59 insertions(+), 61 deletions(-)

1 0

[ooni-probe/master] Set the default value of the failure keys in the HTTP request test
by art＠torproject.org 26 Feb '13

26 Feb '13

commit eace1caba4122b9c8b4d0e9d30808f0ddb9e8463 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Feb 8 21:53:49 2013 +0100 Set the default value of the failure keys in the HTTP request test --- nettests/blocking/http_requests.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/nettests/blocking/http_requests.py b/nettests/blocking/http_requests.py index b6bab00..ef2cece 100644 --- a/nettests/blocking/http_requests.py +++ b/nettests/blocking/http_requests.py @@ -51,6 +51,8 @@ class HTTPRequestsTest(httpt.HTTPTest): raise Exception("No input specified") self.factor = self.localOptions['factor'] + self.report['control_failure'] = None + self.report['experiment_failure'] = None def compare_body_lengths(self, body_length_a, body_length_b):

1 0

[ooni-probe/master] Disable by default user agent randomization in HTTP Request template
by art＠torproject.org 26 Feb '13

26 Feb '13

commit b515d72eea5239d8977229a54f457406df419566 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Feb 8 22:44:29 2013 +0100 Disable by default user agent randomization in HTTP Request template --- ooni/templates/httpt.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index d9a1a33..a265f4c 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -41,7 +41,7 @@ class HTTPTest(NetTestCase): name = "HTTP Test" version = "0.1.1" - randomizeUA = True + randomizeUA = False followRedirects = False baseParameters = [['socksproxy', 's', None,

1 0

[ooni-probe/master] Update user agent strings for user agent randomization
by art＠torproject.org 26 Feb '13

26 Feb '13

commit 66cbb71acfaf158fa8380a44395f83c95e6e7548 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Feb 8 22:44:48 2013 +0100 Update user agent strings for user agent randomization They are now taken from the PETS2010 paper "How unique is your browser" as the user agents with the largest anonymity set. Note: this is probably no longer true as of today, but I could not find a better source for mostly used user agent strings and it's for sure better than what we had before (We were considering highly used Opera 8.1). --- .../manipulation/http_header_field_manipulation.py | 2 +- ooni/utils/net.py | 27 ++++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/nettests/manipulation/http_header_field_manipulation.py b/nettests/manipulation/http_header_field_manipulation.py index e6e3fde..c831814 100644 --- a/nettests/manipulation/http_header_field_manipulation.py +++ b/nettests/manipulation/http_header_field_manipulation.py @@ -64,7 +64,7 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): else: # XXX generate these from a random choice taken from whatheaders.com # http://s3.amazonaws.com/data.whatheaders.com/whatheaders-latest.xml.zip - headers = {"User-Agent": [random.choice(net.userAgents)[0]], + headers = {"User-Agent": [random.choice(net.userAgents)], "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], "Accept-Encoding": ["gzip,deflate,sdch"], "Accept-Language": ["en-US,en;q=0.8"], diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 865977c..49f9998 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -12,19 +12,20 @@ from ooni.utils import log, txscapy #if sys.platform.system() == 'Windows': # import _winreg as winreg -userAgents = [ - ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), - ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), - ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), - ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP") - ] +# These user agents are taken from the "How Unique Is Your Web Browser?" +# (https://panopticlick.eff.org/browser-uniqueness.pdf) paper as the browser user +# agents with largest anonymity set. + +userAgents = ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7", + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3 1 2 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Mobile/7D11", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2) Gecko/20100115 Firefox/3.6", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 (.NET CLR 3.5.30729)") class UnsupportedPlatform(Exception): """Support for this platform is not currently available."""

1 0

[ooni-probe/master] Set a default collector
by art＠torproject.org 26 Feb '13

26 Feb '13

commit c6d4fff1d3fcf39181311adad57cff576d5d497f Author: aagbsn <aagbsn(a)extc.org> Date: Thu Feb 21 12:01:53 2013 +0100 Set a default collector --- collector | 1 + ooni/oonicli.py | 3 ++- ooni/runner.py | 9 +++++++++ 3 files changed, 12 insertions(+), 1 deletions(-) diff --git a/collector b/collector new file mode 100644 index 0000000..0b260a1 --- /dev/null +++ b/collector @@ -0,0 +1 @@ +httpo://nkvphnp3p6agi5qq.onion diff --git a/ooni/oonicli.py b/ooni/oonicli.py index c18f3bd..4a638e0 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -28,7 +28,8 @@ class Options(usage.Options): " files listed on the command line") optFlags = [["help", "h"], - ["resume", "r"]] + ["resume", "r"], + ["no-default-reporter", "n"]] optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, diff --git a/ooni/runner.py b/ooni/runner.py index 19d7e1f..a602234 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -4,6 +4,7 @@ import time import inspect import traceback import itertools +import random import yaml @@ -413,6 +414,14 @@ def runTestCases(test_cases, options, cmd_line_options): test_inputs = options['inputs'] + # Set a default reporter + if not cmd_line_options['collector'] and not \ + cmd_line_options['no-default-reporter']: + with open('collector') as f: + reporter_url = random.choice(f.readlines()) + reporter_url = reporter_url.split('#')[0].strip() + cmd_line_options['collector'] = reporter_url + oonib_reporter = OONIBReporter(cmd_line_options) yaml_reporter = YAMLReporter(cmd_line_options)

1 0

[ooni-probe/master] Merge pull request #47 from aagbsn/add_default_oonib_reporter_legacy
by art＠torproject.org 26 Feb '13

26 Feb '13

commit 46308d5ef692d3a454ed9309a1d1dd6efc227f60 Merge: 5ff0b8e c6d4fff Author: Jacob Appelbaum <jacob(a)appelbaum.net> Date: Mon Feb 25 11:44:04 2013 -0800 Merge pull request #47 from aagbsn/add_default_oonib_reporter_legacy Add default oonib reporter collector | 1 + ooni/oonicli.py | 3 ++- ooni/runner.py | 9 +++++++++ 3 files changed, 12 insertions(+), 1 deletions(-)

1 0

[ooni-probe/master] Merge remote-tracking branch 'thetorproject/master' into merging
by art＠torproject.org 26 Feb '13

26 Feb '13

commit 5f568cb1b1bc896641145d79b5c9e42067282bab Merge: 691aed3 46308d5 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Feb 26 13:47:27 2013 +0100 Merge remote-tracking branch 'thetorproject/master' into merging * thetorproject/master: Set a default collector collector | 1 + ooni/oonicli.py | 3 ++- ooni/runner.py | 9 +++++++++ 3 files changed, 12 insertions(+), 1 deletions(-)

1 0