commit 777e3fe755b6805293f5ad8e50663fbd61899aae Author: Arturo Filastò <art@torproject.org> Date: Fri Jul 20 18:43:41 2012 +0200 Implement the testing logic for redirect based censorship detection. Closes #6437 --- ooni/assets/redirects.yaml | 17 +++++++++++++ ooni/plugins/httpt.py | 54 +++++++++++++++++++++++++++++++++++++++++++- ooni/protocols/http.py | 17 +++++++++++++ 3 files changed, 87 insertions(+), 1 deletions(-) diff --git a/ooni/assets/redirects.yaml b/ooni/assets/redirects.yaml new file mode 100644 index 0000000..43582c4 --- /dev/null +++ b/ooni/assets/redirects.yaml @@ -0,0 +1,17 @@ +o2: + name: O2 Networks + patterns: + - {type: re, value: '(https://bango.net/).*'} + - {type: re, value: 'http://wap.o2.co.uk/(18plusaccess).*'} +orange: + name: Orange Telecom + patterns: {type: eq, value: 'http://orangeworld.co.uk/r/avblocked/'} +three: + name: Three + patterns: {type: re, value: '(http://mobile.three.co.uk/pc/Live/pcreator/live/).*'} +tmobile: + name: T-Mobile + patterns: {type: eq, value: 'http://www.t-mobile.co.uk/common/system_error_pages/outage_wnw.html'} +vodafone: + name: Vodafone UK + patterns: {type: eq, value: 'http://online.vodafone.co.uk/en_GB/assets/static/contentcontrol/unbranded/restricted_access.html'} diff --git a/ooni/plugins/httpt.py b/ooni/plugins/httpt.py index 4113aef..46f3b17 100644 --- a/ooni/plugins/httpt.py +++ b/ooni/plugins/httpt.py @@ -14,7 +14,8 @@ from ooni.utils import log class httptArgs(usage.Options): optParameters = [['urls', 'f', None, 'Urls file'], ['url', 'u', 'http://torproject.org/', 'Test single site'], - ['resume', 'r', 0, 'Resume at this index']] + ['resume', 'r', 0, 'Resume at this index'], + ['rules', 'y', None, 'Specify the redirect rules file']] class httptTest(http.HTTPTest): implements(IPlugin, ITest) @@ -25,7 +26,58 @@ class httptTest(http.HTTPTest): options = httptArgs blocking = False + + def testPattern(self, value, pattern, type): + if type == 'eq': + return value == pattern + elif type == 're': + import re + if re.match(pattern, value): + return True + else: + return False + else: + return None + + def testPatterns(self, patterns, location): + test_result = False + + if type(patterns) == list: + for pattern in patterns: + test_result |= self.testPattern(location, pattern['value'], pattern['type']) + else: + test_result |= self.testPattern(location, patterns['value'], patterns['type']) + + return test_result + + def testRules(self, rules, location): + result = {} + blocked = False + for rule, value in rules.items(): + current_rule = {} + current_rule['name'] = value['name'] + current_rule['patterns'] = value['patterns'] + current_rule['test'] = self.testPatterns(value['patterns'], location) + blocked |= current_rule['test'] + result[rule] = current_rule + result['blocked'] = blocked + return result + + def processRedirect(self, location): + self.result['redirect'] = None + if self.local_options['rules']: + import yaml + rules = yaml.load(open(self.local_options['rules'])) + log.msg("Testing rules %s" % rules) + redirect = self.testRules(rules, location) + self.result['redirect'] = redirect + else: + log.msg("No rules file. Got a redirect, but nothing to do.") + + def control(self, experiment_result, args): + print self.response + print self.request # What you return here ends up inside of the report. log.msg("Running control") return {} diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py index 100db88..d5573b3 100644 --- a/ooni/protocols/http.py +++ b/ooni/protocols/http.py @@ -71,6 +71,15 @@ class HTTPTest(OONITest): """ pass + def processRedirect(self, location): + """ + Handle a redirection via a 3XX HTTP status code. + + @param location: the url that is being redirected to. + """ + pass + + def experiment(self, args): log.msg("Running experiment") url = self.local_options['url'] if 'url' not in args else args['url'] @@ -85,7 +94,15 @@ class HTTPTest(OONITest): def _cbResponse(self, response): self.response['headers'] = list(response.headers.getAllRawHeaders()) + self.response['code'] = response.code + self.response['length'] = response.length + self.response['version'] = response.length + + if str(self.response['code']).startswith('3'): + self.processRedirect(response.headers.getRawHeaders('Location')[0]) self.processResponseHeaders(self.response['headers']) + self.result['response'] = self.response + finished = defer.Deferred() response.deliverBody(BodyReceiver(finished)) finished.addCallback(self._processResponseBody)