[tor-commits] [ooni-probe/master] Implement the testing logic for redirect based censorship detection.

[art at torproject.org]

commit 777e3fe755b6805293f5ad8e50663fbd61899aae
Author: Arturo Filastò <art at torproject.org>
Date:   Fri Jul 20 18:43:41 2012 +0200

    Implement the testing logic for redirect based censorship detection.
    Closes #6437
---
 ooni/assets/redirects.yaml |   17 +++++++++++++
 ooni/plugins/httpt.py      |   54 +++++++++++++++++++++++++++++++++++++++++++-
 ooni/protocols/http.py     |   17 +++++++++++++
 3 files changed, 87 insertions(+), 1 deletions(-)

diff --git a/ooni/assets/redirects.yaml b/ooni/assets/redirects.yaml
new file mode 100644
index 0000000..43582c4
--- /dev/null
+++ b/ooni/assets/redirects.yaml
@@ -0,0 +1,17 @@
+o2:
+  name: O2 Networks
+  patterns:
+  - {type: re, value: '(https://bango.net/).*'}
+  - {type: re, value: 'http://wap.o2.co.uk/(18plusaccess).*'}
+orange:
+  name: Orange Telecom
+  patterns: {type: eq, value: 'http://orangeworld.co.uk/r/avblocked/'}
+three:
+  name: Three
+  patterns: {type: re, value: '(http://mobile.three.co.uk/pc/Live/pcreator/live/).*'}
+tmobile:
+  name: T-Mobile
+  patterns: {type: eq, value: 'http://www.t-mobile.co.uk/common/system_error_pages/outage_wnw.html'}
+vodafone:
+  name: Vodafone UK
+  patterns: {type: eq, value: 'http://online.vodafone.co.uk/en_GB/assets/static/contentcontrol/unbranded/restricted_access.html'}
diff --git a/ooni/plugins/httpt.py b/ooni/plugins/httpt.py
index 4113aef..46f3b17 100644
--- a/ooni/plugins/httpt.py
+++ b/ooni/plugins/httpt.py
@@ -14,7 +14,8 @@ from ooni.utils import log
 class httptArgs(usage.Options):
     optParameters = [['urls', 'f', None, 'Urls file'],
                      ['url', 'u', 'http://torproject.org/', 'Test single site'],
-                     ['resume', 'r', 0, 'Resume at this index']]
+                     ['resume', 'r', 0, 'Resume at this index'],
+                     ['rules', 'y', None, 'Specify the redirect rules file']]
 
 class httptTest(http.HTTPTest):
     implements(IPlugin, ITest)
@@ -25,7 +26,58 @@ class httptTest(http.HTTPTest):
     options = httptArgs
     blocking = False
 
+
+    def testPattern(self, value, pattern, type):
+        if type == 'eq':
+            return value == pattern
+        elif type == 're':
+            import re
+            if re.match(pattern, value):
+                return True
+            else:
+                return False
+        else:
+            return None
+
+    def testPatterns(self, patterns, location):
+        test_result = False
+
+        if type(patterns) == list:
+            for pattern in patterns:
+                test_result |= self.testPattern(location, pattern['value'], pattern['type'])
+        else:
+            test_result |= self.testPattern(location, patterns['value'], patterns['type'])
+
+        return test_result
+
+    def testRules(self, rules, location):
+        result = {}
+        blocked = False
+        for rule, value in rules.items():
+            current_rule = {}
+            current_rule['name'] = value['name']
+            current_rule['patterns'] = value['patterns']
+            current_rule['test'] = self.testPatterns(value['patterns'], location)
+            blocked |= current_rule['test']
+            result[rule] = current_rule
+        result['blocked'] = blocked
+        return result
+
+    def processRedirect(self, location):
+        self.result['redirect'] = None
+        if self.local_options['rules']:
+            import yaml
+            rules = yaml.load(open(self.local_options['rules']))
+            log.msg("Testing rules %s" % rules)
+            redirect = self.testRules(rules, location)
+            self.result['redirect'] = redirect
+        else:
+            log.msg("No rules file. Got a redirect, but nothing to do.")
+
+
     def control(self, experiment_result, args):
+        print self.response
+        print self.request
         # What you return here ends up inside of the report.
         log.msg("Running control")
         return {}
diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py
index 100db88..d5573b3 100644
--- a/ooni/protocols/http.py
+++ b/ooni/protocols/http.py
@@ -71,6 +71,15 @@ class HTTPTest(OONITest):
         """
         pass
 
+    def processRedirect(self, location):
+        """
+        Handle a redirection via a 3XX HTTP status code.
+
+        @param location: the url that is being redirected to.
+        """
+        pass
+
+
     def experiment(self, args):
         log.msg("Running experiment")
         url = self.local_options['url'] if 'url' not in args else args['url']
@@ -85,7 +94,15 @@ class HTTPTest(OONITest):
 
     def _cbResponse(self, response):
         self.response['headers'] = list(response.headers.getAllRawHeaders())
+        self.response['code'] = response.code
+        self.response['length'] = response.length
+        self.response['version'] = response.length
+
+        if str(self.response['code']).startswith('3'):
+            self.processRedirect(response.headers.getRawHeaders('Location')[0])
         self.processResponseHeaders(self.response['headers'])
+        self.result['response'] = self.response
+
         finished = defer.Deferred()
         response.deliverBody(BodyReceiver(finished))
         finished.addCallback(self._processResponseBody)