[tor-commits] [ooni-probe/master] Implement keyword filtering detection tool

[art at torproject.org]

commit a5e0b372a326e2b7daa2a350a821ced13271b170
Author: Arturo Filastò <arturo at filasto.net>
Date:   Tue Nov 6 13:43:01 2012 +0100

    Implement keyword filtering detection tool
    * Fix some bugs in reporter and runner
---
 ooni/reporter.py        |    5 +++--
 ooni/runner.py          |    1 +
 ooni/templates/httpt.py |   22 ++++++++++++++++------
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/ooni/reporter.py b/ooni/reporter.py
index 69f0092..8c0b593 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -194,11 +194,11 @@ class ReporterFactory(OReporter):
 
         try:
             import txtorcon
-            client_location = txtorcon.util.NetLocation(client_ip)
+            client_location = txtorcon.util.NetLocation(client_geodata['ip'])
             client_geodata['asn'] = client_location.asn
             client_geodata['city'] = client_location.city
             client_geodata['countrycode'] = client_location.countrycode
-        except:
+        except ImportError:
             log.err("txtorcon is not installed. Geolocation lookup is not"\
                     "supported")
 
@@ -255,6 +255,7 @@ class OONIReporter(OReporter):
 
         self._tests[idx] = {}
         self._tests[idx]['test_started'] = self._getTime()
+
         if isinstance(test.input, packet.Packet):
             test_input = repr(test.input)
         else:
diff --git a/ooni/runner.py b/ooni/runner.py
index acf5519..e1ecaac 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -73,6 +73,7 @@ def processTest(obj, config):
                 if obj.optFlags:
                     log.debug("Got optFlags")
                     optFlags = obj.optFlags
+
             options = Options()
 
         options.parseOptions(config['subArgs'])
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 5c7d552..0b0b7cb 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -96,8 +96,12 @@ class HTTPTest(NetTestCase):
 
         self.request = {}
         self.response = {}
+        self.processInputs()
         log.debug("Finished test setup")
 
+    def processInputs(self):
+        pass
+
     def _processResponseBody(self, data, body_processor):
         log.debug("Processing response body")
         self.response['body'] = data
@@ -152,11 +156,8 @@ class HTTPTest(NetTestCase):
                          This function takes the response body as an argument.
         """
         log.debug("Performing request %s %s %s" % (url, method, headers))
-        try:
-            d = self.build_request(url, method, headers, body)
-        except Exception, e:
-            print e
-            self.report['error'] = e
+
+        d = self.build_request(url, method, headers, body)
 
         def errback(data):
             log.err("Error in test %s" % data)
@@ -205,13 +206,22 @@ class HTTPTest(NetTestCase):
         self.request['url'] = url
         self.request['headers'] = headers if headers else {}
         self.request['body'] = body
+
         if self.randomizeUA:
             self.randomize_useragent()
 
         self.report['request'] = self.request
         self.report['url'] = url
+
+        # If we have a request body payload, set the request body to such
+        # content
+        if body:
+            body_producer = StringProducer(self.request['body'])
+        else:
+            body_producer = None
+
         req = self.agent.request(self.request['method'], self.request['url'],
                                   Headers(self.request['headers']),
-                                  self.request['body'])
+                                  body_producer)
         return req