commit a5e0b372a326e2b7daa2a350a821ced13271b170 Author: Arturo Filastò arturo@filasto.net Date: Tue Nov 6 13:43:01 2012 +0100

Implement keyword filtering detection tool * Fix some bugs in reporter and runner --- ooni/reporter.py | 5 +++-- ooni/runner.py | 1 + ooni/templates/httpt.py | 22 ++++++++++++++++------ 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/ooni/reporter.py b/ooni/reporter.py index 69f0092..8c0b593 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -194,11 +194,11 @@ class ReporterFactory(OReporter):

try: import txtorcon - client_location = txtorcon.util.NetLocation(client_ip) + client_location = txtorcon.util.NetLocation(client_geodata['ip']) client_geodata['asn'] = client_location.asn client_geodata['city'] = client_location.city client_geodata['countrycode'] = client_location.countrycode - except: + except ImportError: log.err("txtorcon is not installed. Geolocation lookup is not"\ "supported")

@@ -255,6 +255,7 @@ class OONIReporter(OReporter):

self._tests[idx] = {} self._tests[idx]['test_started'] = self._getTime() + if isinstance(test.input, packet.Packet): test_input = repr(test.input) else: diff --git a/ooni/runner.py b/ooni/runner.py index acf5519..e1ecaac 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -73,6 +73,7 @@ def processTest(obj, config): if obj.optFlags: log.debug("Got optFlags") optFlags = obj.optFlags + options = Options()

options.parseOptions(config['subArgs']) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 5c7d552..0b0b7cb 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -96,8 +96,12 @@ class HTTPTest(NetTestCase):

self.request = {} self.response = {} + self.processInputs() log.debug("Finished test setup")

+ def processInputs(self): + pass + def _processResponseBody(self, data, body_processor): log.debug("Processing response body") self.response['body'] = data @@ -152,11 +156,8 @@ class HTTPTest(NetTestCase): This function takes the response body as an argument. """ log.debug("Performing request %s %s %s" % (url, method, headers)) - try: - d = self.build_request(url, method, headers, body) - except Exception, e: - print e - self.report['error'] = e + + d = self.build_request(url, method, headers, body)

def errback(data): log.err("Error in test %s" % data) @@ -205,13 +206,22 @@ class HTTPTest(NetTestCase): self.request['url'] = url self.request['headers'] = headers if headers else {} self.request['body'] = body + if self.randomizeUA: self.randomize_useragent()

self.report['request'] = self.request self.report['url'] = url + + # If we have a request body payload, set the request body to such + # content + if body: + body_producer = StringProducer(self.request['body']) + else: + body_producer = None + req = self.agent.request(self.request['method'], self.request['url'], Headers(self.request['headers']), - self.request['body']) + body_producer) return req