commit b1e8d865f121854e9f96ca70b54eb4a656679b66 Author: Arturo Filastò art@baculo.org Date: Wed Jun 13 19:25:55 2012 +0200

Implement utility class for dealing with HTTP based testing --- ooni/ooniprobe.py | 3 +- ooni/plugoo/tests.py | 13 ++++- ooni/plugoo/work.py | 4 +- ooni/protocols/http.py | 118 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 6 deletions(-)

diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 7248e70..a997041 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -64,7 +64,8 @@ def runTest(test, options, global_options):

log.start(global_options['log'], 1)

- wgen = work.WorkGenerator(test_class(options, global_options, report), + wgen = work.WorkGenerator(test_class(options, global_options, report, + reactor=reactor), dict(options), start=options['resume'])

diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 0632670..608cee3 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -19,12 +19,19 @@ from ooni.plugoo.interface import ITest class OONITest(object): blocking = False

- def __init__(self, local_options, global_options, report, ooninet=None): + def __init__(self, local_options, global_options, report, ooninet=None, + reactor=None): self.local_options = local_options self.global_options = global_options self.assets = self.load_assets() self.report = report #self.ooninet = ooninet + self.reactor = reactor + self.initialize() + self.result = {} + + def initialize(self): + pass

def load_assets(self): """ @@ -40,11 +47,11 @@ class OONITest(object): def finished(self, control): #self.ooninet.report(result) self.end_time = datetime.now() - result = {} + result = self.result result['start_time'] = self.start_time result['end_time'] = self.end_time result['run_time'] = self.end_time - self.start_time - result['result'] = control + result['control'] = control log.msg("FINISHED %s" % result) self.report(result) return result diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 16a6e2c..7e9be0c 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -54,8 +54,8 @@ class Worker(object): if self._running < self.maxconcurrent: asset, test, idx = workunit self._running += 1 - test.startTest(asset).addBoth(self._run) - return + return test.startTest(asset).addBoth(self._run) + d = defer.Deferred() self._queued.append((workunit, d)) return d diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py new file mode 100644 index 0000000..068ec0d --- /dev/null +++ b/ooni/protocols/http.py @@ -0,0 +1,118 @@ +import random +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset +from ooni.protocols.useragents import useragents +from ooni import log + +useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + +class BodyReceiver(protocol.Protocol): + def __init__(self, finished): + self.finished = finished + self.data = "" + + def dataReceived(self, bytes): + self.data += bytes + + def connectionLost(self, reason): + self.finished.callback(self.data) + + +from twisted.web.http_headers import Headers +class HTTPTest(OONITest): + """ + A utility class for dealing with HTTP based testing. It provides methods to + be overriden for dealing with HTTP based testing. + The main functions to look at are processResponseBody and + processResponseHeader that are invoked once the headers have been received + and once the request body has been received. + """ + randomize_ua = True + + def initialize(self, reactor=None): + from twisted.web.client import Agent + import yaml + + if not self.reactor: + from twisted.internet import reactor + self.reactor = reactor + self.agent = Agent(self.reactor) + self.request = {} + self.response = {} + + def _processResponseBody(self, data): + self.response['body'] = data + self.result['response'] = self.response + self.processResponseBody(data) + + def processResponseBody(self, data): + """ + This should handle all the response body smushing for getting it ready + to be passed onto the control. + + @param data: The content of the body returned. + """ + pass + + def processResponseHeaders(self, headers): + """ + This should take care of dealing with the returned HTTP headers. + + @param headers: The content of the returned headers. + """ + pass + + def experiment(self, args): + log.msg("Running experiment") + d = self.build_request(args['url']) + def finished(data): + return data + + d.addCallback(self._cbResponse) + d.addCallback(finished) + return d + + def _cbResponse(self, response): + self.response['headers'] = list(response.headers.getAllRawHeaders()) + self.processResponseHeaders(self.response['headers']) + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished)) + finished.addCallback(self._processResponseBody) + + def randomize_useragent(self): + user_agent = random.choice(useragents) + self.request['headers']['User-Agent'] = [user_agent] + + def build_request(self, url, method="GET", headers=None, body=None): + self.request['method'] = method + self.request['url'] = url + self.request['headers'] = headers if headers else {} + self.request['body'] = body + if self.randomize_ua: + self.randomize_useragent() + + self.result['request'] = self.request + return self.agent.request(self.request['method'], self.request['url'], + Headers(self.request['headers']), + self.request['body']) + + def load_assets(self): + if self.local_options: + return {'url': Asset(self.local_options['asset'])} + else: + return {} +