commit f358389ab248fdd4d6ced99e9e2b9f6265cdd38b Author: Arturo Filastò arturo@filasto.net Date: Fri Sep 28 14:13:54 2012 +0000

Implement the first Test Template based on the new API. * Fix some bugs --- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 10 ++-- ooni/plugoo/work.py | 2 - ooni/runner.py | 8 ++- ooni/templates/http.py | 147 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 11 deletions(-)

diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index b48ffdf..1bfdac0 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -4,7 +4,7 @@ import os import yaml

import itertools -from utils import log, date, net +from ooni.utils import log, date, net

class Report: """This is the ooni-probe reporting mechanism. It allows diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 75c23f7..21ac7bf 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -8,10 +8,10 @@ from twisted.internet import reactor, defer, threads ## XXX why is this imported and not used? from twisted.python import failure

-from utils import log, date -from plugoo import assets, work -from plugoo.reports import Report -from plugoo.interface import ITest +from ooni.utils import log, date +from ooni.plugoo import assets, work +from ooni.plugoo.reports import Report +from ooni.plugoo.interface import ITest

class OONITest(object): @@ -58,7 +58,7 @@ class OONITest(object): return {}

def __repr__(self): - return "<OONITest %s %s %s>" % (self.local_options, + return "<OONITest %s %s %s>" % (self.local_options, self.global_options, self.assets)

diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index e32d76d..db88fbf 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -19,8 +19,6 @@ from zope.interface import Interface, Attribute from twisted.python import failure from twisted.internet import reactor, defer

-from plugoo.nodes import LocalNode - class Worker(object): """ This is the core of OONI. It takes as input Work Units and diff --git a/ooni/runner.py b/ooni/runner.py index ff6b47e..1a9b4ad 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -5,7 +5,8 @@ import time import inspect

from twisted.internet import defer, reactor -from twisted.python import reflect, log, failure +from twisted.python import reflect, failure +from twisted.python import log from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader from twisted.trial.runner import isPackage, isTestCase, ErrorHolder @@ -15,7 +16,7 @@ from ooni.reporter import ReporterFactory from ooni.input import InputUnitFactory from ooni.nettest import InputTestSuite from ooni import nettest -from ooni.utils import log +#from ooni.utils import log from ooni.plugoo import tests as oonitests

def isTestCase(thing): @@ -182,7 +183,8 @@ class ORunner(object): result.done()

def run(self): - log.start() + log.startLogging(sys.stdout) + #log.start(True) self.reporterFactory.writeHeader()

for inputUnit in InputUnitFactory(self.inputs): diff --git a/ooni/templates/__init__.py b/ooni/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ooni/templates/http.py b/ooni/templates/http.py new file mode 100644 index 0000000..222e44c --- /dev/null +++ b/ooni/templates/http.py @@ -0,0 +1,147 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import random + +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer +from twisted.web.http_headers import Headers + +from ooni.nettest import TestCase +from ooni.utils import log + +useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + +class BodyReceiver(protocol.Protocol): + def __init__(self, finished): + self.finished = finished + self.data = "" + + def dataReceived(self, bytes): + self.data += bytes + + def connectionLost(self, reason): + self.finished.callback(self.data) + +class HTTPTest(TestCase): + """ + A utility class for dealing with HTTP based testing. It provides methods to + be overriden for dealing with HTTP based testing. + The main functions to look at are processResponseBody and + processResponseHeader that are invoked once the headers have been received + and once the request body has been received. + """ + randomizeUA = True + followRedirects = False + + def setUp(self): + from twisted.web.client import Agent + from twisted.internet import reactor + import yaml + self.agent = Agent(reactor) + if self.followRedirects: + from twisted.web.client import RedirectAgent + self.agent = RedirectAgent(self.agent) + self.request = {} + self.response = {} + + def _processResponseBody(self, data): + self.response['body'] = data + self.report['response'] = self.response + + self.processResponseBody(data) + + def processResponseBody(self, data): + """ + This should handle all the response body smushing for getting it ready + to be passed onto the control. + + @param data: The content of the body returned. + """ + pass + + def processResponseHeaders(self, headers): + """ + This should take care of dealing with the returned HTTP headers. + + @param headers: The content of the returned headers. + """ + pass + + def processRedirect(self, location): + """ + Handle a redirection via a 3XX HTTP status code. + + @param location: the url that is being redirected to. + """ + pass + + def doRequest(self, url): + d = self.build_request(url) + def finished(data): + return data + + d.addCallback(self._cbResponse) + d.addCallback(finished) + return d + + def test_http(self): + log.msg("Running experiment") + + if self.input: + url = self.input + else: + raise Exception("No input supplied") + + self.mainDefer = self.doRequest(url) + return self.mainDefer + + def _cbResponse(self, response): + self.response['headers'] = response.headers + self.response['code'] = response.code + self.response['length'] = response.length + self.response['version'] = response.length + + if str(self.response['code']).startswith('3'): + self.processRedirect(response.headers.getRawHeaders('Location')[0]) + + self.processResponseHeaders(self.response['headers']) + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished)) + finished.addCallback(self._processResponseBody) + + return finished + + def randomize_useragent(self): + user_agent = random.choice(useragents) + self.request['headers']['User-Agent'] = [user_agent] + + def build_request(self, url, method="GET", headers=None, body=None): + self.request['method'] = method + self.request['url'] = url + self.request['headers'] = headers if headers else {} + self.request['body'] = body + if self.randomizeUA: + self.randomize_useragent() + + self.report['request'] = self.request + self.report['url'] = url + return self.agent.request(self.request['method'], self.request['url'], + Headers(self.request['headers']), + self.request['body']) +