commit 9c7f354bcb842be19d687305c8a2e4e57cad6276 Author: Arturo Filastò arturo@filasto.net Date: Sat Oct 6 10:47:03 2012 +0000

Implement input File support for new API. * Port HTTP Host test to new API --- nettests/http_host.py | 28 +++-------- nettests/httphost.py | 124 ------------------------------------------------- ooni/nettest.py | 14 +++--- ooni/runner.py | 5 ++ 4 files changed, 20 insertions(+), 151 deletions(-)

diff --git a/nettests/http_host.py b/nettests/http_host.py index 4834ba3..2228c52 100644 --- a/nettests/http_host.py +++ b/nettests/http_host.py @@ -11,37 +11,25 @@

from ooni.templates import httpt

-good_http_server = "http://127.0.0.1:8090/" - class HTTPHost(httpt.HTTPTest): name = "HTTP Host" author = "Arturo Filastò" version = 0.1

+ optParameters = [['url', 'u', 'http://torproject.org/', 'Test single site'], + ['backend', 'b', 'http://ooni.nu/test/', 'Test backend to use'], + ]

- inputs = ['google.com', 'wikileaks.org', - 'torproject.org'] + inputFile = ['urls', 'f', None, 'Urls file']

def test_send_host_header(self): headers = {} headers["Host"] = [self.input] - return self.doRequest(good_http_server, headers=headers) + return self.doRequest(self.localOptions['backend'], headers=headers)

def processResponseBody(self, body): - # XXX here shall go your logic - # for processing the body - if 'blocked' in body: - self.report['censored'] = True + if 'not censored' in body: + self.report['trans_http_proxy'] = False else: - self.report['censored'] = False - - def processResponseHeaders(self, headers): - # XXX place in here all the logic for handling the processing of HTTP - # Headers. - if headers.hasHeader('location'): - self.report['redirect'] = True - - server = headers.getRawHeaders("Server") - if server: - self.report['http_server'] = str(server.pop()) + self.report['trans_http_proxy'] = True

diff --git a/nettests/httphost.py b/nettests/httphost.py deleted file mode 100644 index b57a50d..0000000 --- a/nettests/httphost.py +++ /dev/null @@ -1,124 +0,0 @@ -""" - HTTP Host based filtering - ************************* - - This test detect HTTP Host field - based filtering. -""" -import os -from datetime import datetime - -import urllib2 -import httplib -try: - from BeautifulSoup import BeautifulSoup -except: - print "Beautiful Soup not installed. HTTPHost may not work" - -# XXX reduce boilerplating. -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin - -from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, OONITest - -class HTTPHostArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['controlserver', 'c', 'google.com', 'Specify the control server'], - ['resume', 'r', 0, 'Resume at this index'], - ['other', 'o', None, 'Other arguments']] - def control(self, experiment_result, args): - print "Experiment Result:", experiment_result - print "Args", args - return experiment_result - - def experiment(self, args): - import urllib - req = urllib.urlopen(args['asset']) - return {'page': req.readlines()} - -class HTTPHostAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - return line.split(',')[1].replace('\n','') - - -class HTTPHostTest(OONITest): - implements(IPlugin, ITest) - - shortName = "httphost" - description = "HTTP Host plugin" - requirements = None - options = HTTPHostArgs - # Tells this to be blocking. - blocking = True - - def check_response(self, response): - soup = BeautifulSoup(response) - if soup.head.title.string == "Blocked": - # Response indicates censorship - return True - else: - # Response does not indicate censorship - return False - - def is_censored(self, response): - if response: - soup = BeautifulSoup(response) - censored = self.check_response(response) - else: - censored = "unreachable" - return censored - - def urllib2_test(self, control_server, host): - req = urllib2.Request(control_server) - req.add_header('Host', host) - try: - r = urllib2.urlopen(req) - response = r.read() - censored = self.is_censored(response) - except Exception, e: - censored = "Error! %s" % e - - return censored - - def httplib_test(self, control_server, host): - censored = None - response = None - try: - conn = httplib.HTTPConnection(control_server) - conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) - conn.putheader("Host", host) - conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") - conn.endheaders() - r = conn.getresponse() - response = r.read() - censored = self.is_censored(response) - except Exception, e: - censored = "Error! %s" % e - - return (censored, response) - - def experiment(self, args): - control_server = self.local_options['controlserver'] - url = 'http://torproject.org/' if not 'asset' in args else args['asset'] - censored = self.httplib_test(control_server, url) - return {'control': control_server, - 'host': url, - 'censored': censored} - - def load_assets(self): - if self.local_options and self.local_options['asset']: - return {'asset': Asset(self.local_options['asset'])} - else: - return {} - -httphost = HTTPHostTest(None, None, None) diff --git a/ooni/nettest.py b/ooni/nettest.py index a01050e..89dd279 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -24,11 +24,6 @@ class InputTestSuite(pyunit.TestSuite): self._idx += 1 return result

-def lineByLine(fp): - for x in fp.readlines(): - yield x.strip() - fp.close() - class TestCase(unittest.TestCase): """ This is the monad of the OONI nettest universe. When you write a nettest @@ -75,17 +70,22 @@ class TestCase(unittest.TestCase):

inputs = [None] inputFile = None - inputProcessor = lineByLine +

report = {} report['errors'] = []

optParameters = None

+ def inputProcessor(self, fp): + for x in fp.readlines(): + yield x.strip() + fp.close() + def getOptions(self): if self.inputFile: fp = open(self.inputFile) - self.inputs = inputProcessor(fp) + self.inputs = self.inputProcessor(fp) # XXX perhaps we may want to name and version to be inside of a # different object that is not called options. return {'inputs': self.inputs, diff --git a/ooni/runner.py b/ooni/runner.py index c6958bc..a753d02 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -96,10 +96,15 @@ def processTest(obj, config): class Options(usage.Options): optParameters = obj.optParameters

+ inputFile = obj.inputFile + Options.optParameters.append(inputFile) + options = Options() options.parseOptions(config['subArgs'])

obj.localOptions = options + obj.inputFile = options[inputFile[0]] + return obj

def findTestClassesFromConfig(config):