[tor-commits] [ooni-probe/master] Implement input File support for new API.
art at torproject.org
art at torproject.org
Sat Oct 6 10:47:27 UTC 2012
commit 9c7f354bcb842be19d687305c8a2e4e57cad6276
Author: Arturo Filastò <arturo at filasto.net>
Date: Sat Oct 6 10:47:03 2012 +0000
Implement input File support for new API.
* Port HTTP Host test to new API
---
nettests/http_host.py | 28 +++--------
nettests/httphost.py | 124 -------------------------------------------------
ooni/nettest.py | 14 +++---
ooni/runner.py | 5 ++
4 files changed, 20 insertions(+), 151 deletions(-)
diff --git a/nettests/http_host.py b/nettests/http_host.py
index 4834ba3..2228c52 100644
--- a/nettests/http_host.py
+++ b/nettests/http_host.py
@@ -11,37 +11,25 @@
from ooni.templates import httpt
-good_http_server = "http://127.0.0.1:8090/"
-
class HTTPHost(httpt.HTTPTest):
name = "HTTP Host"
author = "Arturo Filastò"
version = 0.1
+ optParameters = [['url', 'u', 'http://torproject.org/', 'Test single site'],
+ ['backend', 'b', 'http://ooni.nu/test/', 'Test backend to use'],
+ ]
- inputs = ['google.com', 'wikileaks.org',
- 'torproject.org']
+ inputFile = ['urls', 'f', None, 'Urls file']
def test_send_host_header(self):
headers = {}
headers["Host"] = [self.input]
- return self.doRequest(good_http_server, headers=headers)
+ return self.doRequest(self.localOptions['backend'], headers=headers)
def processResponseBody(self, body):
- # XXX here shall go your logic
- # for processing the body
- if 'blocked' in body:
- self.report['censored'] = True
+ if 'not censored' in body:
+ self.report['trans_http_proxy'] = False
else:
- self.report['censored'] = False
-
- def processResponseHeaders(self, headers):
- # XXX place in here all the logic for handling the processing of HTTP
- # Headers.
- if headers.hasHeader('location'):
- self.report['redirect'] = True
-
- server = headers.getRawHeaders("Server")
- if server:
- self.report['http_server'] = str(server.pop())
+ self.report['trans_http_proxy'] = True
diff --git a/nettests/httphost.py b/nettests/httphost.py
deleted file mode 100644
index b57a50d..0000000
--- a/nettests/httphost.py
+++ /dev/null
@@ -1,124 +0,0 @@
-"""
- HTTP Host based filtering
- *************************
-
- This test detect HTTP Host field
- based filtering.
-"""
-import os
-from datetime import datetime
-
-import urllib2
-import httplib
-try:
- from BeautifulSoup import BeautifulSoup
-except:
- print "Beautiful Soup not installed. HTTPHost may not work"
-
-# XXX reduce boilerplating.
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-
-from ooni.plugoo.assets import Asset
-from ooni.plugoo.tests import ITest, OONITest
-
-class HTTPHostArgs(usage.Options):
- optParameters = [['asset', 'a', None, 'Asset file'],
- ['controlserver', 'c', 'google.com', 'Specify the control server'],
- ['resume', 'r', 0, 'Resume at this index'],
- ['other', 'o', None, 'Other arguments']]
- def control(self, experiment_result, args):
- print "Experiment Result:", experiment_result
- print "Args", args
- return experiment_result
-
- def experiment(self, args):
- import urllib
- req = urllib.urlopen(args['asset'])
- return {'page': req.readlines()}
-
-class HTTPHostAsset(Asset):
- """
- This is the asset that should be used by the Test. It will
- contain all the code responsible for parsing the asset file
- and should be passed on instantiation to the test.
- """
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
- def parse_line(self, line):
- return line.split(',')[1].replace('\n','')
-
-
-class HTTPHostTest(OONITest):
- implements(IPlugin, ITest)
-
- shortName = "httphost"
- description = "HTTP Host plugin"
- requirements = None
- options = HTTPHostArgs
- # Tells this to be blocking.
- blocking = True
-
- def check_response(self, response):
- soup = BeautifulSoup(response)
- if soup.head.title.string == "Blocked":
- # Response indicates censorship
- return True
- else:
- # Response does not indicate censorship
- return False
-
- def is_censored(self, response):
- if response:
- soup = BeautifulSoup(response)
- censored = self.check_response(response)
- else:
- censored = "unreachable"
- return censored
-
- def urllib2_test(self, control_server, host):
- req = urllib2.Request(control_server)
- req.add_header('Host', host)
- try:
- r = urllib2.urlopen(req)
- response = r.read()
- censored = self.is_censored(response)
- except Exception, e:
- censored = "Error! %s" % e
-
- return censored
-
- def httplib_test(self, control_server, host):
- censored = None
- response = None
- try:
- conn = httplib.HTTPConnection(control_server)
- conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True)
- conn.putheader("Host", host)
- conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6")
- conn.endheaders()
- r = conn.getresponse()
- response = r.read()
- censored = self.is_censored(response)
- except Exception, e:
- censored = "Error! %s" % e
-
- return (censored, response)
-
- def experiment(self, args):
- control_server = self.local_options['controlserver']
- url = 'http://torproject.org/' if not 'asset' in args else args['asset']
- censored = self.httplib_test(control_server, url)
- return {'control': control_server,
- 'host': url,
- 'censored': censored}
-
- def load_assets(self):
- if self.local_options and self.local_options['asset']:
- return {'asset': Asset(self.local_options['asset'])}
- else:
- return {}
-
-httphost = HTTPHostTest(None, None, None)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index a01050e..89dd279 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -24,11 +24,6 @@ class InputTestSuite(pyunit.TestSuite):
self._idx += 1
return result
-def lineByLine(fp):
- for x in fp.readlines():
- yield x.strip()
- fp.close()
-
class TestCase(unittest.TestCase):
"""
This is the monad of the OONI nettest universe. When you write a nettest
@@ -75,17 +70,22 @@ class TestCase(unittest.TestCase):
inputs = [None]
inputFile = None
- inputProcessor = lineByLine
+
report = {}
report['errors'] = []
optParameters = None
+ def inputProcessor(self, fp):
+ for x in fp.readlines():
+ yield x.strip()
+ fp.close()
+
def getOptions(self):
if self.inputFile:
fp = open(self.inputFile)
- self.inputs = inputProcessor(fp)
+ self.inputs = self.inputProcessor(fp)
# XXX perhaps we may want to name and version to be inside of a
# different object that is not called options.
return {'inputs': self.inputs,
diff --git a/ooni/runner.py b/ooni/runner.py
index c6958bc..a753d02 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -96,10 +96,15 @@ def processTest(obj, config):
class Options(usage.Options):
optParameters = obj.optParameters
+ inputFile = obj.inputFile
+ Options.optParameters.append(inputFile)
+
options = Options()
options.parseOptions(config['subArgs'])
obj.localOptions = options
+ obj.inputFile = options[inputFile[0]]
+
return obj
def findTestClassesFromConfig(config):
More information about the tor-commits
mailing list