commit 213fcf08a4dee4776590c3321796bc7d9f7eb09c Author: Arturo Filastò art@fuffa.org Date: Tue Nov 20 22:26:51 2012 +0100

Add socks proxy support to HTTP Test * Use required options in http keyword filtering test * Fix before_i_commit script --- before_i_commit.sh | 2 - nettests/core/http_keyword_filtering.py | 23 ++++++++++++--------- ooni/templates/httpt.py | 32 +++++++++++++++++++++++++++--- 3 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/before_i_commit.sh b/before_i_commit.sh index 96117b3..3e01366 100755 --- a/before_i_commit.sh +++ b/before_i_commit.sh @@ -24,8 +24,6 @@ find . -type f -name "*.py[co]" -delete

./bin/ooniprobe -l before_i_commit.log -o http_keyword_filtering.yamloo nettests/core/http_keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt

-./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt - echo "Below you should not see anything" echo "---------------------------------" grep "Error: " before_i_commit.log diff --git a/nettests/core/http_keyword_filtering.py b/nettests/core/http_keyword_filtering.py index fabdba6..0ae9c52 100644 --- a/nettests/core/http_keyword_filtering.py +++ b/nettests/core/http_keyword_filtering.py @@ -3,7 +3,14 @@ # :authors: Arturo Filastò # :licence: see LICENSE

+from twisted.python import usage + from ooni.templates import httpt + +class UsageOptions(usage.Options): + optParameters = [['backend', 'b', 'http://127.0.0.1:57001', + 'URL of the test backend to use']] + class HTTPKeywordFiltering(httpt.HTTPTest): """ This test involves performing HTTP requests containing to be tested for @@ -14,29 +21,25 @@ class HTTPKeywordFiltering(httpt.HTTPTest): """ name = "HTTP Keyword Filtering" author = "Arturo Filastò" - version = 0.1 - - optParameters = [['backend', 'b', None, 'URL of the backend system to use for testing']] + version = "0.1.1"

inputFile = ['file', 'f', None, 'List of keywords to use for censorship testing']

- def processInputs(self): - if 'backend' in self.localOptions: - self.url = self.localOptions['backend'] - else: - raise Exception("No backend specified") + usageOptions = UsageOptions + + requiredOptions = ['backend']

def test_get(self): """ Perform a HTTP GET request to the backend containing the keyword to be tested inside of the request body. """ - return self.doRequest(self.url, method="GET", body=self.input) + return self.doRequest(self.localOptions['backend'], method="GET", body=self.input)

def test_post(self): """ Perform a HTTP POST request to the backend containing the keyword to be tested inside of the request body. """ - return self.doRequest(self.url, method="POST", body=self.input) + return self.doRequest(self.localOptions['backend'], method="POST", body=self.input)

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 85f8586..7caab39 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -44,6 +44,10 @@ class HTTPTest(NetTestCase):

randomizeUA = True followRedirects = False + + baseParameters = [['socksproxy', 's', None, + 'Specify a socks proxy to use for requests (ip:port)']] + request = {} response = {}

@@ -55,13 +59,21 @@ class HTTPTest(NetTestCase): log.err("Warning! pyOpenSSL is not installed. https websites will" "not work")

- self.agent = Agent(reactor, - sockhost="127.0.0.1", + self.control_agent = Agent(reactor, sockhost="127.0.0.1", sockport=config.advanced.tor_socksport)

+ sockshost, socksport = (None, None) + if self.localOptions['socksproxy']: + sockshost, socksport = self.localOptions['socksproxy'].split(':') + socksport = int(socksport) + + self.agent = Agent(reactor, sockhost=sockshost, + sockport=socksport) + if self.followRedirects: try: from twisted.web.client import RedirectAgent + self.control_agent = RedirectAgent(self.control_agent) self.agent = RedirectAgent(self.agent) except: log.err("Warning! You are running an old version of twisted"\ @@ -150,6 +162,13 @@ class HTTPTest(NetTestCase): if use_tor: log.debug("Using tor for the request") url = 's'+url + agent = self.tor_agent + else: + agent = self.agent + + if self.localOptions['socksproxy']: + log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy'])) + url = 's'+url

log.debug("Performing request %s %s %s" % (url, method, headers))

@@ -188,7 +207,7 @@ class HTTPTest(NetTestCase): def finished(data): return

- d = self.agent.request(request['method'], request['url'], headers, + d = agent.request(request['method'], request['url'], headers, body_producer)

d.addErrback(errback) @@ -198,7 +217,12 @@ class HTTPTest(NetTestCase):

def _cbResponse(self, response, request, headers_processor, body_processor): - log.debug("Got response %s" % response) + + if not response: + log.err("Got no response") + return + else: + log.debug("Got response %s" % response)

if str(response.code).startswith('3'): self.processRedirect(response.headers.getRawHeaders('Location')[0])