[tor-commits] [ooni-probe/master] Add socks proxy support to HTTP Test

[art at torproject.org]

commit 213fcf08a4dee4776590c3321796bc7d9f7eb09c
Author: Arturo Filastò <art at fuffa.org>
Date:   Tue Nov 20 22:26:51 2012 +0100

    Add socks proxy support to HTTP Test
    * Use required options in http keyword filtering test
    * Fix before_i_commit script
---
 before_i_commit.sh                      |    2 -
 nettests/core/http_keyword_filtering.py |   23 ++++++++++++---------
 ooni/templates/httpt.py                 |   32 +++++++++++++++++++++++++++---
 3 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/before_i_commit.sh b/before_i_commit.sh
index 96117b3..3e01366 100755
--- a/before_i_commit.sh
+++ b/before_i_commit.sh
@@ -24,8 +24,6 @@ find . -type f -name "*.py[co]" -delete
 
 ./bin/ooniprobe -l before_i_commit.log -o http_keyword_filtering.yamloo nettests/core/http_keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt
 
-./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt
-
 echo "Below you should not see anything"
 echo "---------------------------------"
 grep "Error: " before_i_commit.log
diff --git a/nettests/core/http_keyword_filtering.py b/nettests/core/http_keyword_filtering.py
index fabdba6..0ae9c52 100644
--- a/nettests/core/http_keyword_filtering.py
+++ b/nettests/core/http_keyword_filtering.py
@@ -3,7 +3,14 @@
 # :authors: Arturo Filastò
 # :licence: see LICENSE
 
+from twisted.python import usage
+
 from ooni.templates import httpt
+
+class UsageOptions(usage.Options):
+    optParameters = [['backend', 'b', 'http://127.0.0.1:57001',
+                        'URL of the test backend to use']]
+
 class HTTPKeywordFiltering(httpt.HTTPTest):
     """
     This test involves performing HTTP requests containing to be tested for
@@ -14,29 +21,25 @@ class HTTPKeywordFiltering(httpt.HTTPTest):
     """
     name = "HTTP Keyword Filtering"
     author = "Arturo Filastò"
-    version = 0.1
-
-    optParameters = [['backend', 'b', None, 'URL of the backend system to use for testing']]
+    version = "0.1.1"
 
     inputFile = ['file', 'f', None, 'List of keywords to use for censorship testing']
 
-    def processInputs(self):
-        if 'backend' in self.localOptions:
-            self.url = self.localOptions['backend']
-        else:
-            raise Exception("No backend specified")
+    usageOptions = UsageOptions
+
+    requiredOptions = ['backend']
 
     def test_get(self):
         """
         Perform a HTTP GET request to the backend containing the keyword to be
         tested inside of the request body.
         """
-        return self.doRequest(self.url, method="GET", body=self.input)
+        return self.doRequest(self.localOptions['backend'], method="GET", body=self.input)
 
     def test_post(self):
         """
         Perform a HTTP POST request to the backend containing the keyword to be
         tested inside of the request body.
         """
-        return self.doRequest(self.url, method="POST", body=self.input)
+        return self.doRequest(self.localOptions['backend'], method="POST", body=self.input)
 
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 85f8586..7caab39 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -44,6 +44,10 @@ class HTTPTest(NetTestCase):
 
     randomizeUA = True
     followRedirects = False
+
+    baseParameters = [['socksproxy', 's', None,
+        'Specify a socks proxy to use for requests (ip:port)']]
+
     request = {}
     response = {}
 
@@ -55,13 +59,21 @@ class HTTPTest(NetTestCase):
             log.err("Warning! pyOpenSSL is not installed. https websites will"
                      "not work")
 
-        self.agent = Agent(reactor,
-                sockhost="127.0.0.1",
+        self.control_agent = Agent(reactor, sockhost="127.0.0.1",
                 sockport=config.advanced.tor_socksport)
 
+        sockshost, socksport = (None, None)
+        if self.localOptions['socksproxy']:
+            sockshost, socksport = self.localOptions['socksproxy'].split(':')
+            socksport = int(socksport)
+
+        self.agent = Agent(reactor, sockhost=sockshost,
+                sockport=socksport)
+
         if self.followRedirects:
             try:
                 from twisted.web.client import RedirectAgent
+                self.control_agent = RedirectAgent(self.control_agent)
                 self.agent = RedirectAgent(self.agent)
             except:
                 log.err("Warning! You are running an old version of twisted"\
@@ -150,6 +162,13 @@ class HTTPTest(NetTestCase):
         if use_tor:
             log.debug("Using tor for the request")
             url = 's'+url
+            agent = self.tor_agent
+        else:
+            agent = self.agent
+
+        if self.localOptions['socksproxy']:
+            log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy']))
+            url = 's'+url
 
         log.debug("Performing request %s %s %s" % (url, method, headers))
 
@@ -188,7 +207,7 @@ class HTTPTest(NetTestCase):
         def finished(data):
             return
 
-        d = self.agent.request(request['method'], request['url'], headers,
+        d = agent.request(request['method'], request['url'], headers,
                 body_producer)
 
         d.addErrback(errback)
@@ -198,7 +217,12 @@ class HTTPTest(NetTestCase):
 
     def _cbResponse(self, response, request, headers_processor,
             body_processor):
-        log.debug("Got response %s" % response)
+
+        if not response:
+            log.err("Got no response")
+            return
+        else:
+            log.debug("Got response %s" % response)
 
         if str(response.code).startswith('3'):
             self.processRedirect(response.headers.getRawHeaders('Location')[0])