[tor-commits] [ooni-probe/master] Implement basic HTTP request test that does capitalization variations on the HTTP method

art at torproject.org art at torproject.org
Sat Nov 10 17:21:20 UTC 2012


commit 2266e961595a167b12ee1a4aca617eb2dbb56b12
Author: Arturo Filastò <art at fuffa.org>
Date:   Sat Nov 10 17:50:22 2012 +0100

    Implement basic HTTP request test that does capitalization variations on the HTTP method
    * Due to a quirk in twisted.client.Agent that sets HTTP request headers to canonical form we are
      currently unable to test the request headers.
      This is ticketized here: https://trac.torproject.org/projects/tor/ticket/7432
---
 nettests/core/http_requests.py |   90 +++++++++++++++++++++++++++++++++-------
 ooni/templates/httpt.py        |   58 ++++++++++++++------------
 oonib/testhelpers/httph.py     |   26 ++++++++----
 3 files changed, 124 insertions(+), 50 deletions(-)

diff --git a/nettests/core/http_requests.py b/nettests/core/http_requests.py
index 4fc8205..0315145 100644
--- a/nettests/core/http_requests.py
+++ b/nettests/core/http_requests.py
@@ -4,6 +4,9 @@
 # :licence: see LICENSE
 
 import random
+import json
+
+from ooni.utils import log, net
 from ooni.templates import httpt
 
 def random_capitalization(string):
@@ -21,11 +24,16 @@ class HTTPRequests(httpt.HTTPTest):
     This test is also known as Header Field manipulation. It performes HTTP
     requests with variations in capitalization towards the backend.
     """
-    name = "HTTPRequests"
+    name = "HTTP Requests"
     author = "Arturo Filastò"
     version = 0.1
 
-    optParameters = [['backend', 'b', None, 'URL of the backend to use for sending the requests']]
+    optParameters = [
+            ['backend', 'b', None, 
+                'URL of the backend to use for sending the requests'],
+            ['headers', 'h', None,
+                'Specify a yaml formatted file from which to read the request headers to send']
+            ]
 
     requiredOptions = ['backend']
 
@@ -35,24 +43,76 @@ class HTTPRequests(httpt.HTTPTest):
         else:
             raise Exception("No backend specified")
 
+    def processResponseBody(self, data):
+        response = json.loads(data)
+        if response['request_method'] != self.request_method:
+            self.report['tampering'] = True
+        else:
+            self.report['tampering'] = False
+        # XXX add checks for validation of sent headers
+
+    def get_headers(self):
+        headers = {}
+        if self.localOptions['headers']:
+            # XXX test this code
+            try:
+                f = open(self.localOptions['headers'])
+            except IOError:
+                raise Exception("Specified input file does not exist")
+            content = ''.join(f.readlines())
+            f.close()
+            headers = yaml.load(content)
+            return headers
+        else:
+            headers = {"User-Agent": [random.choice(net.userAgents)],
+                "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
+                "Accept-Encoding": ["gzip,deflate,sdch"],
+                "Accept-Language": ["en-US,en;q=0.8"],
+                "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"]}
+            return headers
+
+    def get_random_caps_headers(self):
+        headers = {}
+        normal_headers = self.get_headers()
+        for k, v in normal_headers.items():
+            new_key = random_capitalization(k)
+            headers[new_key] = v
+        return headers
+
     def test_get(self):
-        return self.doRequest(self.url, "GET")
+        self.request_method = "GET"
+        self.request_headers = self.get_random_caps_headers()
+        return self.doRequest(self.url, self.request_method, 
+                headers=self.request_headers)
 
-    def test_get_random_capitalization(self):
-        method = random_capitalization("GET")
-        return self.doRequest(self.url, method)
+    def a_test_get_random_capitalization(self):
+        self.request_method = random_capitalization("GET")
+        self.request_headers = self.get_random_caps_headers()
+        return self.doRequest(self.url, self.request_method, 
+                headers=self.request_headers)
 
-    def test_post(self):
-        return self.doRequest(self.url, "POST")
+    def a_test_post(self):
+        self.request_method = "POST"
+        self.request_headers = self.get_headers()
+        return self.doRequest(self.url, self.request_method, 
+                headers=self.request_headers)
 
-    def test_post_random_capitalization(self):
-        method = random_capitalization("POST")
-        return self.doRequest(self.url, method)
+    def a_test_post_random_capitalization(self):
+        self.request_method = random_capitalization("POST")
+        self.request_headers = self.get_random_caps_headers()
+        return self.doRequest(self.url, self.request_method, 
+                headers=self.request_headers)
 
-    def test_put(self):
-        return self.doRequest(self.url, "PUT")
+    def a_test_put(self):
+        self.request_method = "PUT"
+        self.request_headers = self.get_headers()
+        return self.doRequest(self.url, self.request_method, 
+                headers=self.request_headers)
 
     def test_put_random_capitalization(self):
-        method = random_capitalization("PUT")
-        return self.doRequest(self.url, method)
+        self.request_method = random_capitalization("PUT")
+        self.request_headers = self.get_random_caps_headers()
+        return self.doRequest(self.url, self.request_method, 
+                headers=self.request_headers)
+
 
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 4c42a3a..96b7cc8 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -2,7 +2,7 @@
 #
 # :authors: Arturo Filastò
 # :licence: see LICENSE
-
+import copy
 import random
 
 from zope.interface import implements
@@ -12,6 +12,10 @@ from twisted.plugin import IPlugin
 from twisted.internet import protocol, defer
 from twisted.internet.ssl import ClientContextFactory
 
+from twisted.web.client import Agent
+from twisted.internet import reactor
+
+from twisted.web._newclient import Request
 from twisted.web.http_headers import Headers
 from ooni.nettest import NetTestCase
 from ooni.utils import log
@@ -39,8 +43,6 @@ class HTTPTest(NetTestCase):
         except:
             log.err("Warning! pyOpenSSL is not installed. https websites will"
                      "not work")
-        from twisted.web.client import Agent
-        from twisted.internet import reactor
 
         self.agent = Agent(reactor)
 
@@ -107,7 +109,7 @@ class HTTPTest(NetTestCase):
 
         method: the HTTP Method to be used
 
-        headers: the request headers to be sent
+        headers: the request headers to be sent as a dict
 
         body: the request body
 
@@ -137,6 +139,31 @@ class HTTPTest(NetTestCase):
         d.addCallback(finished)
         return d
 
+    def build_request(self, url, method="GET", headers=None, body=None):
+        self.request['method'] = method
+        self.request['url'] = url
+        self.request['headers'] = headers if headers else {}
+        self.request['body'] = body
+
+        if self.randomizeUA:
+            self.randomize_useragent()
+
+        self.report['request'] = self.request
+        self.report['url'] = url
+
+        # If we have a request body payload, set the request body to such
+        # content
+        if body:
+            body_producer = StringProducer(self.request['body'])
+        else:
+            body_producer = None
+
+        headers = Headers(self.request['headers'])
+
+        req = self.agent.request(self.request['method'], self.request['url'],
+                                  headers, body_producer)
+        return req
+
     def _cbResponse(self, response, headers_processor, body_processor):
         log.debug("Got response %s" % response)
         if not response:
@@ -167,27 +194,4 @@ class HTTPTest(NetTestCase):
         user_agent = random.choice(userAgents)
         self.request['headers']['User-Agent'] = [user_agent]
 
-    def build_request(self, url, method="GET", headers=None, body=None):
-        self.request['method'] = method
-        self.request['url'] = url
-        self.request['headers'] = headers if headers else {}
-        self.request['body'] = body
-
-        if self.randomizeUA:
-            self.randomize_useragent()
-
-        self.report['request'] = self.request
-        self.report['url'] = url
-
-        # If we have a request body payload, set the request body to such
-        # content
-        if body:
-            body_producer = StringProducer(self.request['body'])
-        else:
-            body_producer = None
-
-        req = self.agent.request(self.request['method'], self.request['url'],
-                                  Headers(self.request['headers']),
-                                  body_producer)
-        return req
 
diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py
index b793852..7001fa6 100644
--- a/oonib/testhelpers/httph.py
+++ b/oonib/testhelpers/httph.py
@@ -12,16 +12,26 @@ from cyclone.web import RequestHandler, Application
 
 class HTTPTrapAll(RequestHandler):
     """
-    Master class to be used to trap all the HTTP methods
+    Master class to be used to trap all the HTTP methods and make capitalized
+    requests pass.
     """
-    def get(self, *arg, **kw):
-        self.all(*arg, **kw)
+    def _execute(self, transforms, *args, **kwargs):
+        self._transforms = transforms
+        defer.maybeDeferred(self.prepare).addCallbacks(
+                    self._execute_handler,
+                    lambda f: self._handle_request_exception(f.value),
+                    callbackArgs=(args, kwargs))
 
-    def post(self, *arg, **kw):
-        self.all(*arg, **kw)
-
-    def put(self, *arg, **kw):
-        self.all(*arg, **kw)
+    def _execute_handler(self, r, args, kwargs):
+        if not self._finished:
+            args = [self.decode_argument(arg) for arg in args]
+            kwargs = dict((k, self.decode_argument(v, name=k))
+                            for (k, v) in kwargs.iteritems())
+            # This is where we do the patching
+            # XXX this is somewhat hackish
+            d = defer.maybeDeferred(self.all, *args, **kwargs)
+            d.addCallbacks(self._execute_success, self._execute_failure)
+            self.notifyFinish().addCallback(self.on_connection_close)
 
 class HTTPReturnJSONHeaders(HTTPTrapAll):
     def all(self):





More information about the tor-commits mailing list