commit 2266e961595a167b12ee1a4aca617eb2dbb56b12 Author: Arturo Filastò art@fuffa.org Date: Sat Nov 10 17:50:22 2012 +0100
Implement basic HTTP request test that does capitalization variations on the HTTP method * Due to a quirk in twisted.client.Agent that sets HTTP request headers to canonical form we are currently unable to test the request headers. This is ticketized here: https://trac.torproject.org/projects/tor/ticket/7432 --- nettests/core/http_requests.py | 90 +++++++++++++++++++++++++++++++++------- ooni/templates/httpt.py | 58 ++++++++++++++------------ oonib/testhelpers/httph.py | 26 ++++++++---- 3 files changed, 124 insertions(+), 50 deletions(-)
diff --git a/nettests/core/http_requests.py b/nettests/core/http_requests.py index 4fc8205..0315145 100644 --- a/nettests/core/http_requests.py +++ b/nettests/core/http_requests.py @@ -4,6 +4,9 @@ # :licence: see LICENSE
import random +import json + +from ooni.utils import log, net from ooni.templates import httpt
def random_capitalization(string): @@ -21,11 +24,16 @@ class HTTPRequests(httpt.HTTPTest): This test is also known as Header Field manipulation. It performes HTTP requests with variations in capitalization towards the backend. """ - name = "HTTPRequests" + name = "HTTP Requests" author = "Arturo Filastò" version = 0.1
- optParameters = [['backend', 'b', None, 'URL of the backend to use for sending the requests']] + optParameters = [ + ['backend', 'b', None, + 'URL of the backend to use for sending the requests'], + ['headers', 'h', None, + 'Specify a yaml formatted file from which to read the request headers to send'] + ]
requiredOptions = ['backend']
@@ -35,24 +43,76 @@ class HTTPRequests(httpt.HTTPTest): else: raise Exception("No backend specified")
+ def processResponseBody(self, data): + response = json.loads(data) + if response['request_method'] != self.request_method: + self.report['tampering'] = True + else: + self.report['tampering'] = False + # XXX add checks for validation of sent headers + + def get_headers(self): + headers = {} + if self.localOptions['headers']: + # XXX test this code + try: + f = open(self.localOptions['headers']) + except IOError: + raise Exception("Specified input file does not exist") + content = ''.join(f.readlines()) + f.close() + headers = yaml.load(content) + return headers + else: + headers = {"User-Agent": [random.choice(net.userAgents)], + "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], + "Accept-Encoding": ["gzip,deflate,sdch"], + "Accept-Language": ["en-US,en;q=0.8"], + "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"]} + return headers + + def get_random_caps_headers(self): + headers = {} + normal_headers = self.get_headers() + for k, v in normal_headers.items(): + new_key = random_capitalization(k) + headers[new_key] = v + return headers + def test_get(self): - return self.doRequest(self.url, "GET") + self.request_method = "GET" + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers)
- def test_get_random_capitalization(self): - method = random_capitalization("GET") - return self.doRequest(self.url, method) + def a_test_get_random_capitalization(self): + self.request_method = random_capitalization("GET") + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers)
- def test_post(self): - return self.doRequest(self.url, "POST") + def a_test_post(self): + self.request_method = "POST" + self.request_headers = self.get_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers)
- def test_post_random_capitalization(self): - method = random_capitalization("POST") - return self.doRequest(self.url, method) + def a_test_post_random_capitalization(self): + self.request_method = random_capitalization("POST") + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers)
- def test_put(self): - return self.doRequest(self.url, "PUT") + def a_test_put(self): + self.request_method = "PUT" + self.request_headers = self.get_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers)
def test_put_random_capitalization(self): - method = random_capitalization("PUT") - return self.doRequest(self.url, method) + self.request_method = random_capitalization("PUT") + self.request_headers = self.get_random_caps_headers() + return self.doRequest(self.url, self.request_method, + headers=self.request_headers) +
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 4c42a3a..96b7cc8 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -2,7 +2,7 @@ # # :authors: Arturo Filastò # :licence: see LICENSE - +import copy import random
from zope.interface import implements @@ -12,6 +12,10 @@ from twisted.plugin import IPlugin from twisted.internet import protocol, defer from twisted.internet.ssl import ClientContextFactory
+from twisted.web.client import Agent +from twisted.internet import reactor + +from twisted.web._newclient import Request from twisted.web.http_headers import Headers from ooni.nettest import NetTestCase from ooni.utils import log @@ -39,8 +43,6 @@ class HTTPTest(NetTestCase): except: log.err("Warning! pyOpenSSL is not installed. https websites will" "not work") - from twisted.web.client import Agent - from twisted.internet import reactor
self.agent = Agent(reactor)
@@ -107,7 +109,7 @@ class HTTPTest(NetTestCase):
method: the HTTP Method to be used
- headers: the request headers to be sent + headers: the request headers to be sent as a dict
body: the request body
@@ -137,6 +139,31 @@ class HTTPTest(NetTestCase): d.addCallback(finished) return d
+ def build_request(self, url, method="GET", headers=None, body=None): + self.request['method'] = method + self.request['url'] = url + self.request['headers'] = headers if headers else {} + self.request['body'] = body + + if self.randomizeUA: + self.randomize_useragent() + + self.report['request'] = self.request + self.report['url'] = url + + # If we have a request body payload, set the request body to such + # content + if body: + body_producer = StringProducer(self.request['body']) + else: + body_producer = None + + headers = Headers(self.request['headers']) + + req = self.agent.request(self.request['method'], self.request['url'], + headers, body_producer) + return req + def _cbResponse(self, response, headers_processor, body_processor): log.debug("Got response %s" % response) if not response: @@ -167,27 +194,4 @@ class HTTPTest(NetTestCase): user_agent = random.choice(userAgents) self.request['headers']['User-Agent'] = [user_agent]
- def build_request(self, url, method="GET", headers=None, body=None): - self.request['method'] = method - self.request['url'] = url - self.request['headers'] = headers if headers else {} - self.request['body'] = body - - if self.randomizeUA: - self.randomize_useragent() - - self.report['request'] = self.request - self.report['url'] = url - - # If we have a request body payload, set the request body to such - # content - if body: - body_producer = StringProducer(self.request['body']) - else: - body_producer = None - - req = self.agent.request(self.request['method'], self.request['url'], - Headers(self.request['headers']), - body_producer) - return req
diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py index b793852..7001fa6 100644 --- a/oonib/testhelpers/httph.py +++ b/oonib/testhelpers/httph.py @@ -12,16 +12,26 @@ from cyclone.web import RequestHandler, Application
class HTTPTrapAll(RequestHandler): """ - Master class to be used to trap all the HTTP methods + Master class to be used to trap all the HTTP methods and make capitalized + requests pass. """ - def get(self, *arg, **kw): - self.all(*arg, **kw) + def _execute(self, transforms, *args, **kwargs): + self._transforms = transforms + defer.maybeDeferred(self.prepare).addCallbacks( + self._execute_handler, + lambda f: self._handle_request_exception(f.value), + callbackArgs=(args, kwargs))
- def post(self, *arg, **kw): - self.all(*arg, **kw) - - def put(self, *arg, **kw): - self.all(*arg, **kw) + def _execute_handler(self, r, args, kwargs): + if not self._finished: + args = [self.decode_argument(arg) for arg in args] + kwargs = dict((k, self.decode_argument(v, name=k)) + for (k, v) in kwargs.iteritems()) + # This is where we do the patching + # XXX this is somewhat hackish + d = defer.maybeDeferred(self.all, *args, **kwargs) + d.addCallbacks(self._execute_success, self._execute_failure) + self.notifyFinish().addCallback(self.on_connection_close)
class HTTPReturnJSONHeaders(HTTPTrapAll): def all(self):