commit 01d80a2abc235fedbd2944500e259e537fd46c45 Author: Arturo Filastò art@fuffa.org Date: Thu Nov 22 23:23:07 2012 +0100

Get HTTP Requests test in a working state. * Performs random capitalization of a set of static HTTP Header fields and measures on the backend if they are as expected. * XXX the detection logic needs some more work * XXX We need to keep track of the order in which we send the headers from the HTTP Agent. Perhaps it makes sense to implement a specific HTTP Agent that does not implement the full HTTP protocol like what we did for the backend component. --- nettests/core/http_requests.py | 90 +++++++++++++++++++++++++++++++----- ooni/templates/httpt.py | 5 +- ooni/utils/txagentwithsocks.py | 30 +++++++------ oonib/testhelpers/http_helpers.py | 9 +++- 4 files changed, 102 insertions(+), 32 deletions(-)

diff --git a/nettests/core/http_requests.py b/nettests/core/http_requests.py index 0f3a2d8..5d67070 100644 --- a/nettests/core/http_requests.py +++ b/nettests/core/http_requests.py @@ -8,22 +8,26 @@ import json

from twisted.python import usage

-from ooni.utils import log, net +from ooni.utils import log, net, randomStr from ooni.templates import httpt

def random_capitalization(string): output = "" + original_string = string string = string.swapcase() for i in range(len(string)): if random.randint(0, 1): output += string[i].swapcase() else: output += string[i] - return output + if original_string == output: + return random_capitalization(output) + else: + return output

class UsageOptions(usage.Options): optParameters = [ - ['backend', 'b', None, + ['backend', 'b', 'http://127.0.0.1:57001', 'URL of the backend to use for sending the requests'], ['headers', 'h', None, 'Specify a yaml formatted file from which to read the request headers to send'] @@ -36,8 +40,9 @@ class HTTPRequests(httpt.HTTPTest): """ name = "HTTP Requests" author = "Arturo Filastò" - version = 0.1 + version = "0.1.1"

+ randomizeUA = False usageOptions = UsageOptions

requiredOptions = ['backend'] @@ -49,12 +54,69 @@ class HTTPRequests(httpt.HTTPTest): raise Exception("No backend specified")

def processResponseBody(self, data): + self.check_for_tampering(data) + + def check_for_tampering(self, data): + """ + Here we do checks to verify if the request we made has been tampered + with. We have 3 categories of tampering: + + * **total** when the response is not a json object and therefore we were not + able to reach the ooniprobe test backend + + * **request_line_capitalization** when the HTTP Request line (e.x. GET / + HTTP/1.1) does not match the capitalization we set. + + * **header_field_number** when the number of headers we sent does not match + with the ones the backend received + + * **header_name_capitalization** when the header field names do not match + those that we sent. + + * **header_field_value** when the header field value does not match with the + one we transmitted. + """ + self.report['tampering'] = {'total': False, + 'request_line_capitalization': False, + 'header_name_capitalization': False, + 'header_field_value': False, + 'header_field_number': False + } + try: response = json.loads(data) except ValueError: - self.report['tampering'] = True - - # XXX add checks for validation of sent headers + self.report['tampering']['total'] = True + return + + requestLine = "%s / HTTP/1.1" % self.request_method + if response['request_line'] != requestLine: + self.report['tampering']['request_line_capitalization'] = True + + # We compare against length -1 because the response will also contain + # the Connection: close header since we do not do persistent + # connections + if len(self.request_headers) != (len(response['headers_dict']) - 1): + self.report['tampering']['header_field_number'] = True + + for header, value in self.request_headers.items(): + # XXX this still needs some work + # in particular if the response headers are of different length or + # some extra headers get added in the response (so the lengths + # match), we will get header_name_capitalization set to true, while + # the actual tampering is the addition of an extraneous header + # field. + if header == "Connection": + # Ignore Connection header + continue + try: + response_value = response['headers_dict'][header] + if response_value != value[0]: + log.msg("Tampering detected because %s != %s" % (response_value, value[0])) + self.report['tampering']['header_field_value'] = True + except KeyError: + log.msg("Tampering detected because %s not in %s" % (header, response['headers_dict'])) + self.report['tampering']['header_name_capitalization'] = True

def get_headers(self): headers = {} @@ -69,11 +131,13 @@ class HTTPRequests(httpt.HTTPTest): headers = yaml.load(content) return headers else: - headers = {"User-Agent": [random.choice(net.userAgents)], + headers = {"User-Agent": [random.choice(net.userAgents)[0]], "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], "Accept-Encoding": ["gzip,deflate,sdch"], "Accept-Language": ["en-US,en;q=0.8"], - "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"]} + "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"], + "Host": [randomStr(15)+'.com'] + } return headers

def get_random_caps_headers(self): @@ -90,25 +154,25 @@ class HTTPRequests(httpt.HTTPTest): return self.doRequest(self.url, self.request_method, headers=self.request_headers)

- def a_test_get_random_capitalization(self): + def test_get_random_capitalization(self): self.request_method = random_capitalization("GET") self.request_headers = self.get_random_caps_headers() return self.doRequest(self.url, self.request_method, headers=self.request_headers)

- def a_test_post(self): + def test_post(self): self.request_method = "POST" self.request_headers = self.get_headers() return self.doRequest(self.url, self.request_method, headers=self.request_headers)

- def a_test_post_random_capitalization(self): + def test_post_random_capitalization(self): self.request_method = random_capitalization("POST") self.request_headers = self.get_random_caps_headers() return self.doRequest(self.url, self.request_method, headers=self.request_headers)

- def a_test_put(self): + def test_put(self): self.request_method = "PUT" self.request_headers = self.get_headers() return self.doRequest(self.url, self.request_method, diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 46bde28..e166263 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -15,7 +15,6 @@ from twisted.internet import reactor from twisted.internet.error import ConnectionRefusedError

from twisted.web._newclient import Request -from twisted.web.http_headers import Headers

from ooni.nettest import NetTestCase from ooni.utils import log @@ -23,7 +22,7 @@ from ooni import config

from ooni.utils.net import BodyReceiver, StringProducer, userAgents

-from ooni.utils.txagentwithsocks import Agent, SOCKSError +from ooni.utils.txagentwithsocks import Agent, SOCKSError, TrueHeaders

class HTTPTest(NetTestCase): """ @@ -195,7 +194,7 @@ class HTTPTest(NetTestCase): else: body_producer = None

- headers = Headers(request['headers']) + headers = TrueHeaders(request['headers'])

def errback(failure): failure.trap(ConnectionRefusedError, SOCKSError) diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py index fc1c17c..05da54f 100644 --- a/ooni/utils/txagentwithsocks.py +++ b/ooni/utils/txagentwithsocks.py @@ -50,13 +50,13 @@ class SOCKSv5ClientProtocol(_WrappingProtocol):

errcode = ord(data[1]) self._connectedDeferred.errback(SOCKSError(errcode)) - + return

self.ready = True self._wrappedProtocol.transport = self.transport self._wrappedProtocol.connectionMade() - + self._connectedDeferred.callback(self._wrappedProtocol)

def connectionMade(self): @@ -81,7 +81,7 @@ class SOCKSv5ClientProtocol(_WrappingProtocol):

class SOCKSv5ClientFactory(_WrappingFactory): protocol = SOCKSv5ClientProtocol - + def __init__(self, wrappedFactory, host, port): _WrappingFactory.__init__(self, wrappedFactory) self._host, self._port = host, port @@ -119,7 +119,7 @@ class SOCKS5ClientEndpoint(object): except: return defer.fail()

-class Headers(http_headers.Headers): +class TrueHeaders(http_headers.Headers): def __init__(self, rawHeaders=None): self._rawHeaders = dict() if rawHeaders is not None: @@ -141,12 +141,13 @@ class Headers(http_headers.Headers):

def getRawHeaders(self, name, default=None): if name.lower() in self._rawHeaders: - return self._rawHeaders[name.lower()]["values"] + return self._rawHeaders[name.lower()]['values'] return default + class HTTPClientParser(_newclient.HTTPClientParser): def connectionMade(self): - self.headers = Headers() - self.connHeaders = Headers() + self.headers = TrueHeaders() + self.connHeaders = TrueHeaders() self.state = STATUS self._partialHeader = None

@@ -224,31 +225,32 @@ class Agent(client.Agent): return TCP4ClientEndpoint(self._reactor, host, port, **kwargs) elif scheme == 'shttp': return SOCKS5ClientEndpoint(self._reactor, self._sockshost, - self._socksport, host, port, **kwargs) + self._socksport, host, port, **kwargs) elif scheme == 'httpo': return SOCKS5ClientEndpoint(self._reactor, self._sockshost, - self._socksport, host, port, **kwargs) + self._socksport, host, port, **kwargs) elif scheme == 'https': return SSL4ClientEndpoint(self._reactor, host, port, - self._wrapContextFactory(host, port), - **kwargs) + self._wrapContextFactory(host, port), **kwargs) else: raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,))

def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath): if headers is None: - headers = Headers() + headers = TrueHeaders() if not headers.hasHeader('host'): headers = headers.copy() headers.addRawHeader( - 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host, - parsedURI.port)) + 'host', self._computeHostValue(parsedURI.scheme, + parsedURI.host, parsedURI.port))

d = self._pool.getConnection(key, endpoint) + print headers._rawHeaders def cbConnected(proto): return proto.request( Request(method, requestPath, headers, bodyProducer, persistent=self._pool.persistent)) d.addCallback(cbConnected) return d + diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py index cbd6caa..b384216 100644 --- a/oonib/testhelpers/http_helpers.py +++ b/oonib/testhelpers/http_helpers.py @@ -72,12 +72,17 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin):

def headerReceived(self, line): header, data = line.split(':', 1) - self.headers.append((header, data)) + self.headers.append((header, data.strip()))

def allHeadersReceived(self): + headers_dict = {} + for k, v in self.headers: + headers_dict[k] = v response = {'request_headers': self.headers, - 'request_line': self.requestLine + 'request_line': self.requestLine, + 'headers_dict': headers_dict } + self.transport.write('HTTP/1.1 200 OK\r\n\r\n') self.transport.write(json.dumps(response)) self.transport.loseConnection()