[tor-commits] [ooni-probe/master] Get HTTP Requests test in a working state.
art at torproject.org
art at torproject.org
Thu Nov 22 22:27:22 UTC 2012
commit 01d80a2abc235fedbd2944500e259e537fd46c45
Author: Arturo Filastò <art at fuffa.org>
Date: Thu Nov 22 23:23:07 2012 +0100
Get HTTP Requests test in a working state.
* Performs random capitalization of a set of static HTTP Header fields and
measures on the backend if they are as expected.
* XXX the detection logic needs some more work
* XXX We need to keep track of the order in which we send the headers from the
HTTP Agent. Perhaps it makes sense to implement a specific HTTP Agent that
does not implement the full HTTP protocol like what we did for the backend
component.
---
nettests/core/http_requests.py | 90 +++++++++++++++++++++++++++++++-----
ooni/templates/httpt.py | 5 +-
ooni/utils/txagentwithsocks.py | 30 +++++++------
oonib/testhelpers/http_helpers.py | 9 +++-
4 files changed, 102 insertions(+), 32 deletions(-)
diff --git a/nettests/core/http_requests.py b/nettests/core/http_requests.py
index 0f3a2d8..5d67070 100644
--- a/nettests/core/http_requests.py
+++ b/nettests/core/http_requests.py
@@ -8,22 +8,26 @@ import json
from twisted.python import usage
-from ooni.utils import log, net
+from ooni.utils import log, net, randomStr
from ooni.templates import httpt
def random_capitalization(string):
output = ""
+ original_string = string
string = string.swapcase()
for i in range(len(string)):
if random.randint(0, 1):
output += string[i].swapcase()
else:
output += string[i]
- return output
+ if original_string == output:
+ return random_capitalization(output)
+ else:
+ return output
class UsageOptions(usage.Options):
optParameters = [
- ['backend', 'b', None,
+ ['backend', 'b', 'http://127.0.0.1:57001',
'URL of the backend to use for sending the requests'],
['headers', 'h', None,
'Specify a yaml formatted file from which to read the request headers to send']
@@ -36,8 +40,9 @@ class HTTPRequests(httpt.HTTPTest):
"""
name = "HTTP Requests"
author = "Arturo Filastò"
- version = 0.1
+ version = "0.1.1"
+ randomizeUA = False
usageOptions = UsageOptions
requiredOptions = ['backend']
@@ -49,12 +54,69 @@ class HTTPRequests(httpt.HTTPTest):
raise Exception("No backend specified")
def processResponseBody(self, data):
+ self.check_for_tampering(data)
+
+ def check_for_tampering(self, data):
+ """
+ Here we do checks to verify if the request we made has been tampered
+ with. We have 3 categories of tampering:
+
+ * **total** when the response is not a json object and therefore we were not
+ able to reach the ooniprobe test backend
+
+ * **request_line_capitalization** when the HTTP Request line (e.x. GET /
+ HTTP/1.1) does not match the capitalization we set.
+
+ * **header_field_number** when the number of headers we sent does not match
+ with the ones the backend received
+
+ * **header_name_capitalization** when the header field names do not match
+ those that we sent.
+
+ * **header_field_value** when the header field value does not match with the
+ one we transmitted.
+ """
+ self.report['tampering'] = {'total': False,
+ 'request_line_capitalization': False,
+ 'header_name_capitalization': False,
+ 'header_field_value': False,
+ 'header_field_number': False
+ }
+
try:
response = json.loads(data)
except ValueError:
- self.report['tampering'] = True
-
- # XXX add checks for validation of sent headers
+ self.report['tampering']['total'] = True
+ return
+
+ requestLine = "%s / HTTP/1.1" % self.request_method
+ if response['request_line'] != requestLine:
+ self.report['tampering']['request_line_capitalization'] = True
+
+ # We compare against length -1 because the response will also contain
+ # the Connection: close header since we do not do persistent
+ # connections
+ if len(self.request_headers) != (len(response['headers_dict']) - 1):
+ self.report['tampering']['header_field_number'] = True
+
+ for header, value in self.request_headers.items():
+ # XXX this still needs some work
+ # in particular if the response headers are of different length or
+ # some extra headers get added in the response (so the lengths
+ # match), we will get header_name_capitalization set to true, while
+ # the actual tampering is the addition of an extraneous header
+ # field.
+ if header == "Connection":
+ # Ignore Connection header
+ continue
+ try:
+ response_value = response['headers_dict'][header]
+ if response_value != value[0]:
+ log.msg("Tampering detected because %s != %s" % (response_value, value[0]))
+ self.report['tampering']['header_field_value'] = True
+ except KeyError:
+ log.msg("Tampering detected because %s not in %s" % (header, response['headers_dict']))
+ self.report['tampering']['header_name_capitalization'] = True
def get_headers(self):
headers = {}
@@ -69,11 +131,13 @@ class HTTPRequests(httpt.HTTPTest):
headers = yaml.load(content)
return headers
else:
- headers = {"User-Agent": [random.choice(net.userAgents)],
+ headers = {"User-Agent": [random.choice(net.userAgents)[0]],
"Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
"Accept-Encoding": ["gzip,deflate,sdch"],
"Accept-Language": ["en-US,en;q=0.8"],
- "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"]}
+ "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"],
+ "Host": [randomStr(15)+'.com']
+ }
return headers
def get_random_caps_headers(self):
@@ -90,25 +154,25 @@ class HTTPRequests(httpt.HTTPTest):
return self.doRequest(self.url, self.request_method,
headers=self.request_headers)
- def a_test_get_random_capitalization(self):
+ def test_get_random_capitalization(self):
self.request_method = random_capitalization("GET")
self.request_headers = self.get_random_caps_headers()
return self.doRequest(self.url, self.request_method,
headers=self.request_headers)
- def a_test_post(self):
+ def test_post(self):
self.request_method = "POST"
self.request_headers = self.get_headers()
return self.doRequest(self.url, self.request_method,
headers=self.request_headers)
- def a_test_post_random_capitalization(self):
+ def test_post_random_capitalization(self):
self.request_method = random_capitalization("POST")
self.request_headers = self.get_random_caps_headers()
return self.doRequest(self.url, self.request_method,
headers=self.request_headers)
- def a_test_put(self):
+ def test_put(self):
self.request_method = "PUT"
self.request_headers = self.get_headers()
return self.doRequest(self.url, self.request_method,
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 46bde28..e166263 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -15,7 +15,6 @@ from twisted.internet import reactor
from twisted.internet.error import ConnectionRefusedError
from twisted.web._newclient import Request
-from twisted.web.http_headers import Headers
from ooni.nettest import NetTestCase
from ooni.utils import log
@@ -23,7 +22,7 @@ from ooni import config
from ooni.utils.net import BodyReceiver, StringProducer, userAgents
-from ooni.utils.txagentwithsocks import Agent, SOCKSError
+from ooni.utils.txagentwithsocks import Agent, SOCKSError, TrueHeaders
class HTTPTest(NetTestCase):
"""
@@ -195,7 +194,7 @@ class HTTPTest(NetTestCase):
else:
body_producer = None
- headers = Headers(request['headers'])
+ headers = TrueHeaders(request['headers'])
def errback(failure):
failure.trap(ConnectionRefusedError, SOCKSError)
diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py
index fc1c17c..05da54f 100644
--- a/ooni/utils/txagentwithsocks.py
+++ b/ooni/utils/txagentwithsocks.py
@@ -50,13 +50,13 @@ class SOCKSv5ClientProtocol(_WrappingProtocol):
errcode = ord(data[1])
self._connectedDeferred.errback(SOCKSError(errcode))
-
+
return
self.ready = True
self._wrappedProtocol.transport = self.transport
self._wrappedProtocol.connectionMade()
-
+
self._connectedDeferred.callback(self._wrappedProtocol)
def connectionMade(self):
@@ -81,7 +81,7 @@ class SOCKSv5ClientProtocol(_WrappingProtocol):
class SOCKSv5ClientFactory(_WrappingFactory):
protocol = SOCKSv5ClientProtocol
-
+
def __init__(self, wrappedFactory, host, port):
_WrappingFactory.__init__(self, wrappedFactory)
self._host, self._port = host, port
@@ -119,7 +119,7 @@ class SOCKS5ClientEndpoint(object):
except:
return defer.fail()
-class Headers(http_headers.Headers):
+class TrueHeaders(http_headers.Headers):
def __init__(self, rawHeaders=None):
self._rawHeaders = dict()
if rawHeaders is not None:
@@ -141,12 +141,13 @@ class Headers(http_headers.Headers):
def getRawHeaders(self, name, default=None):
if name.lower() in self._rawHeaders:
- return self._rawHeaders[name.lower()]["values"]
+ return self._rawHeaders[name.lower()]['values']
return default
+
class HTTPClientParser(_newclient.HTTPClientParser):
def connectionMade(self):
- self.headers = Headers()
- self.connHeaders = Headers()
+ self.headers = TrueHeaders()
+ self.connHeaders = TrueHeaders()
self.state = STATUS
self._partialHeader = None
@@ -224,31 +225,32 @@ class Agent(client.Agent):
return TCP4ClientEndpoint(self._reactor, host, port, **kwargs)
elif scheme == 'shttp':
return SOCKS5ClientEndpoint(self._reactor, self._sockshost,
- self._socksport, host, port, **kwargs)
+ self._socksport, host, port, **kwargs)
elif scheme == 'httpo':
return SOCKS5ClientEndpoint(self._reactor, self._sockshost,
- self._socksport, host, port, **kwargs)
+ self._socksport, host, port, **kwargs)
elif scheme == 'https':
return SSL4ClientEndpoint(self._reactor, host, port,
- self._wrapContextFactory(host, port),
- **kwargs)
+ self._wrapContextFactory(host, port), **kwargs)
else:
raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,))
def _requestWithEndpoint(self, key, endpoint, method, parsedURI,
headers, bodyProducer, requestPath):
if headers is None:
- headers = Headers()
+ headers = TrueHeaders()
if not headers.hasHeader('host'):
headers = headers.copy()
headers.addRawHeader(
- 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host,
- parsedURI.port))
+ 'host', self._computeHostValue(parsedURI.scheme,
+ parsedURI.host, parsedURI.port))
d = self._pool.getConnection(key, endpoint)
+ print headers._rawHeaders
def cbConnected(proto):
return proto.request(
Request(method, requestPath, headers, bodyProducer,
persistent=self._pool.persistent))
d.addCallback(cbConnected)
return d
+
diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py
index cbd6caa..b384216 100644
--- a/oonib/testhelpers/http_helpers.py
+++ b/oonib/testhelpers/http_helpers.py
@@ -72,12 +72,17 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin):
def headerReceived(self, line):
header, data = line.split(':', 1)
- self.headers.append((header, data))
+ self.headers.append((header, data.strip()))
def allHeadersReceived(self):
+ headers_dict = {}
+ for k, v in self.headers:
+ headers_dict[k] = v
response = {'request_headers': self.headers,
- 'request_line': self.requestLine
+ 'request_line': self.requestLine,
+ 'headers_dict': headers_dict
}
+ self.transport.write('HTTP/1.1 200 OK\r\n\r\n')
self.transport.write(json.dumps(response))
self.transport.loseConnection()
More information about the tor-commits
mailing list