commit 1de07f659f1393d969a1b3766baffeecb111355d Author: Arturo Filastò art@fuffa.org Date: Thu Nov 29 15:41:01 2012 +0100
Iterate on HTTP Invalid Request Line test * Rename it to HTTP Invalid Request Line * Extend it's fuzzing capabilities to support some more specific tests * Improve TCPT test template --- nettests/blocking/http_invalid_requests.py | 63 ------------ nettests/manipulation/http_invalid_request_line.py | 106 ++++++++++++++++++++ ooni/nettest.py | 38 +++++++- ooni/templates/httpt.py | 21 +---- ooni/templates/tcpt.py | 48 ++++++---- 5 files changed, 174 insertions(+), 102 deletions(-)
diff --git a/nettests/blocking/http_invalid_requests.py b/nettests/blocking/http_invalid_requests.py deleted file mode 100644 index 7e6f47f..0000000 --- a/nettests/blocking/http_invalid_requests.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- encoding: utf-8 -*- -from twisted.python import usage - -from ooni.utils import randomStr -from ooni.templates import tcpt - -class UsageOptions(usage.Options): - optParameters = [['backend', 'b', '127.0.0.1:57002', - 'The OONI backend that runs a TCP echo server (must be on port 80)']] - - optFlags = [['nopayloadmatch', 'n', - "Don't match the payload of the response. This option is used when you don't have a TCP echo server running"]] - -class HTTPInvalidRequests(tcpt.TCPTest): - name = "HTTP Invalid Requests" - version = "0.1.1" - authors = "Arturo Filastò" - - inputFile = ['file', 'f', None, - 'Input file of list of hostnames to attempt to resolve'] - - usageOptions = UsageOptions - requiredOptions = ['backend'] - - def setUp(self): - try: - self.address, self.port = self.localOptions['backend'].split(":") - self.port = int(self.port) - except: - raise usage.UsageError("Invalid backend address specified (must be address:port)") - - def test_random_invalid_request(self): - """ - We test sending data to a TCP echo server, if what we get back is not - what we have sent then there is tampering going on. - This is for example what squid will return when performing such - request: - - HTTP/1.0 400 Bad Request - Server: squid/2.6.STABLE21 - Date: Sat, 23 Jul 2011 02:22:44 GMT - Content-Type: text/html - Content-Length: 1178 - Expires: Sat, 23 Jul 2011 02:22:44 GMT - X-Squid-Error: ERR_INVALID_REQ 0 - X-Cache: MISS from cache_server - X-Cache-Lookup: NONE from cache_server:3128 - Via: 1.0 cache_server:3128 (squid/2.6.STABLE21) - Proxy-Connection: close - - """ - payload = randomStr(10) + "\n\r" - def got_all_data(received_array): - if not self.localOptions['nopayloadmatch']: - first = received_array[0] - if first != payload: - self.report['tampering'] = True - else: - self.report['tampering'] = 'unknown' - - d = self.sendPayload(payload) - d.addCallback(got_all_data) - return d diff --git a/nettests/manipulation/http_invalid_request_line.py b/nettests/manipulation/http_invalid_request_line.py new file mode 100644 index 0000000..00ab24d --- /dev/null +++ b/nettests/manipulation/http_invalid_request_line.py @@ -0,0 +1,106 @@ +# -*- encoding: utf-8 -*- +from twisted.python import usage + +from ooni.utils import randomStr +from ooni.templates import tcpt + +class UsageOptions(usage.Options): + optParameters = [['backend', 'b', '127.0.0.1', + 'The OONI backend that runs a TCP echo server']] + +class HTTPInvalidRequestLine(tcpt.TCPTest): + """ + The goal of this test is to do some very basic and not very noisy fuzzing + on the HTTP request line. We generate a series of requests that are not + valid HTTP requests. + + Unless elsewhere stated 'Xx'*N refers to N*2 random upper or lowercase ascii + letters or numbers ('XxXx' will be 4). + """ + name = "HTTP Invalid Requests" + version = "0.1.3" + authors = "Arturo Filastò" + + inputFile = ['file', 'f', None, + 'Input file of list of hostnames to attempt to resolve'] + + usageOptions = UsageOptions + requiredOptions = ['backend'] + + def setUp(self): + self.port = 80 + self.address = self.localOptions['backend'] + + def check_for_manipulation(self, response, payload): + if response != payload: + self.report['tampering'] = True + else: + self.report['tampering'] = 'unknown' + + def test_random_invalid_method(self): + """ + We test sending data to a TCP echo server listening on port 80, if what + we get back is not what we have sent then there is tampering going on. + This is for example what squid will return when performing such + request: + + HTTP/1.0 400 Bad Request + Server: squid/2.6.STABLE21 + Date: Sat, 23 Jul 2011 02:22:44 GMT + Content-Type: text/html + Content-Length: 1178 + Expires: Sat, 23 Jul 2011 02:22:44 GMT + X-Squid-Error: ERR_INVALID_REQ 0 + X-Cache: MISS from cache_server + X-Cache-Lookup: NONE from cache_server:3128 + Via: 1.0 cache_server:3128 (squid/2.6.STABLE21) + Proxy-Connection: close + + """ + payload = randomStr(10) + " / HTTP/1.1\n\r" + + d = self.sendPayload(payload) + d.addCallback(self.check_for_manipulation, payload) + return d + + def test_random_invalid_field_count(self): + """ + This generates a request that looks like this: + + XxXxX XxXxX XxXxX XxXxX + + This may trigger some bugs in the HTTP parsers of transparent HTTP + proxies. + """ + payload = ' '.join(randomStr(5) for x in range(4)) + payload += "\n\r" + + d = self.sendPayload(payload) + d.addCallback(self.check_for_manipulation, payload) + return d + + def test_random_big_request_method(self): + """ + This generates a request that looks like this: + + Xx*512 / HTTP/1.1 + """ + payload = randomStr(1024) + ' / HTTP/1.1\n\r' + + d = self.sendPayload(payload) + d.addCallback(self.check_for_manipulation, payload) + return d + + def test_random_invalid_version_number(self): + """ + This generates a request that looks like this: + + GET / HTTP/XxX + """ + payload = 'GET / HTTP/' + randomStr(3) + payload += '\n\r' + + d = self.sendPayload(payload) + d.addCallback(self.check_for_manipulation, payload) + return d + diff --git a/ooni/nettest.py b/ooni/nettest.py index e0393e7..4a54414 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -5,7 +5,6 @@ # In here is the NetTest API definition. This is how people # interested in writing ooniprobe tests will be specifying them # -# :authors: Arturo Filastò, Isis Lovecruft # :license: see included LICENSE file
import sys @@ -17,12 +16,47 @@ from twisted.trial import unittest, itrial, util from twisted.internet import defer, utils from twisted.python import usage
+from twisted.internet.error import ConnectionRefusedError, DNSLookupError, TCPTimedOutError + from ooni.utils import log
+def failureToString(failure): + """ + Given a failure instance return a string representing the kind of error + that occurred. + + Args: + + failure: a :class:twisted.internet.error instance + + Returns: + + A string representing the HTTP response error message. + """ + if isinstance(failure.value, ConnectionRefusedError): + log.err("Connection refused. The backend may be down") + string = 'connection_refused_error' + + elif isinstance(failure.value, SOCKSError): + log.err("Sock error. The SOCKS proxy may be down") + string = 'socks_error' + + elif isinstance(failure.value, DNSLookupError): + log.err("DNS lookup failure") + string = 'dns_lookup_error' + + elif isinstance(failure.value, TCPTimedOutError): + log.err("TCP Timed Out Error") + string = 'tcp_timed_out_error' + + elif isinstance(failure.value, ResponseNeverReceived): + log.err("Response Never Received") + string = 'response_never_received' + return string + class NoPostProcessor(Exception): pass
- class NetTestCase(object): """ This is the base of the OONI nettest universe. When you write a nettest diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 804a3e4..01853f3 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -23,6 +23,8 @@ from ooni import config from ooni.utils.net import BodyReceiver, StringProducer, userAgents
from ooni.utils.txagentwithsocks import Agent, SOCKSError, TrueHeaders +from ooni.nettest import failureToString +
class InvalidSocksProxyOption(Exception): pass @@ -131,25 +133,8 @@ class HTTPTest(NetTestCase): 'code': response.code } if failure: - if isinstance(failure.value, ConnectionRefusedError): - log.err("Connection refused. The backend may be down") - request_response['failure'] = 'connection_refused_error' - - elif isinstance(failure.value, SOCKSError): - log.err("Sock error. The SOCKS proxy may be down") - request_response['failure'] = 'socks_error' - - elif isinstance(failure.value, DNSLookupError): - log.err("DNS lookup failure") - request_response['failure'] = 'dns_lookup_error' - - elif isinstance(failure.value, TCPTimedOutError): - log.err("TCP Timed Out Error") - request_response['failure'] = 'tcp_timed_out_error' + request_response['failure'] = failureToString(failure)
- elif isinstance(failure.value, ResponseNeverReceived): - log.err("Response Never Received") - request_response['failure'] = 'response_never_received' self.report['requests'].append(request_response)
def _processResponseBody(self, response_body, request, response, body_processor): diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py index 77ffe3e..26f38ed 100644 --- a/ooni/templates/tcpt.py +++ b/ooni/templates/tcpt.py @@ -2,13 +2,14 @@ from twisted.internet import protocol, defer, reactor from twisted.internet.error import ConnectionDone from twisted.internet.endpoints import TCP4ClientEndpoint
-from ooni.nettest import NetTestCase +from ooni.nettest import NetTestCase, failureToString from ooni.utils import log
class TCPSender(protocol.Protocol): - report = None - payload_len = None - received_data = '' + def __init__(self): + self.received_data = '' + self.sent_data = '' + def dataReceived(self, data): """ We receive data until the total amount of data received reaches that @@ -27,18 +28,19 @@ class TCPSender(protocol.Protocol): """ if self.payload_len: self.received_data += data - if len(self.received_data) >= self.payload_len: - self.transport.loseConnection() - self.report['received'].append(data) - self.deferred.callback(self.report['received'])
def sendPayload(self, payload): """ Write the payload to the wire and set the expected size of the payload we are to receive. + + Args: + + payload: the data to be sent on the wire. + """ self.payload_len = len(payload) - self.report['sent'].append(payload) + self.sent_data = payload self.transport.write(payload)
class TCPSenderFactory(protocol.Factory): @@ -50,7 +52,7 @@ class TCPTest(NetTestCase): version = "0.1"
requiresRoot = False - timeout = 2 + timeout = 5 address = None port = None
@@ -61,26 +63,34 @@ class TCPTest(NetTestCase): def sendPayload(self, payload): d1 = defer.Deferred()
- def closeConnection(p): - p.transport.loseConnection() + def closeConnection(proto): + self.report['sent'].append(proto.sent_data) + self.report['received'].append(proto.received_data) + proto.transport.loseConnection() log.debug("Closing connection") d1.callback(self.report['received'])
+ def timedOut(proto): + self.report['failure'] = 'tcp_timed_out_error' + proto.transport.loseConnection() + def errback(failure): - self.report['error'] = str(failure) + self.report['failure'] = failureToString(failure) d1.errback(failure)
- def connected(p): + def connected(proto): log.debug("Connected to %s:%s" % (self.address, self.port)) - p.report = self.report - p.deferred = d1 - p.sendPayload(payload) - reactor.callLater(self.timeout, closeConnection, p) + proto.report = self.report + proto.deferred = d1 + proto.sendPayload(payload) + if self.timeout: + # XXX-Twisted this logic should probably go inside of the protocol + reactor.callLater(self.timeout, closeConnection, proto)
point = TCP4ClientEndpoint(reactor, self.address, self.port) + log.debug("Connecting to %s:%s" % (self.address, self.port)) d2 = point.connect(TCPSenderFactory()) d2.addCallback(connected) d2.addErrback(errback) return d1
-