[tor-commits] [ooni-probe/master] Iterate on HTTP Invalid Request Line test
art at torproject.org
art at torproject.org
Thu Nov 29 14:42:49 UTC 2012
commit 1de07f659f1393d969a1b3766baffeecb111355d
Author: Arturo Filastò <art at fuffa.org>
Date: Thu Nov 29 15:41:01 2012 +0100
Iterate on HTTP Invalid Request Line test
* Rename it to HTTP Invalid Request Line
* Extend it's fuzzing capabilities to support some more specific tests
* Improve TCPT test template
---
nettests/blocking/http_invalid_requests.py | 63 ------------
nettests/manipulation/http_invalid_request_line.py | 106 ++++++++++++++++++++
ooni/nettest.py | 38 +++++++-
ooni/templates/httpt.py | 21 +----
ooni/templates/tcpt.py | 48 ++++++----
5 files changed, 174 insertions(+), 102 deletions(-)
diff --git a/nettests/blocking/http_invalid_requests.py b/nettests/blocking/http_invalid_requests.py
deleted file mode 100644
index 7e6f47f..0000000
--- a/nettests/blocking/http_invalid_requests.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# -*- encoding: utf-8 -*-
-from twisted.python import usage
-
-from ooni.utils import randomStr
-from ooni.templates import tcpt
-
-class UsageOptions(usage.Options):
- optParameters = [['backend', 'b', '127.0.0.1:57002',
- 'The OONI backend that runs a TCP echo server (must be on port 80)']]
-
- optFlags = [['nopayloadmatch', 'n',
- "Don't match the payload of the response. This option is used when you don't have a TCP echo server running"]]
-
-class HTTPInvalidRequests(tcpt.TCPTest):
- name = "HTTP Invalid Requests"
- version = "0.1.1"
- authors = "Arturo Filastò"
-
- inputFile = ['file', 'f', None,
- 'Input file of list of hostnames to attempt to resolve']
-
- usageOptions = UsageOptions
- requiredOptions = ['backend']
-
- def setUp(self):
- try:
- self.address, self.port = self.localOptions['backend'].split(":")
- self.port = int(self.port)
- except:
- raise usage.UsageError("Invalid backend address specified (must be address:port)")
-
- def test_random_invalid_request(self):
- """
- We test sending data to a TCP echo server, if what we get back is not
- what we have sent then there is tampering going on.
- This is for example what squid will return when performing such
- request:
-
- HTTP/1.0 400 Bad Request
- Server: squid/2.6.STABLE21
- Date: Sat, 23 Jul 2011 02:22:44 GMT
- Content-Type: text/html
- Content-Length: 1178
- Expires: Sat, 23 Jul 2011 02:22:44 GMT
- X-Squid-Error: ERR_INVALID_REQ 0
- X-Cache: MISS from cache_server
- X-Cache-Lookup: NONE from cache_server:3128
- Via: 1.0 cache_server:3128 (squid/2.6.STABLE21)
- Proxy-Connection: close
-
- """
- payload = randomStr(10) + "\n\r"
- def got_all_data(received_array):
- if not self.localOptions['nopayloadmatch']:
- first = received_array[0]
- if first != payload:
- self.report['tampering'] = True
- else:
- self.report['tampering'] = 'unknown'
-
- d = self.sendPayload(payload)
- d.addCallback(got_all_data)
- return d
diff --git a/nettests/manipulation/http_invalid_request_line.py b/nettests/manipulation/http_invalid_request_line.py
new file mode 100644
index 0000000..00ab24d
--- /dev/null
+++ b/nettests/manipulation/http_invalid_request_line.py
@@ -0,0 +1,106 @@
+# -*- encoding: utf-8 -*-
+from twisted.python import usage
+
+from ooni.utils import randomStr
+from ooni.templates import tcpt
+
+class UsageOptions(usage.Options):
+ optParameters = [['backend', 'b', '127.0.0.1',
+ 'The OONI backend that runs a TCP echo server']]
+
+class HTTPInvalidRequestLine(tcpt.TCPTest):
+ """
+ The goal of this test is to do some very basic and not very noisy fuzzing
+ on the HTTP request line. We generate a series of requests that are not
+ valid HTTP requests.
+
+ Unless elsewhere stated 'Xx'*N refers to N*2 random upper or lowercase ascii
+ letters or numbers ('XxXx' will be 4).
+ """
+ name = "HTTP Invalid Requests"
+ version = "0.1.3"
+ authors = "Arturo Filastò"
+
+ inputFile = ['file', 'f', None,
+ 'Input file of list of hostnames to attempt to resolve']
+
+ usageOptions = UsageOptions
+ requiredOptions = ['backend']
+
+ def setUp(self):
+ self.port = 80
+ self.address = self.localOptions['backend']
+
+ def check_for_manipulation(self, response, payload):
+ if response != payload:
+ self.report['tampering'] = True
+ else:
+ self.report['tampering'] = 'unknown'
+
+ def test_random_invalid_method(self):
+ """
+ We test sending data to a TCP echo server listening on port 80, if what
+ we get back is not what we have sent then there is tampering going on.
+ This is for example what squid will return when performing such
+ request:
+
+ HTTP/1.0 400 Bad Request
+ Server: squid/2.6.STABLE21
+ Date: Sat, 23 Jul 2011 02:22:44 GMT
+ Content-Type: text/html
+ Content-Length: 1178
+ Expires: Sat, 23 Jul 2011 02:22:44 GMT
+ X-Squid-Error: ERR_INVALID_REQ 0
+ X-Cache: MISS from cache_server
+ X-Cache-Lookup: NONE from cache_server:3128
+ Via: 1.0 cache_server:3128 (squid/2.6.STABLE21)
+ Proxy-Connection: close
+
+ """
+ payload = randomStr(10) + " / HTTP/1.1\n\r"
+
+ d = self.sendPayload(payload)
+ d.addCallback(self.check_for_manipulation, payload)
+ return d
+
+ def test_random_invalid_field_count(self):
+ """
+ This generates a request that looks like this:
+
+ XxXxX XxXxX XxXxX XxXxX
+
+ This may trigger some bugs in the HTTP parsers of transparent HTTP
+ proxies.
+ """
+ payload = ' '.join(randomStr(5) for x in range(4))
+ payload += "\n\r"
+
+ d = self.sendPayload(payload)
+ d.addCallback(self.check_for_manipulation, payload)
+ return d
+
+ def test_random_big_request_method(self):
+ """
+ This generates a request that looks like this:
+
+ Xx*512 / HTTP/1.1
+ """
+ payload = randomStr(1024) + ' / HTTP/1.1\n\r'
+
+ d = self.sendPayload(payload)
+ d.addCallback(self.check_for_manipulation, payload)
+ return d
+
+ def test_random_invalid_version_number(self):
+ """
+ This generates a request that looks like this:
+
+ GET / HTTP/XxX
+ """
+ payload = 'GET / HTTP/' + randomStr(3)
+ payload += '\n\r'
+
+ d = self.sendPayload(payload)
+ d.addCallback(self.check_for_manipulation, payload)
+ return d
+
diff --git a/ooni/nettest.py b/ooni/nettest.py
index e0393e7..4a54414 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -5,7 +5,6 @@
# In here is the NetTest API definition. This is how people
# interested in writing ooniprobe tests will be specifying them
#
-# :authors: Arturo Filastò, Isis Lovecruft
# :license: see included LICENSE file
import sys
@@ -17,12 +16,47 @@ from twisted.trial import unittest, itrial, util
from twisted.internet import defer, utils
from twisted.python import usage
+from twisted.internet.error import ConnectionRefusedError, DNSLookupError, TCPTimedOutError
+
from ooni.utils import log
+def failureToString(failure):
+ """
+ Given a failure instance return a string representing the kind of error
+ that occurred.
+
+ Args:
+
+ failure: a :class:twisted.internet.error instance
+
+ Returns:
+
+ A string representing the HTTP response error message.
+ """
+ if isinstance(failure.value, ConnectionRefusedError):
+ log.err("Connection refused. The backend may be down")
+ string = 'connection_refused_error'
+
+ elif isinstance(failure.value, SOCKSError):
+ log.err("Sock error. The SOCKS proxy may be down")
+ string = 'socks_error'
+
+ elif isinstance(failure.value, DNSLookupError):
+ log.err("DNS lookup failure")
+ string = 'dns_lookup_error'
+
+ elif isinstance(failure.value, TCPTimedOutError):
+ log.err("TCP Timed Out Error")
+ string = 'tcp_timed_out_error'
+
+ elif isinstance(failure.value, ResponseNeverReceived):
+ log.err("Response Never Received")
+ string = 'response_never_received'
+ return string
+
class NoPostProcessor(Exception):
pass
-
class NetTestCase(object):
"""
This is the base of the OONI nettest universe. When you write a nettest
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 804a3e4..01853f3 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -23,6 +23,8 @@ from ooni import config
from ooni.utils.net import BodyReceiver, StringProducer, userAgents
from ooni.utils.txagentwithsocks import Agent, SOCKSError, TrueHeaders
+from ooni.nettest import failureToString
+
class InvalidSocksProxyOption(Exception):
pass
@@ -131,25 +133,8 @@ class HTTPTest(NetTestCase):
'code': response.code
}
if failure:
- if isinstance(failure.value, ConnectionRefusedError):
- log.err("Connection refused. The backend may be down")
- request_response['failure'] = 'connection_refused_error'
-
- elif isinstance(failure.value, SOCKSError):
- log.err("Sock error. The SOCKS proxy may be down")
- request_response['failure'] = 'socks_error'
-
- elif isinstance(failure.value, DNSLookupError):
- log.err("DNS lookup failure")
- request_response['failure'] = 'dns_lookup_error'
-
- elif isinstance(failure.value, TCPTimedOutError):
- log.err("TCP Timed Out Error")
- request_response['failure'] = 'tcp_timed_out_error'
+ request_response['failure'] = failureToString(failure)
- elif isinstance(failure.value, ResponseNeverReceived):
- log.err("Response Never Received")
- request_response['failure'] = 'response_never_received'
self.report['requests'].append(request_response)
def _processResponseBody(self, response_body, request, response, body_processor):
diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py
index 77ffe3e..26f38ed 100644
--- a/ooni/templates/tcpt.py
+++ b/ooni/templates/tcpt.py
@@ -2,13 +2,14 @@ from twisted.internet import protocol, defer, reactor
from twisted.internet.error import ConnectionDone
from twisted.internet.endpoints import TCP4ClientEndpoint
-from ooni.nettest import NetTestCase
+from ooni.nettest import NetTestCase, failureToString
from ooni.utils import log
class TCPSender(protocol.Protocol):
- report = None
- payload_len = None
- received_data = ''
+ def __init__(self):
+ self.received_data = ''
+ self.sent_data = ''
+
def dataReceived(self, data):
"""
We receive data until the total amount of data received reaches that
@@ -27,18 +28,19 @@ class TCPSender(protocol.Protocol):
"""
if self.payload_len:
self.received_data += data
- if len(self.received_data) >= self.payload_len:
- self.transport.loseConnection()
- self.report['received'].append(data)
- self.deferred.callback(self.report['received'])
def sendPayload(self, payload):
"""
Write the payload to the wire and set the expected size of the payload
we are to receive.
+
+ Args:
+
+ payload: the data to be sent on the wire.
+
"""
self.payload_len = len(payload)
- self.report['sent'].append(payload)
+ self.sent_data = payload
self.transport.write(payload)
class TCPSenderFactory(protocol.Factory):
@@ -50,7 +52,7 @@ class TCPTest(NetTestCase):
version = "0.1"
requiresRoot = False
- timeout = 2
+ timeout = 5
address = None
port = None
@@ -61,26 +63,34 @@ class TCPTest(NetTestCase):
def sendPayload(self, payload):
d1 = defer.Deferred()
- def closeConnection(p):
- p.transport.loseConnection()
+ def closeConnection(proto):
+ self.report['sent'].append(proto.sent_data)
+ self.report['received'].append(proto.received_data)
+ proto.transport.loseConnection()
log.debug("Closing connection")
d1.callback(self.report['received'])
+ def timedOut(proto):
+ self.report['failure'] = 'tcp_timed_out_error'
+ proto.transport.loseConnection()
+
def errback(failure):
- self.report['error'] = str(failure)
+ self.report['failure'] = failureToString(failure)
d1.errback(failure)
- def connected(p):
+ def connected(proto):
log.debug("Connected to %s:%s" % (self.address, self.port))
- p.report = self.report
- p.deferred = d1
- p.sendPayload(payload)
- reactor.callLater(self.timeout, closeConnection, p)
+ proto.report = self.report
+ proto.deferred = d1
+ proto.sendPayload(payload)
+ if self.timeout:
+ # XXX-Twisted this logic should probably go inside of the protocol
+ reactor.callLater(self.timeout, closeConnection, proto)
point = TCP4ClientEndpoint(reactor, self.address, self.port)
+ log.debug("Connecting to %s:%s" % (self.address, self.port))
d2 = point.connect(TCPSenderFactory())
d2.addCallback(connected)
d2.addErrback(errback)
return d1
-
More information about the tor-commits
mailing list