commit 36c8a8e149fa5d975fc0d39bd0e5bf72ab38086a Author: Arturo Filastò arturo@filasto.net Date: Fri Sep 28 16:11:05 2012 +0000
Implement port of ScapyTest to new API * Rename templates to avoid namespace conflicts * XXX need to fix some bugs in how yaml serializes scapy objects. Basically the issue is that scapy Packet objects override getattr and setattr. This leads to pickle having some issues in serializing it. Some links describing the issue: http://stackoverflow.com/questions/2049849/why-cant-i-pickle-this-object http://stackoverflow.com/questions/569754/how-to-tell-for-which-object-attri...
There was also an open bug on the pyyaml trac repo, but it got closed because they could not reproduce. http://pyyaml.org/ticket/190 --- nettests/example_http.py | 28 -------- nettests/example_httpt.py | 28 ++++++++ nettests/example_scapyt.py | 9 +++ ooni/templates/http.py | 147 -------------------------------------------- ooni/templates/httpt.py | 147 ++++++++++++++++++++++++++++++++++++++++++++ ooni/templates/scapyt.py | 65 +++++++++++++++++++ 6 files changed, 249 insertions(+), 175 deletions(-)
diff --git a/nettests/example_http.py b/nettests/example_http.py deleted file mode 100644 index 7528d48..0000000 --- a/nettests/example_http.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -from ooni.templates import http -class Example(http.HTTPTest): - inputs = ['http://google.com/', 'http://wikileaks.org/', - 'http://torproject.org/'] - - def processResponseBody(self, body): - # XXX here shall go your logic - # for processing the body - if 'blocked' in body: - self.report['censored'] = True - else: - self.report['censored'] = False - - def processResponseHeaders(self, headers): - # XXX place in here all the logic for handling the processing of HTTP - # Headers. - if headers.hasHeader('location'): - self.report['redirect'] = True - - server = headers.getRawHeaders("Server") - if server: - self.report['http_server'] = str(server.pop()) - diff --git a/nettests/example_httpt.py b/nettests/example_httpt.py new file mode 100644 index 0000000..1814a90 --- /dev/null +++ b/nettests/example_httpt.py @@ -0,0 +1,28 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni.templates import httpt +class Example(httpt.HTTPTest): + inputs = ['http://google.com/', 'http://wikileaks.org/', + 'http://torproject.org/'] + + def processResponseBody(self, body): + # XXX here shall go your logic + # for processing the body + if 'blocked' in body: + self.report['censored'] = True + else: + self.report['censored'] = False + + def processResponseHeaders(self, headers): + # XXX place in here all the logic for handling the processing of HTTP + # Headers. + if headers.hasHeader('location'): + self.report['redirect'] = True + + server = headers.getRawHeaders("Server") + if server: + self.report['http_server'] = str(server.pop()) + diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py new file mode 100644 index 0000000..ef61f3d --- /dev/null +++ b/nettests/example_scapyt.py @@ -0,0 +1,9 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni.templates import scapyt +from scapy.all import * +class ExampleScapy(scapyt.ScapyTest): + inputs = [IP()/TCP()] diff --git a/ooni/templates/http.py b/ooni/templates/http.py deleted file mode 100644 index 222e44c..0000000 --- a/ooni/templates/http.py +++ /dev/null @@ -1,147 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -import random - -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import protocol, defer -from twisted.web.http_headers import Headers - -from ooni.nettest import TestCase -from ooni.utils import log - -useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), - ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), - ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), - ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] - -class BodyReceiver(protocol.Protocol): - def __init__(self, finished): - self.finished = finished - self.data = "" - - def dataReceived(self, bytes): - self.data += bytes - - def connectionLost(self, reason): - self.finished.callback(self.data) - -class HTTPTest(TestCase): - """ - A utility class for dealing with HTTP based testing. It provides methods to - be overriden for dealing with HTTP based testing. - The main functions to look at are processResponseBody and - processResponseHeader that are invoked once the headers have been received - and once the request body has been received. - """ - randomizeUA = True - followRedirects = False - - def setUp(self): - from twisted.web.client import Agent - from twisted.internet import reactor - import yaml - self.agent = Agent(reactor) - if self.followRedirects: - from twisted.web.client import RedirectAgent - self.agent = RedirectAgent(self.agent) - self.request = {} - self.response = {} - - def _processResponseBody(self, data): - self.response['body'] = data - self.report['response'] = self.response - - self.processResponseBody(data) - - def processResponseBody(self, data): - """ - This should handle all the response body smushing for getting it ready - to be passed onto the control. - - @param data: The content of the body returned. - """ - pass - - def processResponseHeaders(self, headers): - """ - This should take care of dealing with the returned HTTP headers. - - @param headers: The content of the returned headers. - """ - pass - - def processRedirect(self, location): - """ - Handle a redirection via a 3XX HTTP status code. - - @param location: the url that is being redirected to. - """ - pass - - def doRequest(self, url): - d = self.build_request(url) - def finished(data): - return data - - d.addCallback(self._cbResponse) - d.addCallback(finished) - return d - - def test_http(self): - log.msg("Running experiment") - - if self.input: - url = self.input - else: - raise Exception("No input supplied") - - self.mainDefer = self.doRequest(url) - return self.mainDefer - - def _cbResponse(self, response): - self.response['headers'] = response.headers - self.response['code'] = response.code - self.response['length'] = response.length - self.response['version'] = response.length - - if str(self.response['code']).startswith('3'): - self.processRedirect(response.headers.getRawHeaders('Location')[0]) - - self.processResponseHeaders(self.response['headers']) - - finished = defer.Deferred() - response.deliverBody(BodyReceiver(finished)) - finished.addCallback(self._processResponseBody) - - return finished - - def randomize_useragent(self): - user_agent = random.choice(useragents) - self.request['headers']['User-Agent'] = [user_agent] - - def build_request(self, url, method="GET", headers=None, body=None): - self.request['method'] = method - self.request['url'] = url - self.request['headers'] = headers if headers else {} - self.request['body'] = body - if self.randomizeUA: - self.randomize_useragent() - - self.report['request'] = self.request - self.report['url'] = url - return self.agent.request(self.request['method'], self.request['url'], - Headers(self.request['headers']), - self.request['body']) - diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py new file mode 100644 index 0000000..222e44c --- /dev/null +++ b/ooni/templates/httpt.py @@ -0,0 +1,147 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import random + +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer +from twisted.web.http_headers import Headers + +from ooni.nettest import TestCase +from ooni.utils import log + +useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + +class BodyReceiver(protocol.Protocol): + def __init__(self, finished): + self.finished = finished + self.data = "" + + def dataReceived(self, bytes): + self.data += bytes + + def connectionLost(self, reason): + self.finished.callback(self.data) + +class HTTPTest(TestCase): + """ + A utility class for dealing with HTTP based testing. It provides methods to + be overriden for dealing with HTTP based testing. + The main functions to look at are processResponseBody and + processResponseHeader that are invoked once the headers have been received + and once the request body has been received. + """ + randomizeUA = True + followRedirects = False + + def setUp(self): + from twisted.web.client import Agent + from twisted.internet import reactor + import yaml + self.agent = Agent(reactor) + if self.followRedirects: + from twisted.web.client import RedirectAgent + self.agent = RedirectAgent(self.agent) + self.request = {} + self.response = {} + + def _processResponseBody(self, data): + self.response['body'] = data + self.report['response'] = self.response + + self.processResponseBody(data) + + def processResponseBody(self, data): + """ + This should handle all the response body smushing for getting it ready + to be passed onto the control. + + @param data: The content of the body returned. + """ + pass + + def processResponseHeaders(self, headers): + """ + This should take care of dealing with the returned HTTP headers. + + @param headers: The content of the returned headers. + """ + pass + + def processRedirect(self, location): + """ + Handle a redirection via a 3XX HTTP status code. + + @param location: the url that is being redirected to. + """ + pass + + def doRequest(self, url): + d = self.build_request(url) + def finished(data): + return data + + d.addCallback(self._cbResponse) + d.addCallback(finished) + return d + + def test_http(self): + log.msg("Running experiment") + + if self.input: + url = self.input + else: + raise Exception("No input supplied") + + self.mainDefer = self.doRequest(url) + return self.mainDefer + + def _cbResponse(self, response): + self.response['headers'] = response.headers + self.response['code'] = response.code + self.response['length'] = response.length + self.response['version'] = response.length + + if str(self.response['code']).startswith('3'): + self.processRedirect(response.headers.getRawHeaders('Location')[0]) + + self.processResponseHeaders(self.response['headers']) + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished)) + finished.addCallback(self._processResponseBody) + + return finished + + def randomize_useragent(self): + user_agent = random.choice(useragents) + self.request['headers']['User-Agent'] = [user_agent] + + def build_request(self, url, method="GET", headers=None, body=None): + self.request['method'] = method + self.request['url'] = url + self.request['headers'] = headers if headers else {} + self.request['body'] = body + if self.randomizeUA: + self.randomize_useragent() + + self.report['request'] = self.request + self.report['url'] = url + return self.agent.request(self.request['method'], self.request['url'], + Headers(self.request['headers']), + self.request['body']) + diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py new file mode 100644 index 0000000..894a476 --- /dev/null +++ b/ooni/templates/scapyt.py @@ -0,0 +1,65 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import random +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer + +from ooni.nettest import TestCase +from ooni.utils import log + +from ooni.lib.txscapy import txsr, txsend + +from scapy.all import * +class ScapyTest(TestCase): + """ + A utility class for writing scapy driven OONI tests. + + * pcapfile: specify where to store the logged pcapfile + + * timeout: timeout in ms of when we should stop waiting to receive packets + + * receive: if we should also receive packets and not just send + """ + + receive = True + timeout = None + pcapfile = 'scapytest.pcap' + input = IP()/TCP() + def setUp(self): + + if not self.reactor: + from twisted.internet import reactor + self.reactor = reactor + + self.request = {} + self.response = {} + + def test_sendReceive(self): + log.msg("Running send receive") + if self.receive: + log.msg("Sending and receiving packets.") + d = txsr(self.buildPackets(), pcapfile=self.pcapfile, + timeout=self.timeout) + else: + log.msg("Sending packets.") + d = txsend(self.buildPackets()) + + def finished(data): + log.msg("Finished sending") + return data + + d.addCallback(finished) + return d + + def buildPackets(self): + """ + Override this method to build scapy packets. + """ + return self.input + +