commit 43cf06c1d2c5aa410aa8b220463d84f6ac7f0a26 Author: Arturo Filastò arturo@filasto.net Date: Tue Oct 2 23:45:07 2012 +0000

Fix date structure in reporting sytem. * Collect errors in httpt template * Rename testhelpers to avoid namespace conflicts * Miscelaneous bug fixing --- bin/ooniprobe | 2 +- nettests/example_httpt.py | 9 ++++- nettests/example_scapyt.py | 2 +- ooni/lib/txscapy.py | 2 +- ooni/nettest.py | 7 ++- ooni/reporter.py | 20 +++++---- ooni/runner.py | 6 ++- ooni/templates/httpt.py | 28 ++++++------- ooni/utils/date.py | 4 +- ooni/utils/log.py | 2 +- oonib/oonibackend.py | 5 ++- oonib/testhelpers/http.py | 74 ----------------------------------- oonib/testhelpers/httph.py | 92 ++++++++++++++++++++++++++++++++++++++++++++ 13 files changed, 143 insertions(+), 110 deletions(-)

diff --git a/bin/ooniprobe b/bin/ooniprobe index c653756..4e400ce 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/env python # -*- encoding: utf-8 -*- # # startup script based on twisted trial diff --git a/nettests/example_httpt.py b/nettests/example_httpt.py index 6501ca7..b113b23 100644 --- a/nettests/example_httpt.py +++ b/nettests/example_httpt.py @@ -10,7 +10,14 @@ class ExampleHTTP(httpt.HTTPTest): version = 0.1

inputs = ['http://google.com/', 'http://wikileaks.org/', - 'http://torproject.org/'] + 'http://torproject.org/'] + + def test_http(self): + if self.input: + url = self.input + return self.doRequest(url) + else: + raise Exception("No input specified")

def processResponseBody(self, body): # XXX here shall go your logic diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py index c7f342d..1cc8054 100644 --- a/nettests/example_scapyt.py +++ b/nettests/example_scapyt.py @@ -11,5 +11,5 @@ class ExampleScapy(scapyt.ScapyTest): version = 0.1

inputs = [IP(dst="8.8.8.8")/TCP(dport=31337), - IP(dst="ooni.nu")/TCP(dport=31337)] + IP(dst="ooni.nu")/TCP(dport=31337)]

diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py index ec605e5..d4f146e 100644 --- a/ooni/lib/txscapy.py +++ b/ooni/lib/txscapy.py @@ -84,7 +84,7 @@ class Scapy(object): """ min = 2 max = 6 - debug = True + debug = False write_only_answers = False pcapwriter = None recv = False diff --git a/ooni/nettest.py b/ooni/nettest.py index 403a022..d7188c6 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -81,8 +81,11 @@ class TestCase(unittest.TestCase): if self.inputFile: fp = open(self.inputFile) self.inputs = inputProcessor(fp) - - return {'inputs': self.inputs} + # XXX perhaps we may want to name and version to be inside of a + # different object that is not called options. + return {'inputs': self.inputs, + 'name': self.name, + 'version': self.version}

def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) diff --git a/ooni/reporter.py b/ooni/reporter.py index 51a0166..3e108f9 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -7,6 +7,7 @@ sys.path.insert(0, '/home/x/Documents/pen_drive_bitcoin2012/ooni-probe/ENV/lib/p from datetime import datetime from twisted.python.util import OrderedDict, untilConcludes from twisted.trial import unittest, reporter, runner +from ooni.utils import date

try: from scapy.all import packet @@ -70,19 +71,19 @@ class ReporterFactory(OReporter): self._testSuite = testSuite self._reporters = []

- def writeHeader(self): - pretty_date = "XXX Replace me with date.pretty_date()" + def writeHeader(self, options): self._writeln("###########################################") - self._writeln("# OONI Probe Report for Test %s" % "XXX replace with with the test suite name") - self._writeln("# %s" % pretty_date) + self._writeln("# OONI Probe Report for %s test" % options['name']) + self._writeln("# %s" % date.pretty_date()) self._writeln("###########################################")

address = {'asn': 'XXX replace me with ASN', 'ip': 'XXX replace me with IP'} - test_details = {'start_time': datetime.now(), - 'asn': address['asn'], - 'test_name': 'XXX replace me with the test name', - 'addr': address['ip']} + test_details = {'startTime': repr(date.now()), + 'probeASN': address['asn'], + 'testName': options['name'], + 'testVersion': options['version'], + 'probeIP': address['ip']} self.writeYamlLine(test_details) self._writeln('')

@@ -135,8 +136,9 @@ class OONIReporter(OReporter): test_input = repr(test.input) else: test_input = test.input + self._tests[idx]['input'] = test_input - self._tests[idx]['idx'] = idx + #self._tests[idx]['idx'] = idx self._tests[idx]['name'] = test.name #self._tests[idx]['test'] = test print "Now starting %s" % self._tests[idx] diff --git a/ooni/runner.py b/ooni/runner.py index 7883f07..0738bf8 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -129,6 +129,8 @@ def loadTestsAndOptions(classes): suiteFactory = InputTestSuite options = [] testCases = [] + names = [] + for klass in classes: try: k = klass() @@ -184,8 +186,8 @@ class ORunner(object):

def run(self): #log.startLogging(sys.stdout) - log.start(True) - self.reporterFactory.writeHeader() + log.start() + self.reporterFactory.writeHeader(self.options)

for inputUnit in InputUnitFactory(self.inputs): self.runWithInputUnit(inputUnit) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 51775f0..a537c6e 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -106,27 +106,25 @@ class HTTPTest(TestCase): """ pass

- def doRequest(self, url): - d = self.build_request(url) + def doRequest(self, url, method="GET", headers=None, body=None): + try: + d = self.build_request(url, method, headers, body) + except Exception, e: + print e + self.report['error'] = e + + def errback(data): + print data + #self.report["error"] = data + def finished(data): - #self.mainDefer.callback() - return data + return

+ d.addErrback(errback) d.addCallback(self._cbResponse) d.addCallback(finished) return d

- def test_http(self): - log.msg("Running experiment") - - if self.input: - url = self.input - else: - raise Exception("No input supplied") - - self.mainDefer = self.doRequest(url) - return self.mainDefer - def _cbResponse(self, response): self.response['headers'] = response.headers self.response['code'] = response.code diff --git a/ooni/utils/date.py b/ooni/utils/date.py index 59ec1f8..dcd8e36 100644 --- a/ooni/utils/date.py +++ b/ooni/utils/date.py @@ -3,10 +3,10 @@ from datetime import datetime

class odate(datetime): def __str__(self): - return rfc3339(self) + return "%s" % rfc3339(self)

def __repr__(self): - return "'%s'" % rfc3339(self) + return "%s" % rfc3339(self)

def from_rfc(self, datestr): pass diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 6ff6bf9..06bc3fa 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -73,7 +73,7 @@ class OONILogObserver(log.FileLogObserver): util.untilConcludes(self.write, timeStr + " " + msgStr) util.untilConcludes(self.flush)

-def start(logstdout, logfile=None, verbosity=None): +def start(logfile=None, logstdout=True, verbosity=None): if log.defaultObserver: verbosity = _get_log_level(verbosity)

diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index d6ba3f0..0232829 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -17,8 +17,8 @@ from twisted.web.microdom import escape from twisted.names import dns

from oonib.lib import config -from oonib.testhelpers.http import HTTPBackend from oonib.lib.ssl import SSLContext +from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer from oonib.testhelpers.dns import ProxyDNSServer from oonib.testhelpers.daphn3 import Daphn3Server

@@ -37,6 +37,9 @@ internet.SSLServer(int(config.main.ssl_port), SSLContext(config), ).setServiceParent(serviceCollection)

+debugHTTPServer = DebugHTTPServer() +internet.TCPServer(8090, debugHTTPServer).setServiceParent(serviceCollection) + # Start the DNS Server related services TCPDNSServer = ProxyDNSServer() internet.TCPServer(int(config.main.dns_tcp_port), TCPDNSServer).setServiceParent(serviceCollection) diff --git a/oonib/testhelpers/http.py b/oonib/testhelpers/http.py deleted file mode 100644 index 79a487b..0000000 --- a/oonib/testhelpers/http.py +++ /dev/null @@ -1,74 +0,0 @@ -import json -import random -import string - -from twisted.application import internet, service -from twisted.internet import protocol, reactor, defer -from twisted.protocols import basic -from twisted.web import resource, server, static -from twisted.web.microdom import escape - -server.version = "Apache" - -class HTTPRandomPage(resource.Resource): - """ - This generates a random page of arbitrary length and containing the string - selected by the user. - The format is the following: - /random/<length>/<keyword> - """ - isLeaf = True - def _gen_random_string(self, length): - return ''.join(random.choice(string.letters) for x in range(length)) - - def genRandomPage(self, length=100, keyword=None): - data = self._gen_random_string(length/2) - if keyword: - data += keyword - data += self._gen_random_string(length - length/2) - data += '\n' - return data - - def render(self, request): - length = 100 - keyword = None - path_parts = request.path.split('/') - if len(path_parts) > 2: - length = int(path_parts[2]) - if length > 100000: - length = 100000 - - if len(path_parts) > 3: - keyword = escape(path_parts[3]) - - return self.genRandomPage(length, keyword) - -class HTTPReturnHeaders(resource.Resource): - """ - This returns the headers being sent by the client in JSON format. - """ - isLeaf = True - def render(self, request): - req_headers = request.getAllHeaders() - return json.dumps(req_headers) - -class HTTPSendHeaders(resource.Resource): - """ - This sends to the client the headers that they send inside of the POST - request encoded in json. - """ - isLeaf = True - def render_POST(self, request): - headers = json.loads(request.content.read()) - for header, value in headers.items(): - request.setHeader(str(header), str(value)) - return '' - -class HTTPBackend(resource.Resource): - def __init__(self): - resource.Resource.__init__(self) - self.putChild('random', HTTPRandomPage()) - self.putChild('returnheaders', HTTPReturnHeaders()) - self.putChild('sendheaders', HTTPSendHeaders()) - - diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py new file mode 100644 index 0000000..217dbd5 --- /dev/null +++ b/oonib/testhelpers/httph.py @@ -0,0 +1,92 @@ +import json +import random +import string + +from twisted.application import internet, service +from twisted.internet import protocol, reactor, defer +from twisted.protocols import basic +from twisted.web import resource, server, static, http +from twisted.web.microdom import escape + +server.version = "Apache" + +class HTTPRandomPage(resource.Resource): + """ + This generates a random page of arbitrary length and containing the string + selected by the user. + The format is the following: + /random/<length>/<keyword> + """ + isLeaf = True + def _gen_random_string(self, length): + return ''.join(random.choice(string.letters) for x in range(length)) + + def genRandomPage(self, length=100, keyword=None): + data = self._gen_random_string(length/2) + if keyword: + data += keyword + data += self._gen_random_string(length - length/2) + data += '\n' + return data + + def render(self, request): + length = 100 + keyword = None + path_parts = request.path.split('/') + if len(path_parts) > 2: + length = int(path_parts[2]) + if length > 100000: + length = 100000 + + if len(path_parts) > 3: + keyword = escape(path_parts[3]) + + return self.genRandomPage(length, keyword) + +class HTTPReturnHeaders(resource.Resource): + """ + This returns the headers being sent by the client in JSON format. + """ + isLeaf = True + def render(self, request): + req_headers = request.getAllHeaders() + return json.dumps(req_headers) + +class HTTPSendHeaders(resource.Resource): + """ + This sends to the client the headers that they send inside of the POST + request encoded in json. + """ + isLeaf = True + def render_POST(self, request): + headers = json.loads(request.content.read()) + for header, value in headers.items(): + request.setHeader(str(header), str(value)) + return '' + +class HTTPBackend(resource.Resource): + def __init__(self): + resource.Resource.__init__(self) + self.putChild('random', HTTPRandomPage()) + self.putChild('returnheaders', HTTPReturnHeaders()) + self.putChild('sendheaders', HTTPSendHeaders()) + +class DebugProtocol(http.HTTPChannel): + def headerReceived(self, line): + print "[HEADER] %s" % line + http.HTTPChannel.headerReceived(self, line) + + def allContentReceived(self): + print self.requests[-1].getAllHeaders() + self.transport.loseConnection() + self.connectionLost("Normal closure") + +class DebugHTTPServer(http.HTTPFactory): + protocol = DebugProtocol + + def buildProtocol(self, addr): + print "Got connection from %s" % addr + p = protocol.ServerFactory.buildProtocol(self, addr) + p.timeOut = self.timeOut + return p +