[tor-commits] [ooni-probe/master] Fix date structure in reporting sytem.
art at torproject.org
art at torproject.org
Fri Oct 5 14:09:04 UTC 2012
commit 43cf06c1d2c5aa410aa8b220463d84f6ac7f0a26
Author: Arturo Filastò <arturo at filasto.net>
Date: Tue Oct 2 23:45:07 2012 +0000
Fix date structure in reporting sytem.
* Collect errors in httpt template
* Rename testhelpers to avoid namespace conflicts
* Miscelaneous bug fixing
---
bin/ooniprobe | 2 +-
nettests/example_httpt.py | 9 ++++-
nettests/example_scapyt.py | 2 +-
ooni/lib/txscapy.py | 2 +-
ooni/nettest.py | 7 ++-
ooni/reporter.py | 20 +++++----
ooni/runner.py | 6 ++-
ooni/templates/httpt.py | 28 ++++++-------
ooni/utils/date.py | 4 +-
ooni/utils/log.py | 2 +-
oonib/oonibackend.py | 5 ++-
oonib/testhelpers/http.py | 74 -----------------------------------
oonib/testhelpers/httph.py | 92 ++++++++++++++++++++++++++++++++++++++++++++
13 files changed, 143 insertions(+), 110 deletions(-)
diff --git a/bin/ooniprobe b/bin/ooniprobe
index c653756..4e400ce 100755
--- a/bin/ooniprobe
+++ b/bin/ooniprobe
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# startup script based on twisted trial
diff --git a/nettests/example_httpt.py b/nettests/example_httpt.py
index 6501ca7..b113b23 100644
--- a/nettests/example_httpt.py
+++ b/nettests/example_httpt.py
@@ -10,7 +10,14 @@ class ExampleHTTP(httpt.HTTPTest):
version = 0.1
inputs = ['http://google.com/', 'http://wikileaks.org/',
- 'http://torproject.org/']
+ 'http://torproject.org/']
+
+ def test_http(self):
+ if self.input:
+ url = self.input
+ return self.doRequest(url)
+ else:
+ raise Exception("No input specified")
def processResponseBody(self, body):
# XXX here shall go your logic
diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py
index c7f342d..1cc8054 100644
--- a/nettests/example_scapyt.py
+++ b/nettests/example_scapyt.py
@@ -11,5 +11,5 @@ class ExampleScapy(scapyt.ScapyTest):
version = 0.1
inputs = [IP(dst="8.8.8.8")/TCP(dport=31337),
- IP(dst="ooni.nu")/TCP(dport=31337)]
+ IP(dst="ooni.nu")/TCP(dport=31337)]
diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py
index ec605e5..d4f146e 100644
--- a/ooni/lib/txscapy.py
+++ b/ooni/lib/txscapy.py
@@ -84,7 +84,7 @@ class Scapy(object):
"""
min = 2
max = 6
- debug = True
+ debug = False
write_only_answers = False
pcapwriter = None
recv = False
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 403a022..d7188c6 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -81,8 +81,11 @@ class TestCase(unittest.TestCase):
if self.inputFile:
fp = open(self.inputFile)
self.inputs = inputProcessor(fp)
-
- return {'inputs': self.inputs}
+ # XXX perhaps we may want to name and version to be inside of a
+ # different object that is not called options.
+ return {'inputs': self.inputs,
+ 'name': self.name,
+ 'version': self.version}
def __repr__(self):
return "<%s inputs=%s>" % (self.__class__, self.inputs)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 51a0166..3e108f9 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -7,6 +7,7 @@ sys.path.insert(0, '/home/x/Documents/pen_drive_bitcoin2012/ooni-probe/ENV/lib/p
from datetime import datetime
from twisted.python.util import OrderedDict, untilConcludes
from twisted.trial import unittest, reporter, runner
+from ooni.utils import date
try:
from scapy.all import packet
@@ -70,19 +71,19 @@ class ReporterFactory(OReporter):
self._testSuite = testSuite
self._reporters = []
- def writeHeader(self):
- pretty_date = "XXX Replace me with date.pretty_date()"
+ def writeHeader(self, options):
self._writeln("###########################################")
- self._writeln("# OONI Probe Report for Test %s" % "XXX replace with with the test suite name")
- self._writeln("# %s" % pretty_date)
+ self._writeln("# OONI Probe Report for %s test" % options['name'])
+ self._writeln("# %s" % date.pretty_date())
self._writeln("###########################################")
address = {'asn': 'XXX replace me with ASN',
'ip': 'XXX replace me with IP'}
- test_details = {'start_time': datetime.now(),
- 'asn': address['asn'],
- 'test_name': 'XXX replace me with the test name',
- 'addr': address['ip']}
+ test_details = {'startTime': repr(date.now()),
+ 'probeASN': address['asn'],
+ 'testName': options['name'],
+ 'testVersion': options['version'],
+ 'probeIP': address['ip']}
self.writeYamlLine(test_details)
self._writeln('')
@@ -135,8 +136,9 @@ class OONIReporter(OReporter):
test_input = repr(test.input)
else:
test_input = test.input
+
self._tests[idx]['input'] = test_input
- self._tests[idx]['idx'] = idx
+ #self._tests[idx]['idx'] = idx
self._tests[idx]['name'] = test.name
#self._tests[idx]['test'] = test
print "Now starting %s" % self._tests[idx]
diff --git a/ooni/runner.py b/ooni/runner.py
index 7883f07..0738bf8 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -129,6 +129,8 @@ def loadTestsAndOptions(classes):
suiteFactory = InputTestSuite
options = []
testCases = []
+ names = []
+
for klass in classes:
try:
k = klass()
@@ -184,8 +186,8 @@ class ORunner(object):
def run(self):
#log.startLogging(sys.stdout)
- log.start(True)
- self.reporterFactory.writeHeader()
+ log.start()
+ self.reporterFactory.writeHeader(self.options)
for inputUnit in InputUnitFactory(self.inputs):
self.runWithInputUnit(inputUnit)
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 51775f0..a537c6e 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -106,27 +106,25 @@ class HTTPTest(TestCase):
"""
pass
- def doRequest(self, url):
- d = self.build_request(url)
+ def doRequest(self, url, method="GET", headers=None, body=None):
+ try:
+ d = self.build_request(url, method, headers, body)
+ except Exception, e:
+ print e
+ self.report['error'] = e
+
+ def errback(data):
+ print data
+ #self.report["error"] = data
+
def finished(data):
- #self.mainDefer.callback()
- return data
+ return
+ d.addErrback(errback)
d.addCallback(self._cbResponse)
d.addCallback(finished)
return d
- def test_http(self):
- log.msg("Running experiment")
-
- if self.input:
- url = self.input
- else:
- raise Exception("No input supplied")
-
- self.mainDefer = self.doRequest(url)
- return self.mainDefer
-
def _cbResponse(self, response):
self.response['headers'] = response.headers
self.response['code'] = response.code
diff --git a/ooni/utils/date.py b/ooni/utils/date.py
index 59ec1f8..dcd8e36 100644
--- a/ooni/utils/date.py
+++ b/ooni/utils/date.py
@@ -3,10 +3,10 @@ from datetime import datetime
class odate(datetime):
def __str__(self):
- return rfc3339(self)
+ return "%s" % rfc3339(self)
def __repr__(self):
- return "'%s'" % rfc3339(self)
+ return "%s" % rfc3339(self)
def from_rfc(self, datestr):
pass
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 6ff6bf9..06bc3fa 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -73,7 +73,7 @@ class OONILogObserver(log.FileLogObserver):
util.untilConcludes(self.write, timeStr + " " + msgStr)
util.untilConcludes(self.flush)
-def start(logstdout, logfile=None, verbosity=None):
+def start(logfile=None, logstdout=True, verbosity=None):
if log.defaultObserver:
verbosity = _get_log_level(verbosity)
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index d6ba3f0..0232829 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -17,8 +17,8 @@ from twisted.web.microdom import escape
from twisted.names import dns
from oonib.lib import config
-from oonib.testhelpers.http import HTTPBackend
from oonib.lib.ssl import SSLContext
+from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer
from oonib.testhelpers.dns import ProxyDNSServer
from oonib.testhelpers.daphn3 import Daphn3Server
@@ -37,6 +37,9 @@ internet.SSLServer(int(config.main.ssl_port),
SSLContext(config),
).setServiceParent(serviceCollection)
+debugHTTPServer = DebugHTTPServer()
+internet.TCPServer(8090, debugHTTPServer).setServiceParent(serviceCollection)
+
# Start the DNS Server related services
TCPDNSServer = ProxyDNSServer()
internet.TCPServer(int(config.main.dns_tcp_port), TCPDNSServer).setServiceParent(serviceCollection)
diff --git a/oonib/testhelpers/http.py b/oonib/testhelpers/http.py
deleted file mode 100644
index 79a487b..0000000
--- a/oonib/testhelpers/http.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import json
-import random
-import string
-
-from twisted.application import internet, service
-from twisted.internet import protocol, reactor, defer
-from twisted.protocols import basic
-from twisted.web import resource, server, static
-from twisted.web.microdom import escape
-
-server.version = "Apache"
-
-class HTTPRandomPage(resource.Resource):
- """
- This generates a random page of arbitrary length and containing the string
- selected by the user.
- The format is the following:
- /random/<length>/<keyword>
- """
- isLeaf = True
- def _gen_random_string(self, length):
- return ''.join(random.choice(string.letters) for x in range(length))
-
- def genRandomPage(self, length=100, keyword=None):
- data = self._gen_random_string(length/2)
- if keyword:
- data += keyword
- data += self._gen_random_string(length - length/2)
- data += '\n'
- return data
-
- def render(self, request):
- length = 100
- keyword = None
- path_parts = request.path.split('/')
- if len(path_parts) > 2:
- length = int(path_parts[2])
- if length > 100000:
- length = 100000
-
- if len(path_parts) > 3:
- keyword = escape(path_parts[3])
-
- return self.genRandomPage(length, keyword)
-
-class HTTPReturnHeaders(resource.Resource):
- """
- This returns the headers being sent by the client in JSON format.
- """
- isLeaf = True
- def render(self, request):
- req_headers = request.getAllHeaders()
- return json.dumps(req_headers)
-
-class HTTPSendHeaders(resource.Resource):
- """
- This sends to the client the headers that they send inside of the POST
- request encoded in json.
- """
- isLeaf = True
- def render_POST(self, request):
- headers = json.loads(request.content.read())
- for header, value in headers.items():
- request.setHeader(str(header), str(value))
- return ''
-
-class HTTPBackend(resource.Resource):
- def __init__(self):
- resource.Resource.__init__(self)
- self.putChild('random', HTTPRandomPage())
- self.putChild('returnheaders', HTTPReturnHeaders())
- self.putChild('sendheaders', HTTPSendHeaders())
-
-
diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py
new file mode 100644
index 0000000..217dbd5
--- /dev/null
+++ b/oonib/testhelpers/httph.py
@@ -0,0 +1,92 @@
+import json
+import random
+import string
+
+from twisted.application import internet, service
+from twisted.internet import protocol, reactor, defer
+from twisted.protocols import basic
+from twisted.web import resource, server, static, http
+from twisted.web.microdom import escape
+
+server.version = "Apache"
+
+class HTTPRandomPage(resource.Resource):
+ """
+ This generates a random page of arbitrary length and containing the string
+ selected by the user.
+ The format is the following:
+ /random/<length>/<keyword>
+ """
+ isLeaf = True
+ def _gen_random_string(self, length):
+ return ''.join(random.choice(string.letters) for x in range(length))
+
+ def genRandomPage(self, length=100, keyword=None):
+ data = self._gen_random_string(length/2)
+ if keyword:
+ data += keyword
+ data += self._gen_random_string(length - length/2)
+ data += '\n'
+ return data
+
+ def render(self, request):
+ length = 100
+ keyword = None
+ path_parts = request.path.split('/')
+ if len(path_parts) > 2:
+ length = int(path_parts[2])
+ if length > 100000:
+ length = 100000
+
+ if len(path_parts) > 3:
+ keyword = escape(path_parts[3])
+
+ return self.genRandomPage(length, keyword)
+
+class HTTPReturnHeaders(resource.Resource):
+ """
+ This returns the headers being sent by the client in JSON format.
+ """
+ isLeaf = True
+ def render(self, request):
+ req_headers = request.getAllHeaders()
+ return json.dumps(req_headers)
+
+class HTTPSendHeaders(resource.Resource):
+ """
+ This sends to the client the headers that they send inside of the POST
+ request encoded in json.
+ """
+ isLeaf = True
+ def render_POST(self, request):
+ headers = json.loads(request.content.read())
+ for header, value in headers.items():
+ request.setHeader(str(header), str(value))
+ return ''
+
+class HTTPBackend(resource.Resource):
+ def __init__(self):
+ resource.Resource.__init__(self)
+ self.putChild('random', HTTPRandomPage())
+ self.putChild('returnheaders', HTTPReturnHeaders())
+ self.putChild('sendheaders', HTTPSendHeaders())
+
+class DebugProtocol(http.HTTPChannel):
+ def headerReceived(self, line):
+ print "[HEADER] %s" % line
+ http.HTTPChannel.headerReceived(self, line)
+
+ def allContentReceived(self):
+ print self.requests[-1].getAllHeaders()
+ self.transport.loseConnection()
+ self.connectionLost("Normal closure")
+
+class DebugHTTPServer(http.HTTPFactory):
+ protocol = DebugProtocol
+
+ def buildProtocol(self, addr):
+ print "Got connection from %s" % addr
+ p = protocol.ServerFactory.buildProtocol(self, addr)
+ p.timeOut = self.timeOut
+ return p
+
More information about the tor-commits
mailing list