[tor-commits] [ooni-probe/master] Refactor reporter Object.
art at torproject.org
art at torproject.org
Sat Nov 10 23:45:39 UTC 2012
commit 572b301710f395620c557dde8ecdef0771ccbc88
Author: Arturo Filastò <art at fuffa.org>
Date: Sat Nov 10 23:32:48 2012 +0100
Refactor reporter Object.
* We now have a parent OReporter object that is subclassed by OONIBReporter for
remote reporting and YAMLReporter for reporting to YAML format on file system.
* Move secure YAML serialization hacks to the hacks module
* Do more progress on the implementation of reporting to remote systems
---
ooni/reporter.py | 282 ++++++++++++++++++++++++++-------------------------
ooni/runner.py | 8 +-
ooni/utils/hacks.py | 47 +++++++++
3 files changed, 197 insertions(+), 140 deletions(-)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 5d29d54..b625603 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -26,6 +26,8 @@ from twisted.internet import defer, reactor
from ooni.templates.httpt import BodyReceiver, StringProducer
from ooni.utils import otime, log, geodata
+
+from ooni.utils.hacks import OSafeRepresenter, OSafeDumper
from ooni import config
try:
@@ -38,47 +40,8 @@ except:
Packet = object
packet = FooClass
-class OSafeRepresenter(SafeRepresenter):
- """
- This is a custom YAML representer that allows us to represent reports
- safely.
- It extends the SafeRepresenter to be able to also represent complex numbers
- """
- def represent_complex(self, data):
- if data.imag == 0.0:
- data = u'%r' % data.real
- elif data.real == 0.0:
- data = u'%rj' % data.imag
- elif data.imag > 0:
- data = u'%r+%rj' % (data.real, data.imag)
- else:
- data = u'%r%rj' % (data.real, data.imag)
- return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data)
-
-OSafeRepresenter.add_representer(complex,
- OSafeRepresenter.represent_complex)
-
-class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):
- """
- This is a modification of the YAML Safe Dumper to use our own Safe
- Representer that supports complex numbers.
- """
- def __init__(self, stream,
- default_style=None, default_flow_style=None,
- canonical=None, indent=None, width=None,
- allow_unicode=None, line_break=None,
- encoding=None, explicit_start=None, explicit_end=None,
- version=None, tags=None):
- Emitter.__init__(self, stream, canonical=canonical,
- indent=indent, width=width,
- allow_unicode=allow_unicode, line_break=line_break)
- Serializer.__init__(self, encoding=encoding,
- explicit_start=explicit_start, explicit_end=explicit_end,
- version=version, tags=tags)
- OSafeRepresenter.__init__(self, default_style=default_style,
- default_flow_style=default_flow_style)
- Resolver.__init__(self)
-
+class NoTestIDSpecified(Exception):
+ pass
def safe_dump(data, stream=None, **kw):
"""
@@ -86,42 +49,93 @@ def safe_dump(data, stream=None, **kw):
"""
return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw)
-class OONIBReporter(object):
- def __init__(self, backend_url):
- from twisted.web.client import Agent
- from twisted.internet import reactor
+ at defer.inlineCallbacks
+def getTestDetails(options):
+ client_geodata = {}
+
+ if config.privacy.includeip or \
+ config.privacy.includeasn or \
+ config.privacy.includecountry or \
+ config.privacy.includecity:
+ log.msg("Running geo IP lookup via check.torproject.org")
+ client_ip = yield geodata.myIP()
+ client_location = geodata.IPToLocation(client_ip)
+ else:
+ client_ip = "127.0.0.1"
+
+ if config.privacy.includeip:
+ client_geodata['ip'] = client_ip
+ else:
+ client_geodata['ip'] = "127.0.0.1"
+
+ client_geodata['asn'] = None
+ client_geodata['city'] = None
+ client_geodata['countrycode'] = None
+
+ if config.privacy.includeasn:
+ client_geodata['asn'] = client_location['asn']
+
+ if config.privacy.includecity:
+ client_geodata['city'] = client_location['city']
+
+ if config.privacy.includecountry:
+ client_geodata['countrycode'] = client_location['countrycode']
+
+ test_details = {'start_time': otime.utcTimeNow(),
+ 'probe_asn': client_geodata['asn'],
+ 'probe_cc': client_geodata['countrycode'],
+ 'probe_ip': client_geodata['ip'],
+ 'test_name': options['name'],
+ 'test_version': options['version'],
+ }
+ defer.returnValue(test_details)
+
+
+class OReporter(object):
+ def createReport(options):
+ """
+ Override this with your own logic to implement tests.
+ """
+ raise NotImplemented
- self.agent = Agent(reactor)
- self.backend_url = backend_url
+ def writeReportEntry(self, entry):
+ """
+ Takes as input an entry and writes a report for it.
+ """
+ raise NotImplemented
- def _newReportCreated(self, data):
- log.debug("Got this as result: %s" % data)
- return data
+ def finish():
+ pass
- def _processResponseBody(self, response, body_cb):
- log.debug("Got response %s" % response)
- done = defer.Deferred()
- response.deliverBody(BodyReceiver(done))
- done.addCallback(body_cb)
- return done
+ def testDone(self, test, test_name):
+ test_report = dict(test.report)
- def newReport(self, test_name, test_version):
- url = self.backend_url + '/new'
- software_version = '0.0.1'
+ # XXX the scapy test has an example of how
+ # to do this properly.
+ if isinstance(test.input, packet.Packet):
+ test_input = repr(test.input)
+ else:
+ test_input = test.input
- request = {'software_name': 'ooni-probe',
- 'software_version': software_version,
- 'test_name': test_name, 'test_version': test_version,
- 'progress': 0}
+ test_started = test._start_time
+ test_runtime = test_started - time.time()
- log.debug("Creating report via url %s" % url)
- bodyProducer = StringProducer(json.dumps(request))
- d = self.agent.request("POST", url, bodyProducer=bodyProducer)
- d.addCallback(self._processResponseBody, self._newReportCreated)
- return d
+ report = {'input': test_input,
+ 'test_name': test_name,
+ 'test_started': test_started,
+ 'report': test_report}
+ self.writeReportEntry(report)
+ def allDone(self):
+ log.debug("allDone: Finished running all tests")
+ self.finish()
+ try:
+ reactor.stop()
+ except:
+ pass
+ return None
-class YamlReporter(object):
+class YAMLReporter(OReporter):
"""
These are useful functions for reporting to YAML format.
"""
@@ -145,91 +159,85 @@ class YamlReporter(object):
self._write(safe_dump(entry))
self._write('...\n')
- def finish(self):
- self._stream.close()
-
-class OReporter(YamlReporter):
- """
- This is a reporter factory. It emits new instances of Reports. It is also
- responsible for writing the OONI Report headers.
- """
- def writeTestsReport(self, tests):
- for test in tests.values():
- self.writeReportEntry(test)
-
@defer.inlineCallbacks
- def writeReportHeader(self, options):
- self.firstrun = False
+ def createReport(self, options):
self._writeln("###########################################")
self._writeln("# OONI Probe Report for %s test" % options['name'])
self._writeln("# %s" % otime.prettyDateNow())
self._writeln("###########################################")
- client_geodata = {}
+ test_details = yield getTestDetails(options)
- if config.privacy.includeip or \
- config.privacy.includeasn or \
- config.privacy.includecountry or \
- config.privacy.includecity:
- log.msg("Running geo IP lookup via check.torproject.org")
- client_ip = yield geodata.myIP()
- client_location = geodata.IPToLocation(client_ip)
- else:
- client_ip = "127.0.0.1"
+ self.writeReportEntry(test_details)
- if config.privacy.includeip:
- client_geodata['ip'] = client_ip
- else:
- client_geodata['ip'] = "127.0.0.1"
+ def finish(self):
+ self._stream.close()
- client_geodata['asn'] = None
- client_geodata['city'] = None
- client_geodata['countrycode'] = None
+class OONIBReporter(object):
+ def __init__(self, backend_url):
+ from twisted.web.client import Agent
+ from twisted.internet import reactor
+ self.agent = Agent(reactor)
+ self.backend_url = backend_url
- if config.privacy.includeasn:
- client_geodata['asn'] = client_location['asn']
+ def _newReportCreated(self, data):
+ log.debug("newReportCreated %s" % data)
+ return data
- if config.privacy.includecity:
- client_geodata['city'] = client_location['city']
+ def _processResponseBody(self, response, body_cb):
+ log.debug("processResponseBody %s" % response)
+ done = defer.Deferred()
+ response.deliverBody(BodyReceiver(done))
+ done.addCallback(body_cb)
+ return done
- if config.privacy.includecountry:
- client_geodata['countrycode'] = client_location['countrycode']
+ def createReport(self, test_name,
+ test_version, report_header):
+ url = self.backend_url + '/new'
+ software_version = '0.0.1'
- test_details = {'start_time': otime.utcTimeNow(),
- 'probe_asn': client_geodata['asn'],
- 'probe_cc': client_geodata['countrycode'],
- 'probe_ip': client_geodata['ip'],
- 'test_name': options['name'],
- 'test_version': options['version'],
- }
- self.writeReportEntry(test_details)
+ request = {'software_name': 'ooni-probe',
+ 'software_version': software_version,
+ 'test_name': test_name,
+ 'test_version': test_version,
+ 'progress': 0,
+ 'content': report_header
+ }
+ def gotDetails(test_details):
+ log.debug("Creating report via url %s" % url)
+
+ bodyProducer = StringProducer(json.dumps(request))
+ d = self.agent.request("POST", url,
+ bodyProducer=bodyProducer)
+ d.addCallback(self._processResponseBody,
+ self._newReportCreated)
+ return d
+
+ d = getTestDetails(options)
+ d.addCallback(gotDetails)
+ return d
- def testDone(self, test, test_name):
- test_report = dict(test.report)
+ def writeReportEntry(self, entry, test_id=None):
+ if not test_id:
+ log.err("Write report entry on OONIB requires test id")
+ raise NoTestIDSpecified
- # XXX the scapy test has an example of how
- # to do this properly.
- if isinstance(test.input, packet.Packet):
- test_input = repr(test.input)
- else:
- test_input = test.input
+ report = '---\n'
+ report += safe_dump(entry)
+ report += '...\n'
- test_started = test._start_time
- test_runtime = test_started - time.time()
+ url = self.backend_url + '/new'
- report = {'input': test_input,
- 'test_name': test_name,
- 'test_started': test_started,
- 'report': test_report}
- self.writeReportEntry(report)
+ request = {'test_id': test_id,
+ 'content': report}
+
+ bodyProducer = StringProducer(json.dumps(request))
+ d = self.agent.request("PUT", url,
+ bodyProducer=bodyProducer)
+
+ d.addCallback(self._processResponseBody,
+ self._newReportCreated)
+ return d
- def allDone(self):
- log.debug("allDone: Finished running all tests")
- self.finish()
- try:
- reactor.stop()
- except:
- pass
- return None
diff --git a/ooni/runner.py b/ooni/runner.py
index 1affb15..07882cf 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -194,7 +194,8 @@ def runTestWithInputUnit(test_class,
for test_input in input_unit:
log.debug("IU: %s" % test_input)
try:
- d = runTestWithInput(test_class, test_method, test_input, oreporter)
+ d = runTestWithInput(test_class,
+ test_method, test_input, oreporter)
except Exception, e:
print e
log.debug("here y0")
@@ -223,10 +224,11 @@ def runTestCases(test_cases, options,
test_inputs = [None]
reportFile = open(yamloo_filename, 'w+')
- oreporter = reporter.OReporter(reportFile)
+ oreporter = reporter.YAMLReporter(reportFile)
+
input_unit_factory = InputUnitFactory(test_inputs)
- yield oreporter.writeReportHeader(options)
+ yield oreporter.createReport(options)
# This deferred list is a deferred list of deferred lists
# it is used to store all the deferreds of the tests that
# are run
diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py
index 4eef366..8912bbb 100644
--- a/ooni/utils/hacks.py
+++ b/ooni/utils/hacks.py
@@ -8,6 +8,11 @@
# :authors: Arturo Filastò
# :licence: see LICENSE
+from yaml.representer import *
+from yaml.emitter import *
+from yaml.serializer import *
+from yaml.resolver import *
+
import copy_reg
def patched_reduce_ex(self, proto):
@@ -61,3 +66,45 @@ def patched_reduce_ex(self, proto):
return copy_reg._reconstructor, args, dict
else:
return copy_reg._reconstructor, args
+
+class OSafeRepresenter(SafeRepresenter):
+ """
+ This is a custom YAML representer that allows us to represent reports
+ safely.
+ It extends the SafeRepresenter to be able to also represent complex numbers
+ """
+ def represent_complex(self, data):
+ if data.imag == 0.0:
+ data = u'%r' % data.real
+ elif data.real == 0.0:
+ data = u'%rj' % data.imag
+ elif data.imag > 0:
+ data = u'%r+%rj' % (data.real, data.imag)
+ else:
+ data = u'%r%rj' % (data.real, data.imag)
+ return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data)
+
+OSafeRepresenter.add_representer(complex,
+ OSafeRepresenter.represent_complex)
+
+class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):
+ """
+ This is a modification of the YAML Safe Dumper to use our own Safe
+ Representer that supports complex numbers.
+ """
+ def __init__(self, stream,
+ default_style=None, default_flow_style=None,
+ canonical=None, indent=None, width=None,
+ allow_unicode=None, line_break=None,
+ encoding=None, explicit_start=None, explicit_end=None,
+ version=None, tags=None):
+ Emitter.__init__(self, stream, canonical=canonical,
+ indent=indent, width=width,
+ allow_unicode=allow_unicode, line_break=line_break)
+ Serializer.__init__(self, encoding=encoding,
+ explicit_start=explicit_start, explicit_end=explicit_end,
+ version=version, tags=tags)
+ OSafeRepresenter.__init__(self, default_style=default_style,
+ default_flow_style=default_flow_style)
+ Resolver.__init__(self)
+
More information about the tor-commits
mailing list