commit 572b301710f395620c557dde8ecdef0771ccbc88 Author: Arturo Filastò art@fuffa.org Date: Sat Nov 10 23:32:48 2012 +0100

Refactor reporter Object. * We now have a parent OReporter object that is subclassed by OONIBReporter for remote reporting and YAMLReporter for reporting to YAML format on file system. * Move secure YAML serialization hacks to the hacks module * Do more progress on the implementation of reporting to remote systems --- ooni/reporter.py | 282 ++++++++++++++++++++++++++------------------------- ooni/runner.py | 8 +- ooni/utils/hacks.py | 47 +++++++++ 3 files changed, 197 insertions(+), 140 deletions(-)

diff --git a/ooni/reporter.py b/ooni/reporter.py index 5d29d54..b625603 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -26,6 +26,8 @@ from twisted.internet import defer, reactor

from ooni.templates.httpt import BodyReceiver, StringProducer from ooni.utils import otime, log, geodata + +from ooni.utils.hacks import OSafeRepresenter, OSafeDumper from ooni import config

try: @@ -38,47 +40,8 @@ except: Packet = object packet = FooClass

-class OSafeRepresenter(SafeRepresenter): - """ - This is a custom YAML representer that allows us to represent reports - safely. - It extends the SafeRepresenter to be able to also represent complex numbers - """ - def represent_complex(self, data): - if data.imag == 0.0: - data = u'%r' % data.real - elif data.real == 0.0: - data = u'%rj' % data.imag - elif data.imag > 0: - data = u'%r+%rj' % (data.real, data.imag) - else: - data = u'%r%rj' % (data.real, data.imag) - return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data) - -OSafeRepresenter.add_representer(complex, - OSafeRepresenter.represent_complex) - -class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): - """ - This is a modification of the YAML Safe Dumper to use our own Safe - Representer that supports complex numbers. - """ - def __init__(self, stream, - default_style=None, default_flow_style=None, - canonical=None, indent=None, width=None, - allow_unicode=None, line_break=None, - encoding=None, explicit_start=None, explicit_end=None, - version=None, tags=None): - Emitter.__init__(self, stream, canonical=canonical, - indent=indent, width=width, - allow_unicode=allow_unicode, line_break=line_break) - Serializer.__init__(self, encoding=encoding, - explicit_start=explicit_start, explicit_end=explicit_end, - version=version, tags=tags) - OSafeRepresenter.__init__(self, default_style=default_style, - default_flow_style=default_flow_style) - Resolver.__init__(self) - +class NoTestIDSpecified(Exception): + pass

def safe_dump(data, stream=None, **kw): """ @@ -86,42 +49,93 @@ def safe_dump(data, stream=None, **kw): """ return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw)

-class OONIBReporter(object): - def __init__(self, backend_url): - from twisted.web.client import Agent - from twisted.internet import reactor +@defer.inlineCallbacks +def getTestDetails(options): + client_geodata = {} + + if config.privacy.includeip or \ + config.privacy.includeasn or \ + config.privacy.includecountry or \ + config.privacy.includecity: + log.msg("Running geo IP lookup via check.torproject.org") + client_ip = yield geodata.myIP() + client_location = geodata.IPToLocation(client_ip) + else: + client_ip = "127.0.0.1" + + if config.privacy.includeip: + client_geodata['ip'] = client_ip + else: + client_geodata['ip'] = "127.0.0.1" + + client_geodata['asn'] = None + client_geodata['city'] = None + client_geodata['countrycode'] = None + + if config.privacy.includeasn: + client_geodata['asn'] = client_location['asn'] + + if config.privacy.includecity: + client_geodata['city'] = client_location['city'] + + if config.privacy.includecountry: + client_geodata['countrycode'] = client_location['countrycode'] + + test_details = {'start_time': otime.utcTimeNow(), + 'probe_asn': client_geodata['asn'], + 'probe_cc': client_geodata['countrycode'], + 'probe_ip': client_geodata['ip'], + 'test_name': options['name'], + 'test_version': options['version'], + } + defer.returnValue(test_details) + + +class OReporter(object): + def createReport(options): + """ + Override this with your own logic to implement tests. + """ + raise NotImplemented

- self.agent = Agent(reactor) - self.backend_url = backend_url + def writeReportEntry(self, entry): + """ + Takes as input an entry and writes a report for it. + """ + raise NotImplemented

- def _newReportCreated(self, data): - log.debug("Got this as result: %s" % data) - return data + def finish(): + pass

- def _processResponseBody(self, response, body_cb): - log.debug("Got response %s" % response) - done = defer.Deferred() - response.deliverBody(BodyReceiver(done)) - done.addCallback(body_cb) - return done + def testDone(self, test, test_name): + test_report = dict(test.report)

- def newReport(self, test_name, test_version): - url = self.backend_url + '/new' - software_version = '0.0.1' + # XXX the scapy test has an example of how + # to do this properly. + if isinstance(test.input, packet.Packet): + test_input = repr(test.input) + else: + test_input = test.input

- request = {'software_name': 'ooni-probe', - 'software_version': software_version, - 'test_name': test_name, 'test_version': test_version, - 'progress': 0} + test_started = test._start_time + test_runtime = test_started - time.time()

- log.debug("Creating report via url %s" % url) - bodyProducer = StringProducer(json.dumps(request)) - d = self.agent.request("POST", url, bodyProducer=bodyProducer) - d.addCallback(self._processResponseBody, self._newReportCreated) - return d + report = {'input': test_input, + 'test_name': test_name, + 'test_started': test_started, + 'report': test_report} + self.writeReportEntry(report)

+ def allDone(self): + log.debug("allDone: Finished running all tests") + self.finish() + try: + reactor.stop() + except: + pass + return None

-class YamlReporter(object): +class YAMLReporter(OReporter): """ These are useful functions for reporting to YAML format. """ @@ -145,91 +159,85 @@ class YamlReporter(object): self._write(safe_dump(entry)) self._write('...\n')

- def finish(self): - self._stream.close() - -class OReporter(YamlReporter): - """ - This is a reporter factory. It emits new instances of Reports. It is also - responsible for writing the OONI Report headers. - """ - def writeTestsReport(self, tests): - for test in tests.values(): - self.writeReportEntry(test) - @defer.inlineCallbacks - def writeReportHeader(self, options): - self.firstrun = False + def createReport(self, options): self._writeln("###########################################") self._writeln("# OONI Probe Report for %s test" % options['name']) self._writeln("# %s" % otime.prettyDateNow()) self._writeln("###########################################")

- client_geodata = {} + test_details = yield getTestDetails(options)

- if config.privacy.includeip or \ - config.privacy.includeasn or \ - config.privacy.includecountry or \ - config.privacy.includecity: - log.msg("Running geo IP lookup via check.torproject.org") - client_ip = yield geodata.myIP() - client_location = geodata.IPToLocation(client_ip) - else: - client_ip = "127.0.0.1" + self.writeReportEntry(test_details)

- if config.privacy.includeip: - client_geodata['ip'] = client_ip - else: - client_geodata['ip'] = "127.0.0.1" + def finish(self): + self._stream.close()

- client_geodata['asn'] = None - client_geodata['city'] = None - client_geodata['countrycode'] = None +class OONIBReporter(object): + def __init__(self, backend_url): + from twisted.web.client import Agent + from twisted.internet import reactor + self.agent = Agent(reactor) + self.backend_url = backend_url

- if config.privacy.includeasn: - client_geodata['asn'] = client_location['asn'] + def _newReportCreated(self, data): + log.debug("newReportCreated %s" % data) + return data

- if config.privacy.includecity: - client_geodata['city'] = client_location['city'] + def _processResponseBody(self, response, body_cb): + log.debug("processResponseBody %s" % response) + done = defer.Deferred() + response.deliverBody(BodyReceiver(done)) + done.addCallback(body_cb) + return done

- if config.privacy.includecountry: - client_geodata['countrycode'] = client_location['countrycode'] + def createReport(self, test_name, + test_version, report_header):

+ url = self.backend_url + '/new' + software_version = '0.0.1'

- test_details = {'start_time': otime.utcTimeNow(), - 'probe_asn': client_geodata['asn'], - 'probe_cc': client_geodata['countrycode'], - 'probe_ip': client_geodata['ip'], - 'test_name': options['name'], - 'test_version': options['version'], - } - self.writeReportEntry(test_details) + request = {'software_name': 'ooni-probe', + 'software_version': software_version, + 'test_name': test_name, + 'test_version': test_version, + 'progress': 0, + 'content': report_header + } + def gotDetails(test_details): + log.debug("Creating report via url %s" % url) + + bodyProducer = StringProducer(json.dumps(request)) + d = self.agent.request("POST", url, + bodyProducer=bodyProducer) + d.addCallback(self._processResponseBody, + self._newReportCreated) + return d + + d = getTestDetails(options) + d.addCallback(gotDetails) + return d

- def testDone(self, test, test_name): - test_report = dict(test.report) + def writeReportEntry(self, entry, test_id=None): + if not test_id: + log.err("Write report entry on OONIB requires test id") + raise NoTestIDSpecified

- # XXX the scapy test has an example of how - # to do this properly. - if isinstance(test.input, packet.Packet): - test_input = repr(test.input) - else: - test_input = test.input + report = '---\n' + report += safe_dump(entry) + report += '...\n'

- test_started = test._start_time - test_runtime = test_started - time.time() + url = self.backend_url + '/new'

- report = {'input': test_input, - 'test_name': test_name, - 'test_started': test_started, - 'report': test_report} - self.writeReportEntry(report) + request = {'test_id': test_id, + 'content': report} + + bodyProducer = StringProducer(json.dumps(request)) + d = self.agent.request("PUT", url, + bodyProducer=bodyProducer) + + d.addCallback(self._processResponseBody, + self._newReportCreated) + return d

- def allDone(self): - log.debug("allDone: Finished running all tests") - self.finish() - try: - reactor.stop() - except: - pass - return None

diff --git a/ooni/runner.py b/ooni/runner.py index 1affb15..07882cf 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -194,7 +194,8 @@ def runTestWithInputUnit(test_class, for test_input in input_unit: log.debug("IU: %s" % test_input) try: - d = runTestWithInput(test_class, test_method, test_input, oreporter) + d = runTestWithInput(test_class, + test_method, test_input, oreporter) except Exception, e: print e log.debug("here y0") @@ -223,10 +224,11 @@ def runTestCases(test_cases, options, test_inputs = [None]

reportFile = open(yamloo_filename, 'w+') - oreporter = reporter.OReporter(reportFile) + oreporter = reporter.YAMLReporter(reportFile) + input_unit_factory = InputUnitFactory(test_inputs)

- yield oreporter.writeReportHeader(options) + yield oreporter.createReport(options) # This deferred list is a deferred list of deferred lists # it is used to store all the deferreds of the tests that # are run diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py index 4eef366..8912bbb 100644 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@ -8,6 +8,11 @@ # :authors: Arturo Filastò # :licence: see LICENSE

+from yaml.representer import * +from yaml.emitter import * +from yaml.serializer import * +from yaml.resolver import * + import copy_reg

def patched_reduce_ex(self, proto): @@ -61,3 +66,45 @@ def patched_reduce_ex(self, proto): return copy_reg._reconstructor, args, dict else: return copy_reg._reconstructor, args + +class OSafeRepresenter(SafeRepresenter): + """ + This is a custom YAML representer that allows us to represent reports + safely. + It extends the SafeRepresenter to be able to also represent complex numbers + """ + def represent_complex(self, data): + if data.imag == 0.0: + data = u'%r' % data.real + elif data.real == 0.0: + data = u'%rj' % data.imag + elif data.imag > 0: + data = u'%r+%rj' % (data.real, data.imag) + else: + data = u'%r%rj' % (data.real, data.imag) + return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data) + +OSafeRepresenter.add_representer(complex, + OSafeRepresenter.represent_complex) + +class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): + """ + This is a modification of the YAML Safe Dumper to use our own Safe + Representer that supports complex numbers. + """ + def __init__(self, stream, + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + OSafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style) + Resolver.__init__(self) +