[tor-commits] [ooni-probe/master] Merge branch 'master' into remotereport
art at torproject.org
art at torproject.org
Sun Nov 11 17:18:31 UTC 2012
commit fb7c4962063161e621691b128f23d70402c4da36
Merge: 10f0d90 9e7cf8c
Author: Arturo Filastò <art at fuffa.org>
Date: Sun Nov 11 16:48:30 2012 +0100
Merge branch 'master' into remotereport
* master:
Implement basic keyword filtering detection test. * XXX Does not use the bisection method * XXX Does not establish a TCP session, but just sends
Do some major refactoring of the ooniprobe backend * Implement TCP Echo test helper * Refactor testhelpers into proper namespace * Remove unused imports * Remove files that were not being imported anywhere
Respect the includeip = false option in ooniprobe.conf for scapyt derivate tests * If the option to not include the IP address of the probe is set, change the source and destination ip address of the sent and received packets to 127.0.0.1
Add ooniprobe prefix to logs
Clean up filenames of generated reports * They now follow the format $testName_report_$timestamp.yamloo
Implement ICMP traceroute test * parametrize max_ttl and timeout * make max_ttl and timeout be included in the report
Add not about the fact that tests should return a deferred.
Add software version to the report * Cleanup some debug messages * Remove unused ooni-probe.conf config file
Implement basic reporting to flat files containing the report ID. * XXX the location of such files is not configurable, they will leave tons of crap in your cwd.
Refactor reporter Object. * We now have a parent OReporter object that is subclassed by OONIBReporter for remote reporting and YAMLReporter for reporting to YAML format on file system. * Move secure YAML serialization hacks to the hacks module * Do more progress on the implementation of reporting to remote systems
Conflicts:
ooni/reporter.py
ooni/runner.py
nettests/core/keyword_filtering.py | 52 +++++++
nettests/core/squid.py | 2 +-
nettests/core/traceroute.py | 70 ++++++++--
ooni/__init__.py | 4 +
ooni/config.py | 9 +-
ooni/inputunit.py | 2 +-
ooni/nettest.py | 3 +
ooni/ooni-probe.conf | 100 --------------
ooni/oonicli.py | 8 +-
ooni/reporter.py | 265 ++++++++++++++++++------------------
ooni/runner.py | 14 ++-
ooni/templates/scapyt.py | 18 ++-
ooni/utils/hacks.py | 47 +++++++
ooni/utils/log.py | 4 +-
oonib/config.py | 39 ++++--
oonib/lib/__init__.py | 1 -
oonib/lib/ssl.py | 8 -
oonib/oonibackend.conf.sample | 12 --
oonib/oonibackend.py | 79 ++++++-----
oonib/report/api.py | 123 +++++++++++++++--
oonib/testhelpers/__init__.py | 5 +
oonib/testhelpers/dns.py | 16 --
oonib/testhelpers/dns_helpers.py | 16 ++
oonib/testhelpers/http_helpers.py | 80 +++++++++++
oonib/testhelpers/httph.py | 80 -----------
oonib/testhelpers/ssl_helpers.py | 9 ++
oonib/testhelpers/tcp_helpers.py | 14 ++
ooniprobe.conf | 3 +-
28 files changed, 646 insertions(+), 437 deletions(-)
diff --cc nettests/core/squid.py
index 675119c,675119c..777bc3e
--- a/nettests/core/squid.py
+++ b/nettests/core/squid.py
@@@ -17,7 -17,7 +17,7 @@@ class SquidTest(httpt.HTTPTest)
"""
name = "Squid test"
author = "Arturo Filastò"
-- version = 0.1
++ version = "0.1"
optParameters = [['backend', 'b', 'http://ooni.nu/test/', 'Test backend to use']]
diff --cc ooni/reporter.py
index 31c53a9,b5e2039..7147b35
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@@ -168,122 -162,78 +162,129 @@@ class YAMLReporter(OReporter)
self._writeln("# %s" % otime.prettyDateNow())
self._writeln("###########################################")
- client_geodata = {}
+ test_details = yield getTestDetails(options)
- if config.privacy.includeip or \
- config.privacy.includeasn or \
- config.privacy.includecountry or \
- config.privacy.includecity:
- log.msg("Running geo IP lookup via check.torproject.org")
- client_ip = yield geodata.myIP()
- client_location = geodata.IPToLocation(client_ip)
- else:
- client_ip = "127.0.0.1"
+ self.writeReportEntry(test_details)
- if config.privacy.includeip:
- client_geodata['ip'] = client_ip
- else:
- client_geodata['ip'] = "127.0.0.1"
+ def finish(self):
+ self._stream.close()
- client_geodata['asn'] = None
- client_geodata['city'] = None
- client_geodata['countrycode'] = None
+ class OONIBReporter(object):
+ def __init__(self, backend_url):
+ from twisted.web.client import Agent
+ from twisted.internet import reactor
+ self.agent = Agent(reactor)
+ self.backend_url = backend_url
- if config.privacy.includeasn:
- client_geodata['asn'] = client_location['asn']
+ def _newReportCreated(self, data):
+ log.debug("newReportCreated %s" % data)
+ return data
- if config.privacy.includecity:
- client_geodata['city'] = client_location['city']
+ def _processResponseBody(self, response, body_cb):
+ log.debug("processResponseBody %s" % response)
+ done = defer.Deferred()
+ response.deliverBody(BodyReceiver(done))
+ done.addCallback(body_cb)
+ return done
- if config.privacy.includecountry:
- client_geodata['countrycode'] = client_location['countrycode']
+ def createReport(self, test_name,
+ test_version, report_header):
-
+ url = self.backend_url + '/new'
+ software_version = '0.0.1'
- test_details = {'start_time': otime.utcTimeNow(),
- 'probe_asn': client_geodata['asn'],
- 'probe_cc': client_geodata['countrycode'],
- 'probe_ip': client_geodata['ip'],
- 'test_name': options['name'],
- 'test_version': options['version'],
- }
- self.writeReportEntry(test_details)
+ request = {'software_name': 'ooni-probe',
+ 'software_version': software_version,
+ 'test_name': test_name,
+ 'test_version': test_version,
+ 'progress': 0,
+ 'content': report_header
+ }
+ def gotDetails(test_details):
+ log.debug("Creating report via url %s" % url)
+
+ bodyProducer = StringProducer(json.dumps(request))
+ d = self.agent.request("POST", url,
+ bodyProducer=bodyProducer)
+ d.addCallback(self._processResponseBody,
+ self._newReportCreated)
+ return d
+
+ d = getTestDetails(options)
+ d.addCallback(gotDetails)
+ return d
+
+ def writeReportEntry(self, entry, test_id=None):
+ if not test_id:
+ log.err("Write report entry on OONIB requires test id")
+ raise NoTestIDSpecified
+
+ report = '---\n'
+ report += safe_dump(entry)
+ report += '...\n'
- def testDone(self, test, test_name):
- test_report = dict(test.report)
+ url = self.backend_url + '/new'
- # XXX the scapy test has an example of how
- # to do this properly.
- if isinstance(test.input, packet.Packet):
- test_input = repr(test.input)
- else:
- test_input = test.input
+ request = {'test_id': test_id,
+ 'content': report}
- test_started = test._start_time
- test_runtime = test_started - time.time()
+ bodyProducer = StringProducer(json.dumps(request))
+ d = self.agent.request("PUT", url,
+ bodyProducer=bodyProducer)
- report = {'input': test_input,
- 'test_name': test_name,
- 'test_started': test_started,
- 'report': test_report}
- self.writeReportEntry(report)
+ d.addCallback(self._processResponseBody,
+ self._newReportCreated)
+ return d
- def allDone(self):
- log.debug("allDone: Finished running all tests")
- self.finish()
- try:
- reactor.stop()
- except:
- pass
- return None
+
+class OONIBReporter(OReporter):
+ def __init__(self, backend_url):
+ from twisted.web.client import Agent
+ from twisted.internet import reactor
+ self.agent = Agent(reactor)
+ self.backend_url = backend_url
+
+ def _processResponseBody(self, *arg, **kw):
+ #done = defer.Deferred()
+ #response.deliverBody(BodyReceiver(done))
+ #done.addCallback(self._newReportCreated)
+ #return done
+
+ def createReport(self, options):
+ test_name = options['name']
+ test_version = options['version']
+
+ log.debug("Creating report with OONIB Reporter")
+ url = self.backend_url + '/report/new'
+ software_version = '0.0.1'
+
+ def gotDetails(test_details):
+ content = '---\n'
+ content += safe_dump(test_details)
+ content += '...\n'
+
+ request = {'software_name': 'ooniprobe',
+ 'software_version': software_version,
+ 'test_name': test_name,
+ 'test_version': test_version,
+ 'progress': 0,
+ 'content': content
+ }
+ log.debug("Creating report via url %s" % url)
+ request_json = json.dumps(request)
+ log.debug("Sending %s" % request_json)
+
+ def bothCalls(*arg, **kw):
+ print arg, kw
+
+ body_producer = StringProducer(request_json)
+ d = self.agent.request("POST", url, None,
+ body_producer)
+ d.addBoth(self._processResponseBody)
+ return d
+
+ d = getTestDetails(options)
+ d.addCallback(gotDetails)
+ # XXX handle errors
+ return d
+
diff --cc ooni/runner.py
index c2105ea,23ff898..b92ad7e
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@@ -221,16 -221,13 +221,22 @@@ def runTestCases(test_cases, options
log.msg("Could not find inputs!")
log.msg("options[0] = %s" % first)
test_inputs = [None]
-
+
reportFile = open(yamloo_filename, 'w+')
++
+ #oreporter = reporter.YAMLReporter(reportFile)
+ oreporter = reporter.OONIBReporter('http://127.0.0.1:8888')
+
+ input_unit_factory = InputUnitFactory(test_inputs)
+
+ log.debug("Creating report")
+ yield oreporter.createReport(options)
+
+ oreporter = reporter.YAMLReporter(reportFile)
+
+ input_unit_factory = InputUnitFactory(test_inputs)
+
+ yield oreporter.createReport(options)
# This deferred list is a deferred list of deferred lists
# it is used to store all the deferreds of the tests that
# are run
More information about the tor-commits
mailing list