commit a319db38669313aa8bd5a31f1c991a300434dfbc Author: Arturo Filastò hellais@torproject.org Date: Thu May 31 04:42:55 2012 +0200

Implement logging and reporting * Port various tests to new model --- .gitignore | 2 +- ooni/__init__.py | 1 + ooni/log.py | 35 +++++++++ ooni/oonicli.py | 8 ++- ooni/plugins/blocking.py | 2 +- ooni/plugins/dropin.cache | 164 ------------------------------------------- ooni/plugins/httphost.py | 3 +- ooni/plugins/scaffolding.py | 66 ----------------- ooni/plugins/skel.py | 1 + ooni/plugoo/reports.py | 39 ++++------ ooni/plugoo/tests.py | 13 ++-- ooni/plugoo/work.py | 7 ++- ooni/scaffolding.py | 67 ++++++++++++++++++ 13 files changed, 143 insertions(+), 265 deletions(-)

diff --git a/.gitignore b/.gitignore index 8ed47e9..553482d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ proxy-lists/ips.txt proxy-lists/italy-dns-ips.txt proxy-lists/italy-http-ips.txt private/* -ooni/plugins/dropin.cache +/ooni/plugins/dropin.cache diff --git a/ooni/__init__.py b/ooni/__init__.py new file mode 100644 index 0000000..dbb3824 --- /dev/null +++ b/ooni/__init__.py @@ -0,0 +1 @@ +__all__ = ['plugoo', 'utils', 'config', 'logo', 'lib'] diff --git a/ooni/log.py b/ooni/log.py new file mode 100644 index 0000000..bf6ea69 --- /dev/null +++ b/ooni/log.py @@ -0,0 +1,35 @@ +""" +OONI logging facility. +""" +import sys +import logging +import warnings + +from twisted.python import log + +# Logging levels +DEBUG = logging.DEBUG +INFO = logging.INFO +WARNING = logging.WARNING +ERROR = logging.ERROR +CRITICAL = logging.CRITICAL + +def _get_log_level(level): + if not level: + return INFO + else: + return level + +def start(logfile=None, loglevel=None, logstdout=True): + if log.defaultObserver: + loglevel = _get_log_level(loglevel) + logfile = logfile + file = open(logfile, 'a') if logfile else sys.stderr + log.startLogging(file, setStdout=logstdout) + msg("Started OONI") + +def msg(message, level=INFO, **kw): + log.msg(message, **kw) + +def err(message, **kw): + log.err(message, **kw) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index a0272fb..d612ba7 100755 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -19,9 +19,9 @@ from zope.interface.exceptions import BrokenMethodImplementation from zope.interface.verify import verifyObject from pprint import pprint

-from ooni.plugoo import tests, work, assets +from ooni.plugoo import tests, work, assets, reports from ooni.logo import getlogo -from ooni import plugins +from ooni import plugins, log

__version__ = "0.0.1-prealpha"

@@ -52,13 +52,15 @@ def runTest(test, options, global_options): parallelism = int(global_options['parallelism']) worker = work.Worker(parallelism) test = plugoo[test].__class__ + report = reports.Report(global_options['output'])

#if options['asset']: # print options['asset'] # asset = assets.Asset(options['asset']) # print asset + log.start(global_options['log'], 1)

- wgen = work.WorkGenerator(test(options, global_options), + wgen = work.WorkGenerator(test(options, global_options, report), dict(options), start=options['resume'])

diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py index bbcb1a2..4638834 100644 --- a/ooni/plugins/blocking.py +++ b/ooni/plugins/blocking.py @@ -8,7 +8,7 @@ from ooni.plugoo.tests import ITest, TwistedTest class BlockingArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], ['resume', 'r', 0, 'Resume at this index'], - ['other', 'o', None, 'Other arguments']] + ['shit', 'o', None, 'Other arguments']]

class BlockingTest(TwistedTest): implements(IPlugin, ITest) diff --git a/ooni/plugins/dropin.cache b/ooni/plugins/dropin.cache deleted file mode 100755 index bd26c09..0000000 --- a/ooni/plugins/dropin.cache +++ /dev/null @@ -1,164 +0,0 @@ -(dp1 -S'skel' -p2 -ccopy_reg -_reconstructor -p3 -(ctwisted.plugin -CachedDropin -p4 -c__builtin__ -object -p5 -NtRp6 -(dp7 -S'moduleName' -p8 -S'plugins.skel' -p9 -sS'description' -p10 -NsS'plugins' -p11 -(lp12 -g3 -(ctwisted.plugin -CachedPlugin -p13 -g5 -NtRp14 -(dp15 -S'provided' -p16 -(lp17 -ctwisted.plugin -IPlugin -p18 -acplugoo.tests -ITest -p19 -asS'dropin' -p20 -g6 -sS'name' -p21 -S'skel' -p22 -sg10 -NsbasbsS'bridget' -p23 -g3 -(g4 -g5 -NtRp24 -(dp25 -g8 -S'ooni.plugins.bridget' -p26 -sg10 -Nsg11 -(lp27 -g3 -(g13 -g5 -NtRp28 -(dp29 -g16 -(lp30 -g18 -ag19 -asg20 -g24 -sg21 -S'bridget' -p31 -sg10 -NsbasbsS'blocking' -p32 -g3 -(g4 -g5 -NtRp33 -(dp34 -g8 -S'ooni.plugins.blocking' -p35 -sg10 -Nsg11 -(lp36 -g3 -(g13 -g5 -NtRp37 -(dp38 -g16 -(lp39 -g18 -acooni.plugoo.tests -ITest -p40 -asg20 -g33 -sg21 -S'blocking' -p41 -sg10 -NsbasbsS'httphost' -p42 -g3 -(g4 -g5 -NtRp43 -(dp44 -g8 -S'ooni.plugins.httphost' -p45 -sg10 -S'\n HTTP Host based filtering\n *************************\n\n This test detect HTTP Host field\n based filtering.\n' -p46 -sg11 -(lp47 -g3 -(g13 -g5 -NtRp48 -(dp49 -g16 -(lp50 -g18 -ag40 -asg20 -g43 -sg21 -S'httphost' -p51 -sg10 -NsbasbsS'tcpscan' -p52 -g3 -(g4 -g5 -NtRp53 -(dp54 -g8 -S'plugins.tcpscan' -p55 -sg10 -Nsg11 -(lp56 -g3 -(g13 -g5 -NtRp57 -(dp58 -g16 -(lp59 -g18 -ag40 -asg20 -g53 -sg21 -S'tcpscan' -p60 -sg10 -Nsbasbs. \ No newline at end of file diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py index 17e2e11..9b4de22 100644 --- a/ooni/plugins/httphost.py +++ b/ooni/plugins/httphost.py @@ -89,6 +89,8 @@ class HTTPHostTest(TwistedTest): return censored

def httplib_test(self, control_server, host): + censored = None + response = None try: conn = httplib.HTTPConnection(control_server) conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) @@ -100,7 +102,6 @@ class HTTPHostTest(TwistedTest): censored = self.is_censored(response) except Exception, e: censored = "Error! %s" % e - respopnse = None

return (censored, response)

diff --git a/ooni/plugins/scaffolding.py b/ooni/plugins/scaffolding.py deleted file mode 100755 index fb7002d..0000000 --- a/ooni/plugins/scaffolding.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -""" -This script should be used for creating the scaffolding for a test. -""" -import os -import sys - -test_template = """""" -This is a self genrated test created by scaffolding.py. -you will need to fill it up with all your necessities. -Safe hacking :). -""" -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from plugoo.tests import ITest, TwistedTest - -class %(testName)sArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['resume', 'r', 0, 'Resume at this index']] - -class %(testName)sTest(TwistedTest): - implements(IPlugin, ITest) - - shortName = "%(testShortname)s" - description = "%(testName)s" - requirements = None - options = %(testName)sArgs - blocking = True - - def control(self, experiment_result, args): - # What you return here ends up inside of the report. - return {} - - def experiment(self, args): - # What you return here gets handed as input to control - return {} - - def load_assets(self): - if self.local_options: - return {'asset': Asset(self.local_options['asset'])} - else: - return {} - -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -%(testShortname)s = %(testName)sTest(None, None, None) -""" - -test_vars = {'testName': None, 'testShortname': None} -test_vars['testName'] = raw_input('Test Name: ') -test_vars['testShortname'] = raw_input("Test Short Name: ") - -fname = test_vars['testShortname']+'.py' - -if os.path.exists(fname): - print 'WARNING! File named "%s" already exists.' % fname - if raw_input("Do you wish to continue (y/N)? ").lower() != 'y': - print "gotcha! Dying.." - sys.exit(0) - -fp = open(fname, 'w') -fp.write(test_template % test_vars) -fp.close() - diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py index 00e4311..c5eec47 100644 --- a/ooni/plugins/skel.py +++ b/ooni/plugins/skel.py @@ -2,6 +2,7 @@ from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin from plugoo.tests import ITest, TwistedTest +from ooni import log

class SkelArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index c099456..44634b8 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -2,9 +2,8 @@ import os from datetime import datetime import yaml

-import logging import itertools -import gevent +from ooni import log

class Report: """This is the ooni-probe reporting mechanism. It allows @@ -20,29 +19,27 @@ class Report: inbound connection and accept a stream of data (think of it as a `nc -l -p <port> > filename.txt`) """ - def __init__(self, ooni, - scp="127.0.0.1:22", - file="test.report", - tcp="127.0.0.1:9000"): + def __init__(self, file="test.report", + scp=None, + tcp=None):

self.file = file self.tcp = tcp self.scp = scp - self.config = ooni.config.report - self.logger = ooni.logger + #self.config = ooni.config.report

- if self.config.timestamp: - tmp = self.file.split('.') - self.file = '.'.join(tmp[:-1]) + "-" + \ - datetime.now().isoformat('-') + '.' + \ - tmp[-1] - print self.file + #if self.config.timestamp: + # tmp = self.file.split('.') + # self.file = '.'.join(tmp[:-1]) + "-" + \ + # datetime.now().isoformat('-') + '.' + \ + # tmp[-1] + # print self.file

try: import paramiko except: self.scp = None - self.logger.warn("Could not import paramiko. SCP will not be disabled") + log.err("Could not import paramiko. SCP will not be disabled")

def __call__(self, data): """ @@ -64,13 +61,9 @@ class Report: if self.scp: reports.append("scp")

- jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports] - gevent.joinall(jobs) - ret = [] - for job in jobs: - #print job.value - ret.append(job.value) - return ret + #XXX make this non blocking + for report in reports: + self.send_report(dump, report)

def file_report(self, data, file=None, mode='a+'): """ @@ -169,7 +162,7 @@ class Report: specified type. """ #print "Reporting %s to %s" % (data, type) - self.logger.info("Reporting to %s" % type) + log.msg("Reporting to %s" % type) getattr(self, type+"_report").__call__(data)

diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 75d0765..fd19479 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -10,6 +10,7 @@ import gevent from twisted.internet import reactor, defer, threads from twisted.python import failure

+from ooni import log from ooni.plugoo import assets, work from ooni.plugoo.reports import Report

@@ -142,10 +143,11 @@ class ITest(Interface): class TwistedTest(object): blocking = False

- def __init__(self, local_options, global_options, ooninet=None): + def __init__(self, local_options, global_options, report, ooninet=None): self.local_options = local_options self.global_options = global_options self.assets = self.load_assets() + self.report = report #self.ooninet = ooninet

def load_assets(self): @@ -167,7 +169,8 @@ class TwistedTest(object): result['end_time'] = self.end_time result['run_time'] = self.end_time - self.start_time result['control'] = control - print "FINISHED", result + log.msg("FINISHED %s" % result) + self.report(result) return result

def _do_experiment(self, args): @@ -181,15 +184,15 @@ class TwistedTest(object): return self.d

def control(self, result, args): - print "Doing control..." + log.msg("Doing control") return result

def experiment(self, args): - print "Doing experiment" + log.msg("Doing experiment") return {}

def startTest(self, args): self.start_time = datetime.now() - print "Starting test %s" % self.__class__ + log.msg("Starting test %s" % self.__class__) return self._do_experiment(args)

diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 55bbd1d..b9fc6b8 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -38,13 +38,18 @@ class Worker(object): asset, test, idx = workunit self._running += 1 actuald = test.startTest(asset).addBoth(self._run) + if isinstance(r, failure.Failure): r.trap()

- print "Callback fired!" print r['start_time'] print r['end_time'] print r['run_time'] + + if self._running == 0 and not self._queued: + print "I am done." + reactor.stop() + return r

def push(self, workunit): diff --git a/ooni/scaffolding.py b/ooni/scaffolding.py new file mode 100755 index 0000000..b1d3f59 --- /dev/null +++ b/ooni/scaffolding.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +""" +This script should be used for creating the scaffolding for a test. +""" +import os +import sys +from ooni import log + +test_template = """""" +This is a self genrated test created by scaffolding.py. +you will need to fill it up with all your necessities. +Safe hacking :). +""" +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from plugoo.tests import ITest, TwistedTest + +class %(testName)sArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index']] + +class %(testName)sTest(TwistedTest): + implements(IPlugin, ITest) + + shortName = "%(testShortname)s" + description = "%(testName)s" + requirements = None + options = %(testName)sArgs + blocking = True + + def control(self, experiment_result, args): + # What you return here ends up inside of the report. + return {} + + def experiment(self, args): + # What you return here gets handed as input to control + return {} + + def load_assets(self): + if self.local_options: + return {'asset': Asset(self.local_options['asset'])} + else: + return {} + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +%(testShortname)s = %(testName)sTest(None, None, None) +""" + +test_vars = {'testName': None, 'testShortname': None} +test_vars['testName'] = raw_input('Test Name: ') +test_vars['testShortname'] = raw_input("Test Short Name: ") + +fname = os.path.join('plugins', test_vars['testShortname']+'.py') + +if os.path.exists(fname): + print 'WARNING! File named "%s" already exists.' % fname + if raw_input("Do you wish to continue (y/N)? ").lower() != 'y': + print "gotcha! Dying.." + sys.exit(0) + +fp = open(fname, 'w') +fp.write(test_template % test_vars) +fp.close() +