[tor-commits] [ooni-probe/master] Implement logging and reporting
art at torproject.org
art at torproject.org
Thu May 31 03:01:43 UTC 2012
commit a319db38669313aa8bd5a31f1c991a300434dfbc
Author: Arturo Filastò <hellais at torproject.org>
Date: Thu May 31 04:42:55 2012 +0200
Implement logging and reporting
* Port various tests to new model
---
.gitignore | 2 +-
ooni/__init__.py | 1 +
ooni/log.py | 35 +++++++++
ooni/oonicli.py | 8 ++-
ooni/plugins/blocking.py | 2 +-
ooni/plugins/dropin.cache | 164 -------------------------------------------
ooni/plugins/httphost.py | 3 +-
ooni/plugins/scaffolding.py | 66 -----------------
ooni/plugins/skel.py | 1 +
ooni/plugoo/reports.py | 39 ++++------
ooni/plugoo/tests.py | 13 ++--
ooni/plugoo/work.py | 7 ++-
ooni/scaffolding.py | 67 ++++++++++++++++++
13 files changed, 143 insertions(+), 265 deletions(-)
diff --git a/.gitignore b/.gitignore
index 8ed47e9..553482d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,4 +6,4 @@ proxy-lists/ips.txt
proxy-lists/italy-dns-ips.txt
proxy-lists/italy-http-ips.txt
private/*
-ooni/plugins/dropin.cache
+/ooni/plugins/dropin.cache
diff --git a/ooni/__init__.py b/ooni/__init__.py
new file mode 100644
index 0000000..dbb3824
--- /dev/null
+++ b/ooni/__init__.py
@@ -0,0 +1 @@
+__all__ = ['plugoo', 'utils', 'config', 'logo', 'lib']
diff --git a/ooni/log.py b/ooni/log.py
new file mode 100644
index 0000000..bf6ea69
--- /dev/null
+++ b/ooni/log.py
@@ -0,0 +1,35 @@
+"""
+OONI logging facility.
+"""
+import sys
+import logging
+import warnings
+
+from twisted.python import log
+
+# Logging levels
+DEBUG = logging.DEBUG
+INFO = logging.INFO
+WARNING = logging.WARNING
+ERROR = logging.ERROR
+CRITICAL = logging.CRITICAL
+
+def _get_log_level(level):
+ if not level:
+ return INFO
+ else:
+ return level
+
+def start(logfile=None, loglevel=None, logstdout=True):
+ if log.defaultObserver:
+ loglevel = _get_log_level(loglevel)
+ logfile = logfile
+ file = open(logfile, 'a') if logfile else sys.stderr
+ log.startLogging(file, setStdout=logstdout)
+ msg("Started OONI")
+
+def msg(message, level=INFO, **kw):
+ log.msg(message, **kw)
+
+def err(message, **kw):
+ log.err(message, **kw)
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index a0272fb..d612ba7 100755
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -19,9 +19,9 @@ from zope.interface.exceptions import BrokenMethodImplementation
from zope.interface.verify import verifyObject
from pprint import pprint
-from ooni.plugoo import tests, work, assets
+from ooni.plugoo import tests, work, assets, reports
from ooni.logo import getlogo
-from ooni import plugins
+from ooni import plugins, log
__version__ = "0.0.1-prealpha"
@@ -52,13 +52,15 @@ def runTest(test, options, global_options):
parallelism = int(global_options['parallelism'])
worker = work.Worker(parallelism)
test = plugoo[test].__class__
+ report = reports.Report(global_options['output'])
#if options['asset']:
# print options['asset']
# asset = assets.Asset(options['asset'])
# print asset
+ log.start(global_options['log'], 1)
- wgen = work.WorkGenerator(test(options, global_options),
+ wgen = work.WorkGenerator(test(options, global_options, report),
dict(options),
start=options['resume'])
diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py
index bbcb1a2..4638834 100644
--- a/ooni/plugins/blocking.py
+++ b/ooni/plugins/blocking.py
@@ -8,7 +8,7 @@ from ooni.plugoo.tests import ITest, TwistedTest
class BlockingArgs(usage.Options):
optParameters = [['asset', 'a', None, 'Asset file'],
['resume', 'r', 0, 'Resume at this index'],
- ['other', 'o', None, 'Other arguments']]
+ ['shit', 'o', None, 'Other arguments']]
class BlockingTest(TwistedTest):
implements(IPlugin, ITest)
diff --git a/ooni/plugins/dropin.cache b/ooni/plugins/dropin.cache
deleted file mode 100755
index bd26c09..0000000
--- a/ooni/plugins/dropin.cache
+++ /dev/null
@@ -1,164 +0,0 @@
-(dp1
-S'skel'
-p2
-ccopy_reg
-_reconstructor
-p3
-(ctwisted.plugin
-CachedDropin
-p4
-c__builtin__
-object
-p5
-NtRp6
-(dp7
-S'moduleName'
-p8
-S'plugins.skel'
-p9
-sS'description'
-p10
-NsS'plugins'
-p11
-(lp12
-g3
-(ctwisted.plugin
-CachedPlugin
-p13
-g5
-NtRp14
-(dp15
-S'provided'
-p16
-(lp17
-ctwisted.plugin
-IPlugin
-p18
-acplugoo.tests
-ITest
-p19
-asS'dropin'
-p20
-g6
-sS'name'
-p21
-S'skel'
-p22
-sg10
-NsbasbsS'bridget'
-p23
-g3
-(g4
-g5
-NtRp24
-(dp25
-g8
-S'ooni.plugins.bridget'
-p26
-sg10
-Nsg11
-(lp27
-g3
-(g13
-g5
-NtRp28
-(dp29
-g16
-(lp30
-g18
-ag19
-asg20
-g24
-sg21
-S'bridget'
-p31
-sg10
-NsbasbsS'blocking'
-p32
-g3
-(g4
-g5
-NtRp33
-(dp34
-g8
-S'ooni.plugins.blocking'
-p35
-sg10
-Nsg11
-(lp36
-g3
-(g13
-g5
-NtRp37
-(dp38
-g16
-(lp39
-g18
-acooni.plugoo.tests
-ITest
-p40
-asg20
-g33
-sg21
-S'blocking'
-p41
-sg10
-NsbasbsS'httphost'
-p42
-g3
-(g4
-g5
-NtRp43
-(dp44
-g8
-S'ooni.plugins.httphost'
-p45
-sg10
-S'\n HTTP Host based filtering\n *************************\n\n This test detect HTTP Host field\n based filtering.\n'
-p46
-sg11
-(lp47
-g3
-(g13
-g5
-NtRp48
-(dp49
-g16
-(lp50
-g18
-ag40
-asg20
-g43
-sg21
-S'httphost'
-p51
-sg10
-NsbasbsS'tcpscan'
-p52
-g3
-(g4
-g5
-NtRp53
-(dp54
-g8
-S'plugins.tcpscan'
-p55
-sg10
-Nsg11
-(lp56
-g3
-(g13
-g5
-NtRp57
-(dp58
-g16
-(lp59
-g18
-ag40
-asg20
-g53
-sg21
-S'tcpscan'
-p60
-sg10
-Nsbasbs.
\ No newline at end of file
diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py
index 17e2e11..9b4de22 100644
--- a/ooni/plugins/httphost.py
+++ b/ooni/plugins/httphost.py
@@ -89,6 +89,8 @@ class HTTPHostTest(TwistedTest):
return censored
def httplib_test(self, control_server, host):
+ censored = None
+ response = None
try:
conn = httplib.HTTPConnection(control_server)
conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True)
@@ -100,7 +102,6 @@ class HTTPHostTest(TwistedTest):
censored = self.is_censored(response)
except Exception, e:
censored = "Error! %s" % e
- respopnse = None
return (censored, response)
diff --git a/ooni/plugins/scaffolding.py b/ooni/plugins/scaffolding.py
deleted file mode 100755
index fb7002d..0000000
--- a/ooni/plugins/scaffolding.py
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-
-"""
-This script should be used for creating the scaffolding for a test.
-"""
-import os
-import sys
-
-test_template = """\"\"\"
-This is a self genrated test created by scaffolding.py.
-you will need to fill it up with all your necessities.
-Safe hacking :).
-\"\"\"
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from plugoo.tests import ITest, TwistedTest
-
-class %(testName)sArgs(usage.Options):
- optParameters = [['asset', 'a', None, 'Asset file'],
- ['resume', 'r', 0, 'Resume at this index']]
-
-class %(testName)sTest(TwistedTest):
- implements(IPlugin, ITest)
-
- shortName = "%(testShortname)s"
- description = "%(testName)s"
- requirements = None
- options = %(testName)sArgs
- blocking = True
-
- def control(self, experiment_result, args):
- # What you return here ends up inside of the report.
- return {}
-
- def experiment(self, args):
- # What you return here gets handed as input to control
- return {}
-
- def load_assets(self):
- if self.local_options:
- return {'asset': Asset(self.local_options['asset'])}
- else:
- return {}
-
-# We need to instantiate it otherwise getPlugins does not detect it
-# XXX Find a way to load plugins without instantiating them.
-%(testShortname)s = %(testName)sTest(None, None, None)
-"""
-
-test_vars = {'testName': None, 'testShortname': None}
-test_vars['testName'] = raw_input('Test Name: ')
-test_vars['testShortname'] = raw_input("Test Short Name: ")
-
-fname = test_vars['testShortname']+'.py'
-
-if os.path.exists(fname):
- print 'WARNING! File named "%s" already exists.' % fname
- if raw_input("Do you wish to continue (y/N)? ").lower() != 'y':
- print "gotcha! Dying.."
- sys.exit(0)
-
-fp = open(fname, 'w')
-fp.write(test_template % test_vars)
-fp.close()
-
diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py
index 00e4311..c5eec47 100644
--- a/ooni/plugins/skel.py
+++ b/ooni/plugins/skel.py
@@ -2,6 +2,7 @@ from zope.interface import implements
from twisted.python import usage
from twisted.plugin import IPlugin
from plugoo.tests import ITest, TwistedTest
+from ooni import log
class SkelArgs(usage.Options):
optParameters = [['asset', 'a', None, 'Asset file'],
diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py
index c099456..44634b8 100644
--- a/ooni/plugoo/reports.py
+++ b/ooni/plugoo/reports.py
@@ -2,9 +2,8 @@ import os
from datetime import datetime
import yaml
-import logging
import itertools
-import gevent
+from ooni import log
class Report:
"""This is the ooni-probe reporting mechanism. It allows
@@ -20,29 +19,27 @@ class Report:
inbound connection and accept a stream of data (think of it
as a `nc -l -p <port> > filename.txt`)
"""
- def __init__(self, ooni,
- scp="127.0.0.1:22",
- file="test.report",
- tcp="127.0.0.1:9000"):
+ def __init__(self, file="test.report",
+ scp=None,
+ tcp=None):
self.file = file
self.tcp = tcp
self.scp = scp
- self.config = ooni.config.report
- self.logger = ooni.logger
+ #self.config = ooni.config.report
- if self.config.timestamp:
- tmp = self.file.split('.')
- self.file = '.'.join(tmp[:-1]) + "-" + \
- datetime.now().isoformat('-') + '.' + \
- tmp[-1]
- print self.file
+ #if self.config.timestamp:
+ # tmp = self.file.split('.')
+ # self.file = '.'.join(tmp[:-1]) + "-" + \
+ # datetime.now().isoformat('-') + '.' + \
+ # tmp[-1]
+ # print self.file
try:
import paramiko
except:
self.scp = None
- self.logger.warn("Could not import paramiko. SCP will not be disabled")
+ log.err("Could not import paramiko. SCP will not be disabled")
def __call__(self, data):
"""
@@ -64,13 +61,9 @@ class Report:
if self.scp:
reports.append("scp")
- jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports]
- gevent.joinall(jobs)
- ret = []
- for job in jobs:
- #print job.value
- ret.append(job.value)
- return ret
+ #XXX make this non blocking
+ for report in reports:
+ self.send_report(dump, report)
def file_report(self, data, file=None, mode='a+'):
"""
@@ -169,7 +162,7 @@ class Report:
specified type.
"""
#print "Reporting %s to %s" % (data, type)
- self.logger.info("Reporting to %s" % type)
+ log.msg("Reporting to %s" % type)
getattr(self, type+"_report").__call__(data)
diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py
index 75d0765..fd19479 100644
--- a/ooni/plugoo/tests.py
+++ b/ooni/plugoo/tests.py
@@ -10,6 +10,7 @@ import gevent
from twisted.internet import reactor, defer, threads
from twisted.python import failure
+from ooni import log
from ooni.plugoo import assets, work
from ooni.plugoo.reports import Report
@@ -142,10 +143,11 @@ class ITest(Interface):
class TwistedTest(object):
blocking = False
- def __init__(self, local_options, global_options, ooninet=None):
+ def __init__(self, local_options, global_options, report, ooninet=None):
self.local_options = local_options
self.global_options = global_options
self.assets = self.load_assets()
+ self.report = report
#self.ooninet = ooninet
def load_assets(self):
@@ -167,7 +169,8 @@ class TwistedTest(object):
result['end_time'] = self.end_time
result['run_time'] = self.end_time - self.start_time
result['control'] = control
- print "FINISHED", result
+ log.msg("FINISHED %s" % result)
+ self.report(result)
return result
def _do_experiment(self, args):
@@ -181,15 +184,15 @@ class TwistedTest(object):
return self.d
def control(self, result, args):
- print "Doing control..."
+ log.msg("Doing control")
return result
def experiment(self, args):
- print "Doing experiment"
+ log.msg("Doing experiment")
return {}
def startTest(self, args):
self.start_time = datetime.now()
- print "Starting test %s" % self.__class__
+ log.msg("Starting test %s" % self.__class__)
return self._do_experiment(args)
diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py
index 55bbd1d..b9fc6b8 100644
--- a/ooni/plugoo/work.py
+++ b/ooni/plugoo/work.py
@@ -38,13 +38,18 @@ class Worker(object):
asset, test, idx = workunit
self._running += 1
actuald = test.startTest(asset).addBoth(self._run)
+
if isinstance(r, failure.Failure):
r.trap()
- print "Callback fired!"
print r['start_time']
print r['end_time']
print r['run_time']
+
+ if self._running == 0 and not self._queued:
+ print "I am done."
+ reactor.stop()
+
return r
def push(self, workunit):
diff --git a/ooni/scaffolding.py b/ooni/scaffolding.py
new file mode 100755
index 0000000..b1d3f59
--- /dev/null
+++ b/ooni/scaffolding.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+"""
+This script should be used for creating the scaffolding for a test.
+"""
+import os
+import sys
+from ooni import log
+
+test_template = """\"\"\"
+This is a self genrated test created by scaffolding.py.
+you will need to fill it up with all your necessities.
+Safe hacking :).
+\"\"\"
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+from plugoo.tests import ITest, TwistedTest
+
+class %(testName)sArgs(usage.Options):
+ optParameters = [['asset', 'a', None, 'Asset file'],
+ ['resume', 'r', 0, 'Resume at this index']]
+
+class %(testName)sTest(TwistedTest):
+ implements(IPlugin, ITest)
+
+ shortName = "%(testShortname)s"
+ description = "%(testName)s"
+ requirements = None
+ options = %(testName)sArgs
+ blocking = True
+
+ def control(self, experiment_result, args):
+ # What you return here ends up inside of the report.
+ return {}
+
+ def experiment(self, args):
+ # What you return here gets handed as input to control
+ return {}
+
+ def load_assets(self):
+ if self.local_options:
+ return {'asset': Asset(self.local_options['asset'])}
+ else:
+ return {}
+
+# We need to instantiate it otherwise getPlugins does not detect it
+# XXX Find a way to load plugins without instantiating them.
+%(testShortname)s = %(testName)sTest(None, None, None)
+"""
+
+test_vars = {'testName': None, 'testShortname': None}
+test_vars['testName'] = raw_input('Test Name: ')
+test_vars['testShortname'] = raw_input("Test Short Name: ")
+
+fname = os.path.join('plugins', test_vars['testShortname']+'.py')
+
+if os.path.exists(fname):
+ print 'WARNING! File named "%s" already exists.' % fname
+ if raw_input("Do you wish to continue (y/N)? ").lower() != 'y':
+ print "gotcha! Dying.."
+ sys.exit(0)
+
+fp = open(fname, 'w')
+fp.write(test_template % test_vars)
+fp.close()
+
More information about the tor-commits
mailing list