[tor-commits] [ooni-probe/master] Get the new OONI architecture working properly.
art at torproject.org
art at torproject.org
Thu May 31 03:01:43 UTC 2012
commit c4c91d393ba3956b84d46521e2f58d30584b3ea0
Author: Arturo Filastò <hellais at torproject.org>
Date: Thu May 31 03:31:05 2012 +0200
Get the new OONI architecture working properly.
(kudos to the organizers of FreeBird :) )
---
.gitignore | 3 +-
ooni/oonicli.py | 70 ++++++++----------------
ooni/ooniprobe.log | 3 +-
ooni/plugins/.bridget.py.swp | Bin 12288 -> 0 bytes
ooni/plugins/.dnstamper.py.swp | Bin 12288 -> 0 bytes
ooni/plugins/blocking.py | 41 ++++++++++++++
ooni/plugins/bridget.py | 36 ------------
ooni/plugins/dnstamper.py | 22 --------
ooni/plugins/dropin.cache | 90 ++++++++++++++++++++++++++++++-
ooni/plugins/skel.py | 9 +++-
ooni/plugoo/tests.py | 118 ++++++++++++++--------------------------
ooni/plugoo/work.py | 64 +++++++++++++++-------
ooni/skel.py | 10 ----
oonib/oonibackend.py | 8 ++--
14 files changed, 255 insertions(+), 219 deletions(-)
diff --git a/.gitignore b/.gitignore
index 6433d66..8ed47e9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,5 @@ proxy-lists/ip-cc.txt
proxy-lists/ips.txt
proxy-lists/italy-dns-ips.txt
proxy-lists/italy-http-ips.txt
-private/*
\ No newline at end of file
+private/*
+ooni/plugins/dropin.cache
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index d290e68..a0272fb 100755
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -9,7 +9,6 @@
#
import sys
-from plugoo import tests, work, assets
from twisted.python import usage
from twisted.plugin import getPlugins
@@ -19,8 +18,10 @@ from zope.interface.exceptions import BrokenImplementation
from zope.interface.exceptions import BrokenMethodImplementation
from zope.interface.verify import verifyObject
from pprint import pprint
-from logo import getlogo
-import plugins
+
+from ooni.plugoo import tests, work, assets
+from ooni.logo import getlogo
+from ooni import plugins
__version__ = "0.0.1-prealpha"
@@ -46,33 +47,20 @@ def retrieve_plugoo():
plugoo = retrieve_plugoo()
-class StupidAsset(object):
- def __init__(self):
- self.idx = 0
-
- def __iter__(self):
- return self
-
- def next(self):
- if self.idx > 30:
- raise StopIteration
- self.idx += 1
- return self.idx
-
-
def runTest(test, options, global_options):
- asset = None
- if options['asset']:
- print options['asset']
- asset = assets.Asset(options['asset'])
- print asset
- wgen = work.WorkGenerator(asset, plugoo[test].__class__,
- dict(options), start=options['resume'])
+ parallelism = int(global_options['parallelism'])
+ worker = work.Worker(parallelism)
+ test = plugoo[test].__class__
+
+ #if options['asset']:
+ # print options['asset']
+ # asset = assets.Asset(options['asset'])
+ # print asset
- if global_options['parallelism']:
- wgen.size = int(global_options['parallelism'])
- worker = work.Worker(wgen.size)
+ wgen = work.WorkGenerator(test(options, global_options),
+ dict(options),
+ start=options['resume'])
for x in wgen:
worker.push(x)
@@ -86,16 +74,17 @@ class Options(usage.Options):
subCommands.append([test, None, plugoo[test].options, "Run the %s test" % test])
optFlags = [
- ['local', 'l', "If the test should be run locally (without having oonid running)"],
- ['status', 'x', 'Show current state'],
- ['restart', 'r', 'Restart OONI']
+ #['remote', 'r', "If the test should be run remotely (not supported)"],
+ #['status', 'x', 'Show current state'],
+ #['restart', 'r', 'Restart OONI']
]
optParameters = [
['parallelism', 'n', 10, "Specify the number of parallel tests to run"],
- ['target-node', 't', 'localhost:31415', 'Select target node'],
- ['ooninet', 'o', 'localhost:4242', "Select OONI-net address for reporting"],
- ['password', 'p', 'opennetwork', "Specify the password for authentication"],
+ #['target-node', 't', 'localhost:31415', 'Select target node'],
+ ['output', 'o', 'report.log', "Specify output report file"],
+ ['log', 'l', 'oonicli.log', "Specify output log file"],
+ #['password', 'p', 'opennetwork', "Specify the password for authentication"],
]
def opt_version(self):
@@ -116,24 +105,11 @@ class Options(usage.Options):
config = Options()
config.parseOptions()
-if config['status']:
- print "oonid is not running."
- sys.exit(0)
-
-if config['restart']:
- print "Restarting oonid."
- sys.exit(0)
-
if not config.subCommand:
print "Error! No Test Specified."
config.opt_help()
sys.exit(1)
-if config['local']:
- runTest(config.subCommand, config.subOptions, config)
+runTest(config.subCommand, config.subOptions, config)
-else:
- print "This feature is currently not supported. :("
- print "Use -l to run the test locally."
- sys.exit(0)
diff --git a/ooni/ooniprobe.log b/ooni/ooniprobe.log
index d3914dc..700e820 100644
--- a/ooni/ooniprobe.log
+++ b/ooni/ooniprobe.log
@@ -1,2 +1 @@
-2012-05-29 01:32:22,811 ooniprobe INFO Started ooni-probe
-2012-05-29 01:32:22,972 ooniprobe WARNING Soft fail 'module' object has no attribute '__plugoo__'
+2012-05-31 02:27:28,010 ooniprobe INFO Started ooni-probe
diff --git a/ooni/plugins/.bridget.py.swp b/ooni/plugins/.bridget.py.swp
deleted file mode 100644
index 226d826..0000000
Binary files a/ooni/plugins/.bridget.py.swp and /dev/null differ
diff --git a/ooni/plugins/.dnstamper.py.swp b/ooni/plugins/.dnstamper.py.swp
deleted file mode 100644
index 9acb1c9..0000000
Binary files a/ooni/plugins/.dnstamper.py.swp and /dev/null differ
diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py
new file mode 100644
index 0000000..bbcb1a2
--- /dev/null
+++ b/ooni/plugins/blocking.py
@@ -0,0 +1,41 @@
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+
+from ooni.plugoo.assets import Asset
+from ooni.plugoo.tests import ITest, TwistedTest
+
+class BlockingArgs(usage.Options):
+ optParameters = [['asset', 'a', None, 'Asset file'],
+ ['resume', 'r', 0, 'Resume at this index'],
+ ['other', 'o', None, 'Other arguments']]
+
+class BlockingTest(TwistedTest):
+ implements(IPlugin, ITest)
+
+ shortName = "blocking"
+ description = "Blocking plugin"
+ requirements = None
+ options = BlockingArgs
+ # Tells this to be blocking.
+ blocking = True
+
+ def control(self, experiment_result, args):
+ print "Experiment Result:", experiment_result
+ print "Args", args
+ return experiment_result
+
+ def experiment(self, args):
+ import urllib
+ req = urllib.urlopen(args['asset'])
+ return {'page': req.readlines()}
+
+ def load_assets(self):
+ if self.local_options:
+ return {'asset': Asset(self.local_options['asset'])}
+ else:
+ return {}
+
+# We need to instantiate it otherwise getPlugins does not detect it
+# XXX Find a way to load plugins without instantiating them.
+blocking = BlockingTest(None, None, None)
diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py
deleted file mode 100644
index 7271468..0000000
--- a/ooni/plugins/bridget.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from plugoo.tests import ITest, TwistedTest
-from twisted.internet import threads
-
-from tests.bridget import BridgeT as BridgeTlegacy
-from tests.bridget import BridgeTAsset as BridgeTAsset
-from ooniprobe import ooni
-
-o = ooni()
-
-class BridgetArgs(usage.Options):
- optParameters = [['asset', 'a', None, 'Asset file'],
- ['resume', 'r', 0, 'Resume at this index'],
- ['bridge', 'b', None, 'Specify a single bridge']]
-
-class BridgeT(TwistedTest):
- implements(IPlugin, ITest)
-
- shortName = "bridget"
- description = "Bridget plugin"
- requirements = None
- options = BridgetArgs
-
- def experiment(self):
- bridget = BridgeTlegacy(o)
- o.logger.info("Starting bridget test")
- print "ASSET:%s " % self.asset
- d = threads.deferToThread(bridget.connect, self.asset)
- d.addCallback(self.d_experiment.callback, None)
- return d
-
-# We need to instantiate it otherwise getPlugins does not detect it
-# XXX Find a way to load plugins without instantiating them.
-bridget = BridgeT(None, None)
diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py
deleted file mode 100644
index b8b7c7f..0000000
--- a/ooni/plugins/dnstamper.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from plugoo.tests import ITest, TwistedTest
-
-class SkelArgs(usage.Options):
- optParameters = [['hosts', 'f', None, 'Hostname asset file'],
- ['dnsservers', 'd', None, 'DNS server asset file'],
- ['resume', 'r', 0, 'Resume at this index'],
- ['other', 'o', None, 'Other arguments']]
-
-class SkelTest(TwistedTest):
- implements(IPlugin, ITest)
-
- shortName = "skeleton"
- description = "Skeleton plugin"
- requirements = None
- options = SkelArgs
-
-# We need to instantiate it otherwise getPlugins does not detect it
-# XXX Find a way to load plugins without instantiating them.
-skel = SkelTest(None, None)
diff --git a/ooni/plugins/dropin.cache b/ooni/plugins/dropin.cache
index 9258b18..bd26c09 100755
--- a/ooni/plugins/dropin.cache
+++ b/ooni/plugins/dropin.cache
@@ -53,7 +53,7 @@ g5
NtRp24
(dp25
g8
-S'plugins.bridget'
+S'ooni.plugins.bridget'
p26
sg10
Nsg11
@@ -73,4 +73,92 @@ sg21
S'bridget'
p31
sg10
+NsbasbsS'blocking'
+p32
+g3
+(g4
+g5
+NtRp33
+(dp34
+g8
+S'ooni.plugins.blocking'
+p35
+sg10
+Nsg11
+(lp36
+g3
+(g13
+g5
+NtRp37
+(dp38
+g16
+(lp39
+g18
+acooni.plugoo.tests
+ITest
+p40
+asg20
+g33
+sg21
+S'blocking'
+p41
+sg10
+NsbasbsS'httphost'
+p42
+g3
+(g4
+g5
+NtRp43
+(dp44
+g8
+S'ooni.plugins.httphost'
+p45
+sg10
+S'\n HTTP Host based filtering\n *************************\n\n This test detect HTTP Host field\n based filtering.\n'
+p46
+sg11
+(lp47
+g3
+(g13
+g5
+NtRp48
+(dp49
+g16
+(lp50
+g18
+ag40
+asg20
+g43
+sg21
+S'httphost'
+p51
+sg10
+NsbasbsS'tcpscan'
+p52
+g3
+(g4
+g5
+NtRp53
+(dp54
+g8
+S'plugins.tcpscan'
+p55
+sg10
+Nsg11
+(lp56
+g3
+(g13
+g5
+NtRp57
+(dp58
+g16
+(lp59
+g18
+ag40
+asg20
+g53
+sg21
+S'tcpscan'
+p60
+sg10
Nsbasbs.
\ No newline at end of file
diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py
index 93bc1bb..00e4311 100644
--- a/ooni/plugins/skel.py
+++ b/ooni/plugins/skel.py
@@ -15,7 +15,14 @@ class SkelTest(TwistedTest):
description = "Skeleton plugin"
requirements = None
options = SkelArgs
+ blocking = False
+
+ def load_assets(self):
+ if self.local_options:
+ return {'asset': open(self.local_options['asset'])}
+ else:
+ return {}
# We need to instantiate it otherwise getPlugins does not detect it
# XXX Find a way to load plugins without instantiating them.
-skel = SkelTest(None, None)
+skel = SkelTest(None, None, None)
diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py
index 09363a2..75d0765 100644
--- a/ooni/plugoo/tests.py
+++ b/ooni/plugoo/tests.py
@@ -6,9 +6,13 @@ from zope.interface import Interface, Attribute
import logging
import itertools
import gevent
-from twisted.internet import reactor, defer
+
+from twisted.internet import reactor, defer, threads
from twisted.python import failure
-from plugoo.reports import Report
+
+from ooni.plugoo import assets, work
+from ooni.plugoo.reports import Report
+
class Test:
"""
@@ -128,102 +132,64 @@ class ITest(Interface):
#node = Attribute("""This represents the node that will run the test""")
options = Attribute("""These are the arguments to be passed to the test for it's execution""")
- def startTest():
+ blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""")
+
+ def startTest(asset):
"""
Launches the Test with the specified arguments on a node.
"""
-#class HTTPRequestTest(HTTPClient):
-class HTTPRequestTest(object):
- """
- This is an example of how I would like to be able to write a test.
-
- *BEWARE* this actually does not currently work, it's just an example of the
- kind of API that I am attempting to achieve to simplify the writing of
- tests.
+class TwistedTest(object):
+ blocking = False
- implements(ITest)
+ def __init__(self, local_options, global_options, ooninet=None):
+ self.local_options = local_options
+ self.global_options = global_options
+ self.assets = self.load_assets()
+ #self.ooninet = ooninet
- """
- def startTest():
- # The response object should also contain the request
+ def load_assets(self):
"""
- response = {'response': {'headers': ..., 'content': ...,
- 'runtime': ..., 'timestamp': ...},
- 'request': {'headers': ..., 'content', 'timestamp', ...}
- }
- response = self.http_request(address, headers)
- if response.headers['content'].matches("Some string"):
- self.censorship = True
- return response
- else:
- self.censorship = False
- return response
-
+ This method should be overriden by the test writer to provide the logic
+ for loading their assets.
"""
- pass
-
-class TwistedTest(object):
- def __init__(self, asset, arguments, ooninet=None):
- self.asset = asset
- self.arguments = arguments
- self.start_time = datetime.now()
- self._parse_arguments()
- #self.ooninet = ooninet
+ return {'asset': None}
def __repr__(self):
- return "<TwistedTest %s %s>" % (self.arguments, self.asset)
+ return "<TwistedTest %s %s %s>" % (self.options, self.global_options,
+ self.assets)
- def _parse_arguments(self):
- print self.arguments
- if self.arguments and 'test' in self.arguments:
- self.test = self.arguments['test']
-
- def finished(self, result):
+ def finished(self, control):
#self.ooninet.report(result)
- print "FINIHSED"
self.end_time = datetime.now()
+ result = {}
result['start_time'] = self.start_time
result['end_time'] = self.end_time
result['run_time'] = self.end_time - self.start_time
+ result['control'] = control
+ print "FINISHED", result
+ return result
- def _do_experiment(self):
- self.d = defer.maybeDeferred(self.experiment)
- self.d.addCallback(self.control)
+ def _do_experiment(self, args):
+ if self.blocking:
+ self.d = threads.deferToThread(self.experiment, args)
+ else:
+ self.d = defer.maybeDeferred(self.experiment, args)
+
+ self.d.addCallback(self.control, args)
self.d.addCallback(self.finished)
return self.d
- def control(self, exp):
+ def control(self, result, args):
print "Doing control..."
- self.d.callback(result)
+ return result
- def experiment(self):
+ def experiment(self, args):
print "Doing experiment"
- self.d_experiment.callback(None)
-
- def startTest(self):
- print "Starting test %s" % repr(self)
- return self._do_experiment()
-
-class TwistedTestFactory(object):
+ return {}
- test = None
-
- def __init__(self, assets, node,
- idx=0):
- """
- """
- self.assets = assets
- self.node = node
- self.idx = idx
- self.workunit = WorkUnitFactory(assets)
-
- def build_test(self):
- """
- Returns a TwistedTest instance
- """
- workunit = self.workunit.next()
- t = self.test(node, workunit)
- t.factory = self
- return t
+ def startTest(self, args):
+ self.start_time = datetime.now()
+ print "Starting test %s" % self.__class__
+ return self._do_experiment(args)
diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py
index 7be4758..55bbd1d 100644
--- a/ooni/plugoo/work.py
+++ b/ooni/plugoo/work.py
@@ -10,15 +10,16 @@
:license: see LICENSE for more details.
"""
-from datetime import datetime
+import itertools
import yaml
+from datetime import datetime
from zope.interface import Interface, Attribute
from twisted.python import failure
from twisted.internet import reactor, defer
-from plugoo.nodes import LocalNode
+from ooni.plugoo.nodes import LocalNode
class Worker(object):
"""
@@ -34,9 +35,9 @@ class Worker(object):
self._running -= 1
if self._running < self.maxconcurrent and self._queued:
workunit, d = self._queued.pop(0)
- for work in workunit:
- self._running += 1
- actuald = work.startTest().addBoth(self._run)
+ asset, test, idx = workunit
+ self._running += 1
+ actuald = test.startTest(asset).addBoth(self._run)
if isinstance(r, failure.Failure):
r.trap()
@@ -44,14 +45,13 @@ class Worker(object):
print r['start_time']
print r['end_time']
print r['run_time']
- print repr(r)
return r
def push(self, workunit):
if self._running < self.maxconcurrent:
- for work in workunit:
- self._running += 1
- work.startTest().addBoth(self._run)
+ asset, test, idx = workunit
+ self._running += 1
+ test.startTest(asset).addBoth(self._run)
return
d = defer.Deferred()
self._queued.append((workunit, d))
@@ -59,6 +59,8 @@ class Worker(object):
class WorkUnit(object):
"""
+ XXX This is currently not implemented for KISS sake.
+
This is an object responsible for completing WorkUnits it will
return its result in a deferred.
@@ -77,21 +79,21 @@ class WorkUnit(object):
test = None
arguments = None
- def __init__(self, asset, test, idx, arguments=None):
+ def __init__(self, asset, assetNames, test, idx):
self.asset = asset
if not asset:
self.assetGenerator = iter([1])
else:
self.assetGenerator = iter(asset)
self.Test = test
- self.arguments = arguments
+ self.assetNames = assetNames
self.idx = idx
def __iter__(self):
return self
def __repr__(self):
- return "<WorkUnit %s %s %s>" % (self.arguments, self.Test, self.idx)
+ return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx)
def serialize(self):
"""
@@ -105,7 +107,7 @@ class WorkUnit(object):
"""
try:
asset = self.assetGenerator.next()
- ret = self.Test(asset, self.arguments)
+ ret = self.Test.set_asset(asset)
return ret
except StopIteration:
raise StopIteration
@@ -120,10 +122,12 @@ class WorkGenerator(object):
"""
size = 10
- def __init__(self, asset, test, arguments=None, start=None):
- self.assetGenerator = asset
+ def __init__(self, test, arguments=None, start=None):
self.Test = test
- self.arguments = arguments
+
+ self.assetGenerator = itertools.product(*self.Test.assets.values())
+ self.assetNames = self.Test.assets.keys()
+
self.idx = 0
self.end = False
if start:
@@ -139,12 +143,32 @@ class WorkGenerator(object):
self.idx += 1
def next(self):
+ if not self.assetGenerator:
+ self.end = True
+ return (self.assetNames, self.Test, self.idx)
+
+ try:
+ asset = self.assetGenerator.next()
+ ret = {}
+ for i, v in enumerate(asset):
+ ret[self.assetNames[i]] = v
+ except StopIteration:
+ raise StopIteration
+
+ self.idx += 1
+ print "IDX:%s" % self.idx
+ return (ret, self.Test, self.idx)
+
+ def p_next(self):
+ """
+ XXX This is not used for KISS sake.
+ """
if self.end:
raise StopIteration
if not self.assetGenerator:
self.end = True
- return WorkUnit(None, self.Test, self.idx, self.arguments)
+ return WorkUnit(None, self.assetNames, self.Test, self.idx)
# Plank asset
p_asset = []
@@ -152,11 +176,13 @@ class WorkGenerator(object):
try:
asset = self.assetGenerator.next()
p_asset.append(asset)
- print asset
+ print p_asset
except StopIteration:
+ if self.asset_num > 1:
+ pass
self.end = True
break
self.idx += 1
- return WorkUnit(p_asset, self.Test, self.idx, self.arguments)
+ return WorkUnit(p_asset, self.assetNames, self.Test, self.idx)
diff --git a/ooni/skel.py b/ooni/skel.py
deleted file mode 100644
index 8982106..0000000
--- a/ooni/skel.py
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8
-"""
- XXX
- ***
-
- :copyright: (c) 2012 by Arturo Filastò
- :license: see LICENSE for more details.
-"""
-
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 774a9c5..fc5bbdb 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -16,19 +16,19 @@ from twisted.web import resource, server, static
from twisted.web.microdom import escape
from twisted.names import dns
-from backend.httpbackend import HTTPBackend
-from backend.dnsbackend import ProxyDNSServer
+from oonib.httpbackend import HTTPBackend
+from oonib.dnsbackend import ProxyDNSServer
# This tells twisted to set the
server.version = "Apache"
application = service.Application('oonibackend')
serviceCollection = service.IServiceCollection(application)
-internet.TCPServer(8000, server.Site(HTTPBackend())).setServiceParent(serviceCollection)
+internet.TCPServer(2000, server.Site(HTTPBackend())).setServiceParent(serviceCollection)
# Start the DNS Server related services
TCPDNSServer = ProxyDNSServer()
internet.TCPServer(8002, TCPDNSServer).setServiceParent(serviceCollection)
UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer)
-internet.UDPServer(5353, UDPFactory).setServiceParent(serviceCollection)
+internet.UDPServer(5354, UDPFactory).setServiceParent(serviceCollection)
More information about the tor-commits
mailing list