commit c4c91d393ba3956b84d46521e2f58d30584b3ea0 Author: Arturo Filastò hellais@torproject.org Date: Thu May 31 03:31:05 2012 +0200

Get the new OONI architecture working properly. (kudos to the organizers of FreeBird :) ) --- .gitignore | 3 +- ooni/oonicli.py | 70 ++++++++---------------- ooni/ooniprobe.log | 3 +- ooni/plugins/.bridget.py.swp | Bin 12288 -> 0 bytes ooni/plugins/.dnstamper.py.swp | Bin 12288 -> 0 bytes ooni/plugins/blocking.py | 41 ++++++++++++++ ooni/plugins/bridget.py | 36 ------------ ooni/plugins/dnstamper.py | 22 -------- ooni/plugins/dropin.cache | 90 ++++++++++++++++++++++++++++++- ooni/plugins/skel.py | 9 +++- ooni/plugoo/tests.py | 118 ++++++++++++++-------------------------- ooni/plugoo/work.py | 64 +++++++++++++++------- ooni/skel.py | 10 ---- oonib/oonibackend.py | 8 ++-- 14 files changed, 255 insertions(+), 219 deletions(-)

diff --git a/.gitignore b/.gitignore index 6433d66..8ed47e9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ proxy-lists/ip-cc.txt proxy-lists/ips.txt proxy-lists/italy-dns-ips.txt proxy-lists/italy-http-ips.txt -private/* \ No newline at end of file +private/* +ooni/plugins/dropin.cache diff --git a/ooni/oonicli.py b/ooni/oonicli.py index d290e68..a0272fb 100755 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -9,7 +9,6 @@ #

import sys -from plugoo import tests, work, assets

from twisted.python import usage from twisted.plugin import getPlugins @@ -19,8 +18,10 @@ from zope.interface.exceptions import BrokenImplementation from zope.interface.exceptions import BrokenMethodImplementation from zope.interface.verify import verifyObject from pprint import pprint -from logo import getlogo -import plugins + +from ooni.plugoo import tests, work, assets +from ooni.logo import getlogo +from ooni import plugins

__version__ = "0.0.1-prealpha"

@@ -46,33 +47,20 @@ def retrieve_plugoo():

plugoo = retrieve_plugoo()

-class StupidAsset(object): - def __init__(self): - self.idx = 0 - - def __iter__(self): - return self - - def next(self): - if self.idx > 30: - raise StopIteration - self.idx += 1 - return self.idx - - def runTest(test, options, global_options): - asset = None - if options['asset']: - print options['asset'] - asset = assets.Asset(options['asset']) - print asset

- wgen = work.WorkGenerator(asset, plugoo[test].__class__, - dict(options), start=options['resume']) + parallelism = int(global_options['parallelism']) + worker = work.Worker(parallelism) + test = plugoo[test].__class__ + + #if options['asset']: + # print options['asset'] + # asset = assets.Asset(options['asset']) + # print asset

- if global_options['parallelism']: - wgen.size = int(global_options['parallelism']) - worker = work.Worker(wgen.size) + wgen = work.WorkGenerator(test(options, global_options), + dict(options), + start=options['resume'])

for x in wgen: worker.push(x) @@ -86,16 +74,17 @@ class Options(usage.Options): subCommands.append([test, None, plugoo[test].options, "Run the %s test" % test])

optFlags = [ - ['local', 'l', "If the test should be run locally (without having oonid running)"], - ['status', 'x', 'Show current state'], - ['restart', 'r', 'Restart OONI'] + #['remote', 'r', "If the test should be run remotely (not supported)"], + #['status', 'x', 'Show current state'], + #['restart', 'r', 'Restart OONI'] ]

optParameters = [ ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], - ['target-node', 't', 'localhost:31415', 'Select target node'], - ['ooninet', 'o', 'localhost:4242', "Select OONI-net address for reporting"], - ['password', 'p', 'opennetwork', "Specify the password for authentication"], + #['target-node', 't', 'localhost:31415', 'Select target node'], + ['output', 'o', 'report.log', "Specify output report file"], + ['log', 'l', 'oonicli.log', "Specify output log file"], + #['password', 'p', 'opennetwork', "Specify the password for authentication"], ]

def opt_version(self): @@ -116,24 +105,11 @@ class Options(usage.Options): config = Options() config.parseOptions()

-if config['status']: - print "oonid is not running." - sys.exit(0) - -if config['restart']: - print "Restarting oonid." - sys.exit(0) - if not config.subCommand: print "Error! No Test Specified." config.opt_help() sys.exit(1)

-if config['local']: - runTest(config.subCommand, config.subOptions, config) +runTest(config.subCommand, config.subOptions, config)

-else: - print "This feature is currently not supported. :(" - print "Use -l to run the test locally." - sys.exit(0)

diff --git a/ooni/ooniprobe.log b/ooni/ooniprobe.log index d3914dc..700e820 100644 --- a/ooni/ooniprobe.log +++ b/ooni/ooniprobe.log @@ -1,2 +1 @@ -2012-05-29 01:32:22,811 ooniprobe INFO Started ooni-probe -2012-05-29 01:32:22,972 ooniprobe WARNING Soft fail 'module' object has no attribute '__plugoo__' +2012-05-31 02:27:28,010 ooniprobe INFO Started ooni-probe diff --git a/ooni/plugins/.bridget.py.swp b/ooni/plugins/.bridget.py.swp deleted file mode 100644 index 226d826..0000000 Binary files a/ooni/plugins/.bridget.py.swp and /dev/null differ diff --git a/ooni/plugins/.dnstamper.py.swp b/ooni/plugins/.dnstamper.py.swp deleted file mode 100644 index 9acb1c9..0000000 Binary files a/ooni/plugins/.dnstamper.py.swp and /dev/null differ diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py new file mode 100644 index 0000000..bbcb1a2 --- /dev/null +++ b/ooni/plugins/blocking.py @@ -0,0 +1,41 @@ +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, TwistedTest + +class BlockingArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index'], + ['other', 'o', None, 'Other arguments']] + +class BlockingTest(TwistedTest): + implements(IPlugin, ITest) + + shortName = "blocking" + description = "Blocking plugin" + requirements = None + options = BlockingArgs + # Tells this to be blocking. + blocking = True + + def control(self, experiment_result, args): + print "Experiment Result:", experiment_result + print "Args", args + return experiment_result + + def experiment(self, args): + import urllib + req = urllib.urlopen(args['asset']) + return {'page': req.readlines()} + + def load_assets(self): + if self.local_options: + return {'asset': Asset(self.local_options['asset'])} + else: + return {} + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +blocking = BlockingTest(None, None, None) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py deleted file mode 100644 index 7271468..0000000 --- a/ooni/plugins/bridget.py +++ /dev/null @@ -1,36 +0,0 @@ -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from plugoo.tests import ITest, TwistedTest -from twisted.internet import threads - -from tests.bridget import BridgeT as BridgeTlegacy -from tests.bridget import BridgeTAsset as BridgeTAsset -from ooniprobe import ooni - -o = ooni() - -class BridgetArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['resume', 'r', 0, 'Resume at this index'], - ['bridge', 'b', None, 'Specify a single bridge']] - -class BridgeT(TwistedTest): - implements(IPlugin, ITest) - - shortName = "bridget" - description = "Bridget plugin" - requirements = None - options = BridgetArgs - - def experiment(self): - bridget = BridgeTlegacy(o) - o.logger.info("Starting bridget test") - print "ASSET:%s " % self.asset - d = threads.deferToThread(bridget.connect, self.asset) - d.addCallback(self.d_experiment.callback, None) - return d - -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -bridget = BridgeT(None, None) diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py deleted file mode 100644 index b8b7c7f..0000000 --- a/ooni/plugins/dnstamper.py +++ /dev/null @@ -1,22 +0,0 @@ -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from plugoo.tests import ITest, TwistedTest - -class SkelArgs(usage.Options): - optParameters = [['hosts', 'f', None, 'Hostname asset file'], - ['dnsservers', 'd', None, 'DNS server asset file'], - ['resume', 'r', 0, 'Resume at this index'], - ['other', 'o', None, 'Other arguments']] - -class SkelTest(TwistedTest): - implements(IPlugin, ITest) - - shortName = "skeleton" - description = "Skeleton plugin" - requirements = None - options = SkelArgs - -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -skel = SkelTest(None, None) diff --git a/ooni/plugins/dropin.cache b/ooni/plugins/dropin.cache index 9258b18..bd26c09 100755 --- a/ooni/plugins/dropin.cache +++ b/ooni/plugins/dropin.cache @@ -53,7 +53,7 @@ g5 NtRp24 (dp25 g8 -S'plugins.bridget' +S'ooni.plugins.bridget' p26 sg10 Nsg11 @@ -73,4 +73,92 @@ sg21 S'bridget' p31 sg10 +NsbasbsS'blocking' +p32 +g3 +(g4 +g5 +NtRp33 +(dp34 +g8 +S'ooni.plugins.blocking' +p35 +sg10 +Nsg11 +(lp36 +g3 +(g13 +g5 +NtRp37 +(dp38 +g16 +(lp39 +g18 +acooni.plugoo.tests +ITest +p40 +asg20 +g33 +sg21 +S'blocking' +p41 +sg10 +NsbasbsS'httphost' +p42 +g3 +(g4 +g5 +NtRp43 +(dp44 +g8 +S'ooni.plugins.httphost' +p45 +sg10 +S'\n HTTP Host based filtering\n *************************\n\n This test detect HTTP Host field\n based filtering.\n' +p46 +sg11 +(lp47 +g3 +(g13 +g5 +NtRp48 +(dp49 +g16 +(lp50 +g18 +ag40 +asg20 +g43 +sg21 +S'httphost' +p51 +sg10 +NsbasbsS'tcpscan' +p52 +g3 +(g4 +g5 +NtRp53 +(dp54 +g8 +S'plugins.tcpscan' +p55 +sg10 +Nsg11 +(lp56 +g3 +(g13 +g5 +NtRp57 +(dp58 +g16 +(lp59 +g18 +ag40 +asg20 +g53 +sg21 +S'tcpscan' +p60 +sg10 Nsbasbs. \ No newline at end of file diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py index 93bc1bb..00e4311 100644 --- a/ooni/plugins/skel.py +++ b/ooni/plugins/skel.py @@ -15,7 +15,14 @@ class SkelTest(TwistedTest): description = "Skeleton plugin" requirements = None options = SkelArgs + blocking = False + + def load_assets(self): + if self.local_options: + return {'asset': open(self.local_options['asset'])} + else: + return {}

# We need to instantiate it otherwise getPlugins does not detect it # XXX Find a way to load plugins without instantiating them. -skel = SkelTest(None, None) +skel = SkelTest(None, None, None) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 09363a2..75d0765 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -6,9 +6,13 @@ from zope.interface import Interface, Attribute import logging import itertools import gevent -from twisted.internet import reactor, defer + +from twisted.internet import reactor, defer, threads from twisted.python import failure -from plugoo.reports import Report + +from ooni.plugoo import assets, work +from ooni.plugoo.reports import Report +

class Test: """ @@ -128,102 +132,64 @@ class ITest(Interface): #node = Attribute("""This represents the node that will run the test""") options = Attribute("""These are the arguments to be passed to the test for it's execution""")

- def startTest(): + blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""") + + def startTest(asset): """ Launches the Test with the specified arguments on a node. """

-#class HTTPRequestTest(HTTPClient): -class HTTPRequestTest(object): - """ - This is an example of how I would like to be able to write a test. - - *BEWARE* this actually does not currently work, it's just an example of the - kind of API that I am attempting to achieve to simplify the writing of - tests. +class TwistedTest(object): + blocking = False

- implements(ITest) + def __init__(self, local_options, global_options, ooninet=None): + self.local_options = local_options + self.global_options = global_options + self.assets = self.load_assets() + #self.ooninet = ooninet

- """ - def startTest(): - # The response object should also contain the request + def load_assets(self): """ - response = {'response': {'headers': ..., 'content': ..., - 'runtime': ..., 'timestamp': ...}, - 'request': {'headers': ..., 'content', 'timestamp', ...} - } - response = self.http_request(address, headers) - if response.headers['content'].matches("Some string"): - self.censorship = True - return response - else: - self.censorship = False - return response - + This method should be overriden by the test writer to provide the logic + for loading their assets. """ - pass - -class TwistedTest(object): - def __init__(self, asset, arguments, ooninet=None): - self.asset = asset - self.arguments = arguments - self.start_time = datetime.now() - self._parse_arguments() - #self.ooninet = ooninet + return {'asset': None}

def __repr__(self): - return "<TwistedTest %s %s>" % (self.arguments, self.asset) + return "<TwistedTest %s %s %s>" % (self.options, self.global_options, + self.assets)

- def _parse_arguments(self): - print self.arguments - if self.arguments and 'test' in self.arguments: - self.test = self.arguments['test'] - - def finished(self, result): + def finished(self, control): #self.ooninet.report(result) - print "FINIHSED" self.end_time = datetime.now() + result = {} result['start_time'] = self.start_time result['end_time'] = self.end_time result['run_time'] = self.end_time - self.start_time + result['control'] = control + print "FINISHED", result + return result

- def _do_experiment(self): - self.d = defer.maybeDeferred(self.experiment) - self.d.addCallback(self.control) + def _do_experiment(self, args): + if self.blocking: + self.d = threads.deferToThread(self.experiment, args) + else: + self.d = defer.maybeDeferred(self.experiment, args) + + self.d.addCallback(self.control, args) self.d.addCallback(self.finished) return self.d

- def control(self, exp): + def control(self, result, args): print "Doing control..." - self.d.callback(result) + return result

- def experiment(self): + def experiment(self, args): print "Doing experiment" - self.d_experiment.callback(None) - - def startTest(self): - print "Starting test %s" % repr(self) - return self._do_experiment() - -class TwistedTestFactory(object): + return {}

- test = None - - def __init__(self, assets, node, - idx=0): - """ - """ - self.assets = assets - self.node = node - self.idx = idx - self.workunit = WorkUnitFactory(assets) - - def build_test(self): - """ - Returns a TwistedTest instance - """ - workunit = self.workunit.next() - t = self.test(node, workunit) - t.factory = self - return t + def startTest(self, args): + self.start_time = datetime.now() + print "Starting test %s" % self.__class__ + return self._do_experiment(args)

diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 7be4758..55bbd1d 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -10,15 +10,16 @@ :license: see LICENSE for more details.

""" -from datetime import datetime +import itertools import yaml +from datetime import datetime

from zope.interface import Interface, Attribute

from twisted.python import failure from twisted.internet import reactor, defer

-from plugoo.nodes import LocalNode +from ooni.plugoo.nodes import LocalNode

class Worker(object): """ @@ -34,9 +35,9 @@ class Worker(object): self._running -= 1 if self._running < self.maxconcurrent and self._queued: workunit, d = self._queued.pop(0) - for work in workunit: - self._running += 1 - actuald = work.startTest().addBoth(self._run) + asset, test, idx = workunit + self._running += 1 + actuald = test.startTest(asset).addBoth(self._run) if isinstance(r, failure.Failure): r.trap()

@@ -44,14 +45,13 @@ class Worker(object): print r['start_time'] print r['end_time'] print r['run_time'] - print repr(r) return r

def push(self, workunit): if self._running < self.maxconcurrent: - for work in workunit: - self._running += 1 - work.startTest().addBoth(self._run) + asset, test, idx = workunit + self._running += 1 + test.startTest(asset).addBoth(self._run) return d = defer.Deferred() self._queued.append((workunit, d)) @@ -59,6 +59,8 @@ class Worker(object):

class WorkUnit(object): """ + XXX This is currently not implemented for KISS sake. + This is an object responsible for completing WorkUnits it will return its result in a deferred.

@@ -77,21 +79,21 @@ class WorkUnit(object): test = None arguments = None

- def __init__(self, asset, test, idx, arguments=None): + def __init__(self, asset, assetNames, test, idx): self.asset = asset if not asset: self.assetGenerator = iter([1]) else: self.assetGenerator = iter(asset) self.Test = test - self.arguments = arguments + self.assetNames = assetNames self.idx = idx

def __iter__(self): return self

def __repr__(self): - return "<WorkUnit %s %s %s>" % (self.arguments, self.Test, self.idx) + return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx)

def serialize(self): """ @@ -105,7 +107,7 @@ class WorkUnit(object): """ try: asset = self.assetGenerator.next() - ret = self.Test(asset, self.arguments) + ret = self.Test.set_asset(asset) return ret except StopIteration: raise StopIteration @@ -120,10 +122,12 @@ class WorkGenerator(object): """ size = 10

- def __init__(self, asset, test, arguments=None, start=None): - self.assetGenerator = asset + def __init__(self, test, arguments=None, start=None): self.Test = test - self.arguments = arguments + + self.assetGenerator = itertools.product(*self.Test.assets.values()) + self.assetNames = self.Test.assets.keys() + self.idx = 0 self.end = False if start: @@ -139,12 +143,32 @@ class WorkGenerator(object): self.idx += 1

def next(self): + if not self.assetGenerator: + self.end = True + return (self.assetNames, self.Test, self.idx) + + try: + asset = self.assetGenerator.next() + ret = {} + for i, v in enumerate(asset): + ret[self.assetNames[i]] = v + except StopIteration: + raise StopIteration + + self.idx += 1 + print "IDX:%s" % self.idx + return (ret, self.Test, self.idx) + + def p_next(self): + """ + XXX This is not used for KISS sake. + """ if self.end: raise StopIteration

if not self.assetGenerator: self.end = True - return WorkUnit(None, self.Test, self.idx, self.arguments) + return WorkUnit(None, self.assetNames, self.Test, self.idx)

# Plank asset p_asset = [] @@ -152,11 +176,13 @@ class WorkGenerator(object): try: asset = self.assetGenerator.next() p_asset.append(asset) - print asset + print p_asset except StopIteration: + if self.asset_num > 1: + pass self.end = True break

self.idx += 1 - return WorkUnit(p_asset, self.Test, self.idx, self.arguments) + return WorkUnit(p_asset, self.assetNames, self.Test, self.idx)

diff --git a/ooni/skel.py b/ooni/skel.py deleted file mode 100644 index 8982106..0000000 --- a/ooni/skel.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -""" - XXX - *** - - :copyright: (c) 2012 by Arturo Filastò - :license: see LICENSE for more details. -""" - diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 774a9c5..fc5bbdb 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -16,19 +16,19 @@ from twisted.web import resource, server, static from twisted.web.microdom import escape from twisted.names import dns

-from backend.httpbackend import HTTPBackend -from backend.dnsbackend import ProxyDNSServer +from oonib.httpbackend import HTTPBackend +from oonib.dnsbackend import ProxyDNSServer

# This tells twisted to set the server.version = "Apache"

application = service.Application('oonibackend') serviceCollection = service.IServiceCollection(application) -internet.TCPServer(8000, server.Site(HTTPBackend())).setServiceParent(serviceCollection) +internet.TCPServer(2000, server.Site(HTTPBackend())).setServiceParent(serviceCollection)

# Start the DNS Server related services TCPDNSServer = ProxyDNSServer() internet.TCPServer(8002, TCPDNSServer).setServiceParent(serviceCollection) UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer) -internet.UDPServer(5353, UDPFactory).setServiceParent(serviceCollection) +internet.UDPServer(5354, UDPFactory).setServiceParent(serviceCollection)