commit d68e5f4f8edffbcd7e9c9fc5ce22030472f401a1 Author: Arturo Filastò <art@torproject.org> Date: Mon Jul 9 11:54:37 2012 +0200 Do a big cleanup * Better specify test writing interface * Move example plugins to appropriate directory * Implement RFC3339 timestamp, as spec * Make sure all the defaults tests run and output something even without arguments * Fix bug in worker --- ooni/date.py | 17 ++++- ooni/example_plugins/examplescapy.py | 49 ++++++++++++++ ooni/example_plugins/skel.py | 29 ++++++++ ooni/lib/Makefile | 8 ++- ooni/ooniprobe.py | 1 - ooni/plugins/blocking.py | 6 +- ooni/plugins/chinatrigger.py | 2 +- ooni/plugins/examplescapy.py | 49 -------------- ooni/plugins/httphost.py | 11 ++-- ooni/plugins/httpt.py | 5 +- ooni/plugins/skel.py | 29 -------- ooni/plugins/tcpconnect.py | 9 ++- ooni/plugoo/interface.py | 31 ++++++++- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 29 +++++--- ooni/plugoo/work.py | 118 ++++++++++++++++------------------ ooni/protocols/http.py | 5 +- 17 files changed, 223 insertions(+), 177 deletions(-) diff --git a/ooni/date.py b/ooni/date.py index 6f83191..59ec1f8 100644 --- a/ooni/date.py +++ b/ooni/date.py @@ -1,11 +1,22 @@ +from ooni.lib.rfc3339 import rfc3339 from datetime import datetime +class odate(datetime): + def __str__(self): + return rfc3339(self) + + def __repr__(self): + return "'%s'" % rfc3339(self) + + def from_rfc(self, datestr): + pass + +def now(): + return odate.utcnow() + def pretty_date(): cur_time = datetime.utcnow() d_format = "%d %B %Y %H:%M:%S" pretty = cur_time.strftime(d_format) return pretty -def now(): - return datetime.utcnow() - diff --git a/ooni/example_plugins/examplescapy.py b/ooni/example_plugins/examplescapy.py new file mode 100644 index 0000000..aa6d81b --- /dev/null +++ b/ooni/example_plugins/examplescapy.py @@ -0,0 +1,49 @@ +import random +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset +from ooni import log +from ooni.protocols.scapy import ScapyTest + +from ooni.lib.txscapy import txsr, txsend + +class scapyArgs(usage.Options): + optParameters = [] + +class ExampleScapyTest(ScapyTest): + """ + An example of writing a scapy Test + """ + implements(IPlugin, ITest) + + shortName = "example_scapy" + description = "An example of a scapy test" + requirements = None + options = scapyArgs + blocking = False + + receive = True + pcapfile = 'example_scapy.pcap' + def initialize(self, reactor=None): + if not self.reactor: + from twisted.internet import reactor + self.reactor = reactor + + self.request = {} + self.response = {} + + def build_packets(self): + """ + Override this method to build scapy packets. + """ + from scapy.all import IP, TCP + return IP()/TCP() + + def load_assets(self): + return {} + +examplescapy = ExampleScapyTest(None, None, None) + diff --git a/ooni/example_plugins/skel.py b/ooni/example_plugins/skel.py new file mode 100644 index 0000000..de95b48 --- /dev/null +++ b/ooni/example_plugins/skel.py @@ -0,0 +1,29 @@ +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from plugoo.tests import ITest, OONITest +from ooni import log + +class SkelArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index'], + ['other', 'o', None, 'Other arguments']] + +class SkelTest(OONITest): + implements(IPlugin, ITest) + + shortName = "skeleton" + description = "Skeleton plugin" + requirements = None + options = SkelArgs + blocking = False + + def load_assets(self): + if self.local_options: + return {'asset': open(self.local_options['asset'])} + else: + return {} + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +skel = SkelTest(None, None, None) diff --git a/ooni/lib/Makefile b/ooni/lib/Makefile index 37abe05..a498a35 100644 --- a/ooni/lib/Makefile +++ b/ooni/lib/Makefile @@ -1,4 +1,4 @@ -all: txtorcon txtraceroute txscapy +all: txtorcon txtraceroute txscapy rfc3339 txtraceroute: echo "Processing dependency txtraceroute..." @@ -18,3 +18,9 @@ txscapy: mv txscapy.git/txscapy.py txscapy.py rm -rf txscapy.git +rfc3339: + echo "Processing RFC3339 dependency" + hg clone https://bitbucket.org/henry/rfc3339 rfc3339 + mv rfc3339/rfc3339.py rfc3339.py + rf -rf rfc3339 + diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 662956a..3eb70f6 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -71,7 +71,6 @@ def runTest(test, options, global_options): reactor=reactor), dict(options), start=resume) - for x in wgen: worker.push(x) diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py index b6b63b6..6ac3c91 100644 --- a/ooni/plugins/blocking.py +++ b/ooni/plugins/blocking.py @@ -27,11 +27,13 @@ class BlockingTest(OONITest): def experiment(self, args): import urllib - req = urllib.urlopen(args['asset']) + url = 'https://torproject.org/' if not 'asset' in args else args['asset'] + req = urllib.urlopen(url) + return {'page': req.readlines()} def load_assets(self): - if self.local_options: + if self.local_options and self.local_options['asset']: return {'asset': Asset(self.local_options['asset'])} else: return {} diff --git a/ooni/plugins/chinatrigger.py b/ooni/plugins/chinatrigger.py index 8a2418a..4f2dc8c 100644 --- a/ooni/plugins/chinatrigger.py +++ b/ooni/plugins/chinatrigger.py @@ -32,7 +32,7 @@ class ChinaTriggerTest(ScapyTest): shortName = "chinatrigger" description = "Triggers the chinese probes into scanning" - requirements = None + requirements = ['root'] options = scapyArgs blocking = False diff --git a/ooni/plugins/examplescapy.py b/ooni/plugins/examplescapy.py deleted file mode 100644 index aa6d81b..0000000 --- a/ooni/plugins/examplescapy.py +++ /dev/null @@ -1,49 +0,0 @@ -import random -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import protocol, defer -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset -from ooni import log -from ooni.protocols.scapy import ScapyTest - -from ooni.lib.txscapy import txsr, txsend - -class scapyArgs(usage.Options): - optParameters = [] - -class ExampleScapyTest(ScapyTest): - """ - An example of writing a scapy Test - """ - implements(IPlugin, ITest) - - shortName = "example_scapy" - description = "An example of a scapy test" - requirements = None - options = scapyArgs - blocking = False - - receive = True - pcapfile = 'example_scapy.pcap' - def initialize(self, reactor=None): - if not self.reactor: - from twisted.internet import reactor - self.reactor = reactor - - self.request = {} - self.response = {} - - def build_packets(self): - """ - Override this method to build scapy packets. - """ - from scapy.all import IP, TCP - return IP()/TCP() - - def load_assets(self): - return {} - -examplescapy = ExampleScapyTest(None, None, None) - diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py index 45ab587..ccadff9 100644 --- a/ooni/plugins/httphost.py +++ b/ooni/plugins/httphost.py @@ -23,7 +23,7 @@ from ooni.plugoo.tests import ITest, OONITest class HTTPHostArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], - ['controlserver', 'c', None, 'Specify the control server'], + ['controlserver', 'c', 'google.com', 'Specify the control server'], ['resume', 'r', 0, 'Resume at this index'], ['other', 'o', None, 'Other arguments']] def control(self, experiment_result, args): @@ -61,7 +61,7 @@ class HTTPHostTest(OONITest): def check_response(self, response): soup = BeautifulSoup(response) - if soup.head.title.string == "WikiLeaks": + if soup.head.title.string == "Blocked": # Response indicates censorship return True else: @@ -107,13 +107,14 @@ class HTTPHostTest(OONITest): def experiment(self, args): control_server = self.local_options['controlserver'] - censored = self.httplib_test(control_server, args['asset']) + url = 'http://torproject.org/' if not 'asset' in args else args['asset'] + censored = self.httplib_test(control_server, url) return {'control': control_server, - 'host': args['asset'], + 'host': url, 'censored': censored} def load_assets(self): - if self.local_options: + if self.local_options and self.local_options['asset']: return {'asset': Asset(self.local_options['asset'])} else: return {} diff --git a/ooni/plugins/httpt.py b/ooni/plugins/httpt.py index 3f92019..d0ede0f 100644 --- a/ooni/plugins/httpt.py +++ b/ooni/plugins/httpt.py @@ -12,7 +12,8 @@ from ooni.protocols import http from ooni import log class httptArgs(usage.Options): - optParameters = [['urls', 'u', None, 'Urls file'], + optParameters = [['urls', 'f', None, 'Urls file'], + ['url', 'u', 'http://torproject.org/', 'Test single site'], ['resume', 'r', 0, 'Resume at this index']] class httptTest(http.HTTPTest): @@ -30,7 +31,7 @@ class httptTest(http.HTTPTest): return {} def load_assets(self): - if self.local_options: + if self.local_options and self.local_options['urls']: return {'url': Asset(self.local_options['urls'])} else: return {} diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py deleted file mode 100644 index de95b48..0000000 --- a/ooni/plugins/skel.py +++ /dev/null @@ -1,29 +0,0 @@ -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from plugoo.tests import ITest, OONITest -from ooni import log - -class SkelArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['resume', 'r', 0, 'Resume at this index'], - ['other', 'o', None, 'Other arguments']] - -class SkelTest(OONITest): - implements(IPlugin, ITest) - - shortName = "skeleton" - description = "Skeleton plugin" - requirements = None - options = SkelArgs - blocking = False - - def load_assets(self): - if self.local_options: - return {'asset': open(self.local_options['asset'])} - else: - return {} - -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -skel = SkelTest(None, None, None) diff --git a/ooni/plugins/tcpconnect.py b/ooni/plugins/tcpconnect.py index 75f32c1..27df08d 100644 --- a/ooni/plugins/tcpconnect.py +++ b/ooni/plugins/tcpconnect.py @@ -15,7 +15,7 @@ from ooni.plugoo.assets import Asset from ooni import log class tcpconnectArgs(usage.Options): - optParameters = [['blabla', 'a', None, 'Asset file'], + optParameters = [['asset', 'a', None, 'File containing IP:PORT combinations, one per line.'], ['resume', 'r', 0, 'Resume at this index']] class tcpconnectTest(OONITest): @@ -28,7 +28,10 @@ class tcpconnectTest(OONITest): blocking = False def experiment(self, args): - host, port = args['blabla'].split(':') + try: + host, port = args['asset'].split(':') + except: + raise Exception("Error in parsing asset. Wrong format?") class DummyFactory(Factory): def buildProtocol(self, addr): return Protocol() @@ -53,7 +56,7 @@ class tcpconnectTest(OONITest): def load_assets(self): if self.local_options: - return {'blabla': Asset(self.local_options['blabla'])} + return {'asset': Asset(self.local_options['asset'])} else: return {} diff --git a/ooni/plugoo/interface.py b/ooni/plugoo/interface.py index 68cff1a..c3b3855 100644 --- a/ooni/plugoo/interface.py +++ b/ooni/plugoo/interface.py @@ -10,14 +10,37 @@ class ITest(Interface): requirements = Attribute("""What is required to run this this test, for example raw socket access or UDP or TCP""") - #deferred = Attribute("""This will be fired on test completion""") - #node = Attribute("""This represents the node that will run the test""") options = Attribute("""These are the arguments to be passed to the test for it's execution""") blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""") - def startTest(asset): + def control(experiment_result, args): """ - Launches the Test with the specified arguments on a node. + @param experiment_result: The result returned by the experiment method. + + @param args: the keys of this dict are the names of the assets passed in + from load_assets. The value is one item of the asset. + + Must return a dict containing what should be written to the report. + Anything returned by control ends up inside of the YAMLOONI report. + """ + + def experiment(args): + """ + Perform all the operations that are necessary to running a test. + + @param args: the keys of this dict are the names of the assets passed in + from load_assets. The value is one item of the asset. + + Must return a dict containing the values to be passed to control. + """ + + def load_assets(): + """ + Load the assets that should be passed to the Test. These are the inputs + to the OONI test. + Must return a dict that has as keys the asset names and values the + asset contents. + If the test does not have any assets it should return an empty dict. """ diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index 6140294..63bc3f5 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -44,7 +44,7 @@ class Report: header += "# %s\n\n" % pretty_date self._write_to_report(header) # XXX replace this with something proper - test_details = {'start_time': date.now(), + test_details = {'start_time': str(date.now()), 'asn': 'ASN-1234', 'test_name': self.testname, 'addr': '1234'} diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 608cee3..a3e9150 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -1,16 +1,14 @@ import os -from datetime import datetime import yaml from zope.interface import Interface, Attribute import logging import itertools -import gevent - from twisted.internet import reactor, defer, threads from twisted.python import failure from ooni import log +from ooni import date from ooni.plugoo import assets, work from ooni.plugoo.reports import Report from ooni.plugoo.interface import ITest @@ -38,7 +36,7 @@ class OONITest(object): This method should be overriden by the test writer to provide the logic for loading their assets. """ - return {'asset': None} + return {} def __repr__(self): return "<OONITest %s %s %s>" % (self.options, self.global_options, @@ -46,11 +44,11 @@ class OONITest(object): def finished(self, control): #self.ooninet.report(result) - self.end_time = datetime.now() + self.end_time = date.now() result = self.result - result['start_time'] = self.start_time - result['end_time'] = self.end_time - result['run_time'] = self.end_time - self.start_time + result['start_time'] = str(self.start_time) + result['end_time'] = str(self.end_time) + result['run_time'] = str(self.end_time - self.start_time) result['control'] = control log.msg("FINISHED %s" % result) self.report(result) @@ -68,14 +66,23 @@ class OONITest(object): def control(self, result, args): log.msg("Doing control") - return result + + if self.blocking: + return result + + def end(cb): + return result + d = defer.Deferred() + d.addCallback(end) + return d def experiment(self, args): log.msg("Doing experiment") - return {} + d = defer.Deferred() + return d def startTest(self, args): - self.start_time = datetime.now() + self.start_time = date.now() log.msg("Starting test %s" % self.__class__) return self._do_experiment(args) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 7e9be0c..a3fb168 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -40,12 +40,7 @@ class Worker(object): if isinstance(r, failure.Failure): r.trap() - print r['start_time'] - print r['end_time'] - print r['run_time'] - if self._running == 0 and not self._queued: - print "I am done." reactor.stop() return r @@ -60,62 +55,6 @@ class Worker(object): self._queued.append((workunit, d)) return d -class WorkUnit(object): - """ - XXX This is currently not implemented for KISS sake. - - This is an object responsible for completing WorkUnits it will - return its result in a deferred. - - The execution of a unit of work should be Atomic. - - Reporting to the OONI-net happens on completion of a Unit of Work. - - @Node node: This represents the node associated with the Work Unit - @Asset asset: This is the asset associated with the Work Unit - @Test test: This represents the Test to be with the specified assets - @ivar arguments: These are the extra attributes to be passsed to the Test - """ - - node = None - asset = None - test = None - arguments = None - - def __init__(self, asset, assetNames, test, idx): - self.asset = asset - if not asset: - self.assetGenerator = iter([1]) - else: - self.assetGenerator = iter(asset) - self.Test = test - self.assetNames = assetNames - self.idx = idx - - def __iter__(self): - return self - - def __repr__(self): - return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx) - - def serialize(self): - """ - Serialize this unit of work for RPC activity. - """ - return yaml.dump(self) - - def next(self): - """ - Launches the Unit of Work with the specified assets on the node. - """ - try: - asset = self.assetGenerator.next() - ret = self.Test.set_asset(asset) - return ret - except StopIteration: - raise StopIteration - - class WorkGenerator(object): """ Factory responsible for creating units of work. @@ -129,7 +68,6 @@ class WorkGenerator(object): self.Test = test if self.Test.assets and self.Test.assets.values()[0]: - print self.Test.assets self.assetGenerator = itertools.product(*self.Test.assets.values()) else: self.assetGenerator = None @@ -156,7 +94,7 @@ class WorkGenerator(object): if not self.assetGenerator: self.end = True - return (self.assetNames, self.Test, self.idx) + return ({}, self.Test, self.idx) try: asset = self.assetGenerator.next() @@ -196,3 +134,57 @@ class WorkGenerator(object): self.idx += 1 return WorkUnit(p_asset, self.assetNames, self.Test, self.idx) +class WorkUnit(object): + """ + XXX This is currently not implemented for KISS sake. + + This is an object responsible for completing WorkUnits it will + return its result in a deferred. + + The execution of a unit of work should be Atomic. + + Reporting to the OONI-net happens on completion of a Unit of Work. + + @Node node: This represents the node associated with the Work Unit + @Asset asset: This is the asset associated with the Work Unit + @Test test: This represents the Test to be with the specified assets + @ivar arguments: These are the extra attributes to be passsed to the Test + """ + + node = None + asset = None + test = None + arguments = None + + def __init__(self, asset, assetNames, test, idx): + self.asset = asset + if not asset: + self.assetGenerator = iter([1]) + else: + self.assetGenerator = iter(asset) + self.Test = test + self.assetNames = assetNames + self.idx = idx + + def __iter__(self): + return self + + def __repr__(self): + return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx) + + def serialize(self): + """ + Serialize this unit of work for RPC activity. + """ + return yaml.dump(self) + + def next(self): + """ + Launches the Unit of Work with the specified assets on the node. + """ + try: + asset = self.assetGenerator.next() + ret = self.Test.set_asset(asset) + return ret + except StopIteration: + raise StopIteration diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py index a2081bd..835735b 100644 --- a/ooni/protocols/http.py +++ b/ooni/protocols/http.py @@ -30,7 +30,6 @@ class BodyReceiver(protocol.Protocol): def connectionLost(self, reason): self.finished.callback(self.data) - from twisted.web.http_headers import Headers class HTTPTest(OONITest): """ @@ -77,7 +76,9 @@ class HTTPTest(OONITest): def experiment(self, args): log.msg("Running experiment") - d = self.build_request(args['url']) + url = self.local_options['url'] if 'url' not in args else args['url'] + + d = self.build_request(url) def finished(data): return data