[tor-commits] [ooni-probe/master] Do a big cleanup
art at torproject.org
art at torproject.org
Mon Jul 9 09:55:00 UTC 2012
commit d68e5f4f8edffbcd7e9c9fc5ce22030472f401a1
Author: Arturo Filastò <art at torproject.org>
Date: Mon Jul 9 11:54:37 2012 +0200
Do a big cleanup
* Better specify test writing interface
* Move example plugins to appropriate directory
* Implement RFC3339 timestamp, as spec
* Make sure all the defaults tests run and output something even without arguments
* Fix bug in worker
---
ooni/date.py | 17 ++++-
ooni/example_plugins/examplescapy.py | 49 ++++++++++++++
ooni/example_plugins/skel.py | 29 ++++++++
ooni/lib/Makefile | 8 ++-
ooni/ooniprobe.py | 1 -
ooni/plugins/blocking.py | 6 +-
ooni/plugins/chinatrigger.py | 2 +-
ooni/plugins/examplescapy.py | 49 --------------
ooni/plugins/httphost.py | 11 ++--
ooni/plugins/httpt.py | 5 +-
ooni/plugins/skel.py | 29 --------
ooni/plugins/tcpconnect.py | 9 ++-
ooni/plugoo/interface.py | 31 ++++++++-
ooni/plugoo/reports.py | 2 +-
ooni/plugoo/tests.py | 29 +++++---
ooni/plugoo/work.py | 118 ++++++++++++++++------------------
ooni/protocols/http.py | 5 +-
17 files changed, 223 insertions(+), 177 deletions(-)
diff --git a/ooni/date.py b/ooni/date.py
index 6f83191..59ec1f8 100644
--- a/ooni/date.py
+++ b/ooni/date.py
@@ -1,11 +1,22 @@
+from ooni.lib.rfc3339 import rfc3339
from datetime import datetime
+class odate(datetime):
+ def __str__(self):
+ return rfc3339(self)
+
+ def __repr__(self):
+ return "'%s'" % rfc3339(self)
+
+ def from_rfc(self, datestr):
+ pass
+
+def now():
+ return odate.utcnow()
+
def pretty_date():
cur_time = datetime.utcnow()
d_format = "%d %B %Y %H:%M:%S"
pretty = cur_time.strftime(d_format)
return pretty
-def now():
- return datetime.utcnow()
-
diff --git a/ooni/example_plugins/examplescapy.py b/ooni/example_plugins/examplescapy.py
new file mode 100644
index 0000000..aa6d81b
--- /dev/null
+++ b/ooni/example_plugins/examplescapy.py
@@ -0,0 +1,49 @@
+import random
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+from twisted.internet import protocol, defer
+from ooni.plugoo.tests import ITest, OONITest
+from ooni.plugoo.assets import Asset
+from ooni import log
+from ooni.protocols.scapy import ScapyTest
+
+from ooni.lib.txscapy import txsr, txsend
+
+class scapyArgs(usage.Options):
+ optParameters = []
+
+class ExampleScapyTest(ScapyTest):
+ """
+ An example of writing a scapy Test
+ """
+ implements(IPlugin, ITest)
+
+ shortName = "example_scapy"
+ description = "An example of a scapy test"
+ requirements = None
+ options = scapyArgs
+ blocking = False
+
+ receive = True
+ pcapfile = 'example_scapy.pcap'
+ def initialize(self, reactor=None):
+ if not self.reactor:
+ from twisted.internet import reactor
+ self.reactor = reactor
+
+ self.request = {}
+ self.response = {}
+
+ def build_packets(self):
+ """
+ Override this method to build scapy packets.
+ """
+ from scapy.all import IP, TCP
+ return IP()/TCP()
+
+ def load_assets(self):
+ return {}
+
+examplescapy = ExampleScapyTest(None, None, None)
+
diff --git a/ooni/example_plugins/skel.py b/ooni/example_plugins/skel.py
new file mode 100644
index 0000000..de95b48
--- /dev/null
+++ b/ooni/example_plugins/skel.py
@@ -0,0 +1,29 @@
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+from plugoo.tests import ITest, OONITest
+from ooni import log
+
+class SkelArgs(usage.Options):
+ optParameters = [['asset', 'a', None, 'Asset file'],
+ ['resume', 'r', 0, 'Resume at this index'],
+ ['other', 'o', None, 'Other arguments']]
+
+class SkelTest(OONITest):
+ implements(IPlugin, ITest)
+
+ shortName = "skeleton"
+ description = "Skeleton plugin"
+ requirements = None
+ options = SkelArgs
+ blocking = False
+
+ def load_assets(self):
+ if self.local_options:
+ return {'asset': open(self.local_options['asset'])}
+ else:
+ return {}
+
+# We need to instantiate it otherwise getPlugins does not detect it
+# XXX Find a way to load plugins without instantiating them.
+skel = SkelTest(None, None, None)
diff --git a/ooni/lib/Makefile b/ooni/lib/Makefile
index 37abe05..a498a35 100644
--- a/ooni/lib/Makefile
+++ b/ooni/lib/Makefile
@@ -1,4 +1,4 @@
-all: txtorcon txtraceroute txscapy
+all: txtorcon txtraceroute txscapy rfc3339
txtraceroute:
echo "Processing dependency txtraceroute..."
@@ -18,3 +18,9 @@ txscapy:
mv txscapy.git/txscapy.py txscapy.py
rm -rf txscapy.git
+rfc3339:
+ echo "Processing RFC3339 dependency"
+ hg clone https://bitbucket.org/henry/rfc3339 rfc3339
+ mv rfc3339/rfc3339.py rfc3339.py
+ rf -rf rfc3339
+
diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py
index 662956a..3eb70f6 100755
--- a/ooni/ooniprobe.py
+++ b/ooni/ooniprobe.py
@@ -71,7 +71,6 @@ def runTest(test, options, global_options):
reactor=reactor),
dict(options),
start=resume)
-
for x in wgen:
worker.push(x)
diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py
index b6b63b6..6ac3c91 100644
--- a/ooni/plugins/blocking.py
+++ b/ooni/plugins/blocking.py
@@ -27,11 +27,13 @@ class BlockingTest(OONITest):
def experiment(self, args):
import urllib
- req = urllib.urlopen(args['asset'])
+ url = 'https://torproject.org/' if not 'asset' in args else args['asset']
+ req = urllib.urlopen(url)
+
return {'page': req.readlines()}
def load_assets(self):
- if self.local_options:
+ if self.local_options and self.local_options['asset']:
return {'asset': Asset(self.local_options['asset'])}
else:
return {}
diff --git a/ooni/plugins/chinatrigger.py b/ooni/plugins/chinatrigger.py
index 8a2418a..4f2dc8c 100644
--- a/ooni/plugins/chinatrigger.py
+++ b/ooni/plugins/chinatrigger.py
@@ -32,7 +32,7 @@ class ChinaTriggerTest(ScapyTest):
shortName = "chinatrigger"
description = "Triggers the chinese probes into scanning"
- requirements = None
+ requirements = ['root']
options = scapyArgs
blocking = False
diff --git a/ooni/plugins/examplescapy.py b/ooni/plugins/examplescapy.py
deleted file mode 100644
index aa6d81b..0000000
--- a/ooni/plugins/examplescapy.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import random
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from twisted.internet import protocol, defer
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.plugoo.assets import Asset
-from ooni import log
-from ooni.protocols.scapy import ScapyTest
-
-from ooni.lib.txscapy import txsr, txsend
-
-class scapyArgs(usage.Options):
- optParameters = []
-
-class ExampleScapyTest(ScapyTest):
- """
- An example of writing a scapy Test
- """
- implements(IPlugin, ITest)
-
- shortName = "example_scapy"
- description = "An example of a scapy test"
- requirements = None
- options = scapyArgs
- blocking = False
-
- receive = True
- pcapfile = 'example_scapy.pcap'
- def initialize(self, reactor=None):
- if not self.reactor:
- from twisted.internet import reactor
- self.reactor = reactor
-
- self.request = {}
- self.response = {}
-
- def build_packets(self):
- """
- Override this method to build scapy packets.
- """
- from scapy.all import IP, TCP
- return IP()/TCP()
-
- def load_assets(self):
- return {}
-
-examplescapy = ExampleScapyTest(None, None, None)
-
diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py
index 45ab587..ccadff9 100644
--- a/ooni/plugins/httphost.py
+++ b/ooni/plugins/httphost.py
@@ -23,7 +23,7 @@ from ooni.plugoo.tests import ITest, OONITest
class HTTPHostArgs(usage.Options):
optParameters = [['asset', 'a', None, 'Asset file'],
- ['controlserver', 'c', None, 'Specify the control server'],
+ ['controlserver', 'c', 'google.com', 'Specify the control server'],
['resume', 'r', 0, 'Resume at this index'],
['other', 'o', None, 'Other arguments']]
def control(self, experiment_result, args):
@@ -61,7 +61,7 @@ class HTTPHostTest(OONITest):
def check_response(self, response):
soup = BeautifulSoup(response)
- if soup.head.title.string == "WikiLeaks":
+ if soup.head.title.string == "Blocked":
# Response indicates censorship
return True
else:
@@ -107,13 +107,14 @@ class HTTPHostTest(OONITest):
def experiment(self, args):
control_server = self.local_options['controlserver']
- censored = self.httplib_test(control_server, args['asset'])
+ url = 'http://torproject.org/' if not 'asset' in args else args['asset']
+ censored = self.httplib_test(control_server, url)
return {'control': control_server,
- 'host': args['asset'],
+ 'host': url,
'censored': censored}
def load_assets(self):
- if self.local_options:
+ if self.local_options and self.local_options['asset']:
return {'asset': Asset(self.local_options['asset'])}
else:
return {}
diff --git a/ooni/plugins/httpt.py b/ooni/plugins/httpt.py
index 3f92019..d0ede0f 100644
--- a/ooni/plugins/httpt.py
+++ b/ooni/plugins/httpt.py
@@ -12,7 +12,8 @@ from ooni.protocols import http
from ooni import log
class httptArgs(usage.Options):
- optParameters = [['urls', 'u', None, 'Urls file'],
+ optParameters = [['urls', 'f', None, 'Urls file'],
+ ['url', 'u', 'http://torproject.org/', 'Test single site'],
['resume', 'r', 0, 'Resume at this index']]
class httptTest(http.HTTPTest):
@@ -30,7 +31,7 @@ class httptTest(http.HTTPTest):
return {}
def load_assets(self):
- if self.local_options:
+ if self.local_options and self.local_options['urls']:
return {'url': Asset(self.local_options['urls'])}
else:
return {}
diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py
deleted file mode 100644
index de95b48..0000000
--- a/ooni/plugins/skel.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from plugoo.tests import ITest, OONITest
-from ooni import log
-
-class SkelArgs(usage.Options):
- optParameters = [['asset', 'a', None, 'Asset file'],
- ['resume', 'r', 0, 'Resume at this index'],
- ['other', 'o', None, 'Other arguments']]
-
-class SkelTest(OONITest):
- implements(IPlugin, ITest)
-
- shortName = "skeleton"
- description = "Skeleton plugin"
- requirements = None
- options = SkelArgs
- blocking = False
-
- def load_assets(self):
- if self.local_options:
- return {'asset': open(self.local_options['asset'])}
- else:
- return {}
-
-# We need to instantiate it otherwise getPlugins does not detect it
-# XXX Find a way to load plugins without instantiating them.
-skel = SkelTest(None, None, None)
diff --git a/ooni/plugins/tcpconnect.py b/ooni/plugins/tcpconnect.py
index 75f32c1..27df08d 100644
--- a/ooni/plugins/tcpconnect.py
+++ b/ooni/plugins/tcpconnect.py
@@ -15,7 +15,7 @@ from ooni.plugoo.assets import Asset
from ooni import log
class tcpconnectArgs(usage.Options):
- optParameters = [['blabla', 'a', None, 'Asset file'],
+ optParameters = [['asset', 'a', None, 'File containing IP:PORT combinations, one per line.'],
['resume', 'r', 0, 'Resume at this index']]
class tcpconnectTest(OONITest):
@@ -28,7 +28,10 @@ class tcpconnectTest(OONITest):
blocking = False
def experiment(self, args):
- host, port = args['blabla'].split(':')
+ try:
+ host, port = args['asset'].split(':')
+ except:
+ raise Exception("Error in parsing asset. Wrong format?")
class DummyFactory(Factory):
def buildProtocol(self, addr):
return Protocol()
@@ -53,7 +56,7 @@ class tcpconnectTest(OONITest):
def load_assets(self):
if self.local_options:
- return {'blabla': Asset(self.local_options['blabla'])}
+ return {'asset': Asset(self.local_options['asset'])}
else:
return {}
diff --git a/ooni/plugoo/interface.py b/ooni/plugoo/interface.py
index 68cff1a..c3b3855 100644
--- a/ooni/plugoo/interface.py
+++ b/ooni/plugoo/interface.py
@@ -10,14 +10,37 @@ class ITest(Interface):
requirements = Attribute("""What is required to run this this test, for example raw socket access or UDP or TCP""")
- #deferred = Attribute("""This will be fired on test completion""")
- #node = Attribute("""This represents the node that will run the test""")
options = Attribute("""These are the arguments to be passed to the test for it's execution""")
blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""")
- def startTest(asset):
+ def control(experiment_result, args):
"""
- Launches the Test with the specified arguments on a node.
+ @param experiment_result: The result returned by the experiment method.
+
+ @param args: the keys of this dict are the names of the assets passed in
+ from load_assets. The value is one item of the asset.
+
+ Must return a dict containing what should be written to the report.
+ Anything returned by control ends up inside of the YAMLOONI report.
+ """
+
+ def experiment(args):
+ """
+ Perform all the operations that are necessary to running a test.
+
+ @param args: the keys of this dict are the names of the assets passed in
+ from load_assets. The value is one item of the asset.
+
+ Must return a dict containing the values to be passed to control.
+ """
+
+ def load_assets():
+ """
+ Load the assets that should be passed to the Test. These are the inputs
+ to the OONI test.
+ Must return a dict that has as keys the asset names and values the
+ asset contents.
+ If the test does not have any assets it should return an empty dict.
"""
diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py
index 6140294..63bc3f5 100644
--- a/ooni/plugoo/reports.py
+++ b/ooni/plugoo/reports.py
@@ -44,7 +44,7 @@ class Report:
header += "# %s\n\n" % pretty_date
self._write_to_report(header)
# XXX replace this with something proper
- test_details = {'start_time': date.now(),
+ test_details = {'start_time': str(date.now()),
'asn': 'ASN-1234',
'test_name': self.testname,
'addr': '1234'}
diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py
index 608cee3..a3e9150 100644
--- a/ooni/plugoo/tests.py
+++ b/ooni/plugoo/tests.py
@@ -1,16 +1,14 @@
import os
-from datetime import datetime
import yaml
from zope.interface import Interface, Attribute
import logging
import itertools
-import gevent
-
from twisted.internet import reactor, defer, threads
from twisted.python import failure
from ooni import log
+from ooni import date
from ooni.plugoo import assets, work
from ooni.plugoo.reports import Report
from ooni.plugoo.interface import ITest
@@ -38,7 +36,7 @@ class OONITest(object):
This method should be overriden by the test writer to provide the logic
for loading their assets.
"""
- return {'asset': None}
+ return {}
def __repr__(self):
return "<OONITest %s %s %s>" % (self.options, self.global_options,
@@ -46,11 +44,11 @@ class OONITest(object):
def finished(self, control):
#self.ooninet.report(result)
- self.end_time = datetime.now()
+ self.end_time = date.now()
result = self.result
- result['start_time'] = self.start_time
- result['end_time'] = self.end_time
- result['run_time'] = self.end_time - self.start_time
+ result['start_time'] = str(self.start_time)
+ result['end_time'] = str(self.end_time)
+ result['run_time'] = str(self.end_time - self.start_time)
result['control'] = control
log.msg("FINISHED %s" % result)
self.report(result)
@@ -68,14 +66,23 @@ class OONITest(object):
def control(self, result, args):
log.msg("Doing control")
- return result
+
+ if self.blocking:
+ return result
+
+ def end(cb):
+ return result
+ d = defer.Deferred()
+ d.addCallback(end)
+ return d
def experiment(self, args):
log.msg("Doing experiment")
- return {}
+ d = defer.Deferred()
+ return d
def startTest(self, args):
- self.start_time = datetime.now()
+ self.start_time = date.now()
log.msg("Starting test %s" % self.__class__)
return self._do_experiment(args)
diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py
index 7e9be0c..a3fb168 100644
--- a/ooni/plugoo/work.py
+++ b/ooni/plugoo/work.py
@@ -40,12 +40,7 @@ class Worker(object):
if isinstance(r, failure.Failure):
r.trap()
- print r['start_time']
- print r['end_time']
- print r['run_time']
-
if self._running == 0 and not self._queued:
- print "I am done."
reactor.stop()
return r
@@ -60,62 +55,6 @@ class Worker(object):
self._queued.append((workunit, d))
return d
-class WorkUnit(object):
- """
- XXX This is currently not implemented for KISS sake.
-
- This is an object responsible for completing WorkUnits it will
- return its result in a deferred.
-
- The execution of a unit of work should be Atomic.
-
- Reporting to the OONI-net happens on completion of a Unit of Work.
-
- @Node node: This represents the node associated with the Work Unit
- @Asset asset: This is the asset associated with the Work Unit
- @Test test: This represents the Test to be with the specified assets
- @ivar arguments: These are the extra attributes to be passsed to the Test
- """
-
- node = None
- asset = None
- test = None
- arguments = None
-
- def __init__(self, asset, assetNames, test, idx):
- self.asset = asset
- if not asset:
- self.assetGenerator = iter([1])
- else:
- self.assetGenerator = iter(asset)
- self.Test = test
- self.assetNames = assetNames
- self.idx = idx
-
- def __iter__(self):
- return self
-
- def __repr__(self):
- return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx)
-
- def serialize(self):
- """
- Serialize this unit of work for RPC activity.
- """
- return yaml.dump(self)
-
- def next(self):
- """
- Launches the Unit of Work with the specified assets on the node.
- """
- try:
- asset = self.assetGenerator.next()
- ret = self.Test.set_asset(asset)
- return ret
- except StopIteration:
- raise StopIteration
-
-
class WorkGenerator(object):
"""
Factory responsible for creating units of work.
@@ -129,7 +68,6 @@ class WorkGenerator(object):
self.Test = test
if self.Test.assets and self.Test.assets.values()[0]:
- print self.Test.assets
self.assetGenerator = itertools.product(*self.Test.assets.values())
else:
self.assetGenerator = None
@@ -156,7 +94,7 @@ class WorkGenerator(object):
if not self.assetGenerator:
self.end = True
- return (self.assetNames, self.Test, self.idx)
+ return ({}, self.Test, self.idx)
try:
asset = self.assetGenerator.next()
@@ -196,3 +134,57 @@ class WorkGenerator(object):
self.idx += 1
return WorkUnit(p_asset, self.assetNames, self.Test, self.idx)
+class WorkUnit(object):
+ """
+ XXX This is currently not implemented for KISS sake.
+
+ This is an object responsible for completing WorkUnits it will
+ return its result in a deferred.
+
+ The execution of a unit of work should be Atomic.
+
+ Reporting to the OONI-net happens on completion of a Unit of Work.
+
+ @Node node: This represents the node associated with the Work Unit
+ @Asset asset: This is the asset associated with the Work Unit
+ @Test test: This represents the Test to be with the specified assets
+ @ivar arguments: These are the extra attributes to be passsed to the Test
+ """
+
+ node = None
+ asset = None
+ test = None
+ arguments = None
+
+ def __init__(self, asset, assetNames, test, idx):
+ self.asset = asset
+ if not asset:
+ self.assetGenerator = iter([1])
+ else:
+ self.assetGenerator = iter(asset)
+ self.Test = test
+ self.assetNames = assetNames
+ self.idx = idx
+
+ def __iter__(self):
+ return self
+
+ def __repr__(self):
+ return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx)
+
+ def serialize(self):
+ """
+ Serialize this unit of work for RPC activity.
+ """
+ return yaml.dump(self)
+
+ def next(self):
+ """
+ Launches the Unit of Work with the specified assets on the node.
+ """
+ try:
+ asset = self.assetGenerator.next()
+ ret = self.Test.set_asset(asset)
+ return ret
+ except StopIteration:
+ raise StopIteration
diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py
index a2081bd..835735b 100644
--- a/ooni/protocols/http.py
+++ b/ooni/protocols/http.py
@@ -30,7 +30,6 @@ class BodyReceiver(protocol.Protocol):
def connectionLost(self, reason):
self.finished.callback(self.data)
-
from twisted.web.http_headers import Headers
class HTTPTest(OONITest):
"""
@@ -77,7 +76,9 @@ class HTTPTest(OONITest):
def experiment(self, args):
log.msg("Running experiment")
- d = self.build_request(args['url'])
+ url = self.local_options['url'] if 'url' not in args else args['url']
+
+ d = self.build_request(url)
def finished(data):
return data
More information about the tor-commits
mailing list