[tor-commits] [ooni-probe/master] Big refactor and code reorganization.
art at torproject.org
art at torproject.org
Mon Mar 5 06:54:12 UTC 2012
commit 4258d6e33547ab24d604d6e800dc1ee23df43b64
Author: Arturo Filastò <hellais at gmail.com>
Date: Sun Mar 4 22:53:20 2012 -0800
Big refactor and code reorganization.
---
node.py | 66 ----------
ooniprobe.py | 3 +-
plugoo.py | 348 ---------------------------------------------------
plugoo/__init__.py | 53 ++++++++
plugoo/assets.py | 45 +++++++
plugoo/nodes.py | 71 +++++++++++
plugoo/reports.py | 170 +++++++++++++++++++++++++
plugoo/tests.py | 105 +++++++++++++++
tests/bridget.py | 10 +-
tests/dnstamper.py | 8 +-
tests/traceroute.py | 7 +-
11 files changed, 460 insertions(+), 426 deletions(-)
diff --git a/node.py b/node.py
deleted file mode 100644
index eea78df..0000000
--- a/node.py
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8
-
-import os
-import sys
-import socks
-
-class Node(object):
- def __init__(self, address, port):
- self.address = address
- self.port = port
-
-"""
-[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5")
-[]: node_socket = node.wrap_socket()
-"""
-class NetworkNode(Node):
- def __init__(self, address, port, node_type="SOCKS5", auth_creds=None):
- self.node = Node(address,port)
-
- # XXX support for multiple types
- # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...)
- self.node_type = node_type
- # type-specific authentication credentials
- self.auth_creds = auth_creds
-
- def _get_socksipy_socket(self, proxy_type, auth_creds):
- import socks
- s = socks.socksocket()
- # auth_creds[0] -> username
- # auth_creds[1] -> password
- s.setproxy(proxy_type, self.node.address, self.node.port,
- self.auth_creds[0], self.auth_creds[1])
- return s
-
- def _get_socket_wrapper(self):
- if (self.node_type.startswith("SOCKS")): # SOCKS proxies
- if (self.node_type != "SOCKS5"):
- proxy_type = socks.PROXY_TYPE_SOCKS5
- elif (self.node_type != "SOCKS4"):
- proxy_type = socks.PROXY_TYPE_SOCKS4
- else:
- print "We don't know this proxy type."
- sys.exit(1)
-
- return self._get_socksipy_socket(proxy_type)
- elif (self.node_type == "HTTP"): # HTTP proxies
- return self._get_socksipy_socket(PROXY_TYPE_HTTP)
- else: # Unknown proxies
- print "We don't know this proxy type."
- sys.exit(1)
-
- def wrap_socket(self):
- return self._get_socket_wrapper()
-
-class CodeExecNode(Node):
- def __init__(self, address, port, node_type, auth_creds):
- self.node = Node(address,port)
-
- # node type (SSH proxy, etc.)
- self.node_type = node_type
- # type-specific authentication credentials
- self.auth_creds = auth_creds
-
- def add_unit(self):
- pass
diff --git a/ooniprobe.py b/ooniprobe.py
index d8501c1..a542153 100755
--- a/ooniprobe.py
+++ b/ooniprobe.py
@@ -96,7 +96,8 @@ class ooni(object):
test_name = fname
if not self.config.main.testdir in sys.path:
sys.path.insert(0, self.config.main.testdir)
-
+ #print "Fname: %s\n__import__(%s)" % (fname, fname)
+ #print sys.path
module = __import__(fname)
try:
test.name = module.__plugoo__
diff --git a/plugoo.py b/plugoo.py
deleted file mode 100644
index 0ce1d28..0000000
--- a/plugoo.py
+++ /dev/null
@@ -1,348 +0,0 @@
-# -*- coding: UTF-8
-"""
- plugoo
- ******
-
- This contains all of the "goo" necessary for creating
- ooni-probe plugoonies.
-
- :copyright: (c) 2012 by Arturo Filastò.
- :license: see LICENSE for more details.
-
-"""
-
-import os
-from datetime import datetime
-import yaml
-
-try:
- import socks
-except:
- "Error SocksiPy is not installed!"
-import socket
-
-import logging
-import itertools
-import gevent
-
-class Asset:
- """This is an ooni-probe asset. It is a python
- iterator object, allowing it to be efficiently looped.
- To create your own custom asset your should subclass this
- and override the next_asset method and the len method for
- computing the length of the asset.
- """
- def __init__(self, file=None):
- self.fh = None
- if file:
- self.name = file
- self.fh = open(file, 'r')
- self.eof = False
-
- def __iter__(self):
- return self
-
- def len(self):
- """Returns the length of the asset
- """
- for i, l in enumerate(self.fh):
- pass
- # rewind the file
- self.fh.seek(0)
- return i + 1
-
- def next_asset(self):
- """Return the next asset.
- """
- # XXX this is really written with my feet.
- # clean me up please...
- line = self.fh.readline()
- if line:
- return line.replace('\n','')
- else:
- self.fh.seek(0)
- raise StopIteration
-
- def next(self):
- try:
- return self.next_asset()
- except:
- raise StopIteration
-
-
-class Report:
- """This is the ooni-probe reporting mechanism. It allows
- reporting to multiple destinations and file formats.
-
- :scp the string of <host>:<port> of an ssh server
-
- :yaml the filename of a the yaml file to write
-
- :file the filename of a simple txt file to write
-
- :tcp the <host>:<port> of a TCP server that will just listen for
- inbound connection and accept a stream of data (think of it
- as a `nc -l -p <port> > filename.txt`)
- """
- def __init__(self, ooni,
- scp="127.0.0.1:22",
- file="test.report",
- tcp="127.0.0.1:9000"):
-
- self.file = file
- self.tcp = tcp
- self.scp = scp
- self.config = ooni.config.report
- self.logger = ooni.logger
-
- if self.config.timestamp:
- tmp = self.file.split('.')
- self.file = '.'.join(tmp[:-1]) + "-" + \
- datetime.now().isoformat('-') + '.' + \
- tmp[-1]
- print self.file
-
- try:
- import paramiko
- except:
- self.scp = None
- self.logger.warn("Could not import paramiko. SCP will not be disabled")
-
- def __call__(self, data):
- """
- This should be invoked every time you wish to write some
- data to the reporting system
- """
- #print "Writing report(s)"
- dump = '--- \n'
- dump += yaml.dump(data)
- reports = []
-
- if self.file:
- reports.append("file")
-
- if self.tcp:
- reports.append("tcp")
-
- if self.scp:
- reports.append("scp")
-
- jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports]
- gevent.joinall(jobs)
- ret = []
- for job in jobs:
- #print job.value
- ret.append(job.value)
- return ret
-
- def file_report(self, data, file=None, mode='a+'):
- """
- This reports to a file in YAML format
- """
- if not file:
- file = self.file
- try:
- f = open(file, mode)
- f.write(data)
- except Exception, e:
- raise e
- finally:
- f.close()
-
-
- def tcp_report(self, data):
- """This connect to the specified tcp server
- and writes the data passed as argument.
- """
- host, port = self.tcp.split(":")
- tcp = socket.getprotobyname('tcp')
- send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp)
- try:
- send_socket.connect((host, int(port)))
- send_socket.send(data)
-
- except Exception, e:
- raise e
-
- finally:
- send_socket.close()
-
-
- def scp_report(self, data, rfile=None, mode='a+'):
- """Push data to the remote ssh server.
- :rfile the remote filename to write
- :data the raw data content that should be written
- :mode in what mode the file should be created
- """
- if not rfile:
- rfile = self.file
- host, port = self.scp.split(":")
- transport = paramiko.Transport((host, port))
-
- # The remote path of the remote file to write
- rfpath = os.path.join(self.config.ssh_rpath, rfile)
-
- try:
- username = self.config.ssh_username
- except:
- raise "No username provided"
-
- # Load the local known host key file
- transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts"))
-
- # We prefer to use an ssh keyfile fo authentication
- if self.config.ssh_keyfile:
- keyfile = os.path.expanduser(self.config.ssh_keyfile)
- key = paramiko.RSAKey.from_private_key_file(keylocfile)
- try:
- transport.connect(username=username, pkey=key)
- except Exception, e:
- raise e
-
- # If not even a password is fine
- elif self.config.ssh_password:
- try:
- transport.connect(username=username, password=self.config.ssh_password)
- except Exception, e:
- raise e
-
- # ... but no authentication, that is madness!
- else:
- raise "No key or password provided for ssh"
-
- sftp = paramiko.SFTPClient.from_transport(transport)
- try:
- sftp = ssh.open_sftp()
- remote_file = sftp.file(rfile, mode)
- remote_file.set_pipelined(True)
- remote_file.write(data)
-
- except Exception, e:
- raise e
- sftp.close()
- transport.close()
-
-
- def send_report(self, data, type):
- """This sends the report using the
- specified type.
- """
- #print "Reporting %s to %s" % (data, type)
- self.logger.info("Reporting to %s" % type)
- getattr(self, type+"_report").__call__(data)
-
-class Plugoo():
- def __init__(self, ooni):
- self.config = ooni.config
- self.logger = ooni.logger
- self.name = "test"
- self.report = Report(ooni,
- scp=ooni.config.report.ssh,
- file=ooni.config.report.file,
- tcp=ooni.config.report.tcp)
-
-
- def control(self, *a, **b):
- pass
-
- def experiment(self, *a, **b):
- """Override this method to write your own
- Plugoo.
- """
- pass
-
- def load_assets(self, assets):
- """Takes as input an array of Asset objects and
- outputs an iterator for the loaded assets.
- example:
- assets = [hostlist, portlist, requestlist]
-
- """
- asset_count = len(assets)
- bigsize = 0
- bigidx = 0
-
- if asset_count > 1:
- # If we have more than on asset we try to do some
- # optimizations as how to iterate through them by
- # picking the largest asset set as the main iterator
- # and do a cartesian product on the smaller sets
- for i, v in enumerate(assets):
- size = v.len()
- if size > bigsize:
- bigidx, bigsize = (i, size)
-
- smallassets = list(assets)
- smallassets.pop(bigidx)
-
- for x in assets[bigidx]:
- if asset_count > 1:
- # XXX this will only work in python 2.6, maybe refactor?
- for comb in itertools.product(*smallassets):
- yield (x,) + comb
- else:
- yield (x)
-
- def srun(self, assets=None, buffer=10, timeout=2):
- self.logger.info("Starting %s", self.name)
- if assets:
- self.logger.debug("Running through tests")
- for i, data in enumerate(self.load_assets(assets)):
- args = {'data': data}
- ret = self.experiment(**args)
- print ret
- self.report(ret)
-
- def run(self, assets=None, buffer=10, timeout=100000):
- self.logger.info("Starting %s", self.name)
- jobs = []
- if assets:
- self.logger.debug("Running through tests")
- for i, data in enumerate(self.load_assets(assets)):
- args = {'data': data}
- # Append to the job queue
- jobs.append(gevent.spawn(self.experiment, **args))
- # If the buffer is full run the jobs
- if i % buffer == (buffer-1):
- # Run the jobs with the selected timeout
- gevent.joinall(jobs, timeout=timeout)
- for job in jobs:
- #print "JOB VAL: %s" % job.value
- self.logger.info("Writing report(s)")
- self.report(job.value)
- job.kill()
- jobs = []
-
- if len(jobs) > 0:
- gevent.joinall(jobs, timeout=timeout)
- for job in jobs:
- #print "JOB VAL: %s" % job.value
- self.logger.info("Writing report(s)")
- self.report(job.value)
- job.kill()
- jobs = []
-
-class torify(object):
- """This is the torify decorator. It should be used to
- decorate functions that should use to for connecting to
- the interwebz. The suggary syntax is the following:
- @torify([urllib2])
- def myfunction():
- f = urllib2.urlopen('https://torproject.org/')
- remember to set the proxyaddress in the config file.
- """
- def __init__(self, f):
- print f
- self.f = f
-
- def __get__(self, instance, owner):
- self.modules = instance.modules
- def decorator(*args):
- print instance.config.main.proxyaddress
- host, port = instance.config.main.proxyaddress.split(":")
- socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, int(port))
- # Wrap the modules into socks
- for module in self.modules:
- socks.wrapmodule(module)
- return self.f(instance, *args)
- return decorator
diff --git a/plugoo/__init__.py b/plugoo/__init__.py
new file mode 100644
index 0000000..1bd4446
--- /dev/null
+++ b/plugoo/__init__.py
@@ -0,0 +1,53 @@
+# -*- coding: UTF-8
+"""
+ plugoo
+ ******
+
+ This contains all of the "goo" necessary for creating
+ ooni-probe plugoonies.
+
+ :copyright: (c) 2012 by Arturo Filastò.
+ :license: see LICENSE for more details.
+
+"""
+
+__all__ = ['assets', 'reports', 'nodes']
+
+import os
+from datetime import datetime
+import yaml
+
+try:
+ import socks
+except:
+ "Error SocksiPy is not installed!"
+import socket
+
+import logging
+import itertools
+import gevent
+
+class torify(object):
+ """This is the torify decorator. It should be used to
+ decorate functions that should use to for connecting to
+ the interwebz. The suggary syntax is the following:
+ @torify([urllib2])
+ def myfunction():
+ f = urllib2.urlopen('https://torproject.org/')
+ remember to set the proxyaddress in the config file.
+ """
+ def __init__(self, f):
+ print f
+ self.f = f
+
+ def __get__(self, instance, owner):
+ self.modules = instance.modules
+ def decorator(*args):
+ print instance.config.main.proxyaddress
+ host, port = instance.config.main.proxyaddress.split(":")
+ socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, int(port))
+ # Wrap the modules into socks
+ for module in self.modules:
+ socks.wrapmodule(module)
+ return self.f(instance, *args)
+ return decorator
diff --git a/plugoo/assets.py b/plugoo/assets.py
new file mode 100644
index 0000000..5f93d1b
--- /dev/null
+++ b/plugoo/assets.py
@@ -0,0 +1,45 @@
+class Asset:
+ """This is an ooni-probe asset. It is a python
+ iterator object, allowing it to be efficiently looped.
+ To create your own custom asset your should subclass this
+ and override the next_asset method and the len method for
+ computing the length of the asset.
+ """
+ def __init__(self, file=None, *args, **argv):
+ self.fh = None
+ if file:
+ self.name = file
+ self.fh = open(file, 'r')
+ self.eof = False
+
+ def __iter__(self):
+ return self
+
+ def len(self):
+ """Returns the length of the asset
+ """
+ for i, l in enumerate(self.fh):
+ pass
+ # rewind the file
+ self.fh.seek(0)
+ return i + 1
+
+ def next_asset(self):
+ """Return the next asset.
+ """
+ # XXX this is really written with my feet.
+ # clean me up please...
+ line = self.fh.readline()
+ if line:
+ return line.replace('\n','')
+ else:
+ self.fh.seek(0)
+ raise StopIteration
+
+ def next(self):
+ try:
+ return self.next_asset()
+ except:
+ raise StopIteration
+
+
diff --git a/plugoo/nodes.py b/plugoo/nodes.py
new file mode 100644
index 0000000..a76601d
--- /dev/null
+++ b/plugoo/nodes.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8
+
+import os
+import sys
+import socks
+
+class Node(object):
+ def __init__(self, address, port):
+ self.address = address
+ self.port = port
+
+"""
+[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5")
+[]: node_socket = node.wrap_socket()
+"""
+class NetworkNode(Node):
+ def __init__(self, address, port, node_type="SOCKS5", auth_creds=None):
+ self.node = Node(address,port)
+
+ # XXX support for multiple types
+ # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...)
+ self.node_type = node_type
+ # type-specific authentication credentials
+ self.auth_creds = auth_creds
+
+ def _get_socksipy_socket(self, proxy_type, auth_creds):
+ import socks
+ s = socks.socksocket()
+ # auth_creds[0] -> username
+ # auth_creds[1] -> password
+ s.setproxy(proxy_type, self.node.address, self.node.port,
+ self.auth_creds[0], self.auth_creds[1])
+ return s
+
+ def _get_socket_wrapper(self):
+ if (self.node_type.startswith("SOCKS")): # SOCKS proxies
+ if (self.node_type != "SOCKS5"):
+ proxy_type = socks.PROXY_TYPE_SOCKS5
+ elif (self.node_type != "SOCKS4"):
+ proxy_type = socks.PROXY_TYPE_SOCKS4
+ else:
+ print "We don't know this proxy type."
+ sys.exit(1)
+
+ return self._get_socksipy_socket(proxy_type)
+ elif (self.node_type == "HTTP"): # HTTP proxies
+ return self._get_socksipy_socket(PROXY_TYPE_HTTP)
+ else: # Unknown proxies
+ print "We don't know this proxy type."
+ sys.exit(1)
+
+ def wrap_socket(self):
+ return self._get_socket_wrapper()
+
+class CodeExecNode(Node):
+ def __init__(self, address, port, node_type, auth_creds):
+ self.node = Node(address,port)
+
+ # node type (SSH proxy, etc.)
+ self.node_type = node_type
+ # type-specific authentication credentials
+ self.auth_creds = auth_creds
+
+ def add_unit(self):
+ pass
+
+ def get_status(self):
+ pass
+
+
diff --git a/plugoo/reports.py b/plugoo/reports.py
new file mode 100644
index 0000000..5a9fb8b
--- /dev/null
+++ b/plugoo/reports.py
@@ -0,0 +1,170 @@
+import os
+from datetime import datetime
+import yaml
+
+import logging
+import itertools
+import gevent
+
+class Report:
+ """This is the ooni-probe reporting mechanism. It allows
+ reporting to multiple destinations and file formats.
+
+ :scp the string of <host>:<port> of an ssh server
+
+ :yaml the filename of a the yaml file to write
+
+ :file the filename of a simple txt file to write
+
+ :tcp the <host>:<port> of a TCP server that will just listen for
+ inbound connection and accept a stream of data (think of it
+ as a `nc -l -p <port> > filename.txt`)
+ """
+ def __init__(self, ooni,
+ scp="127.0.0.1:22",
+ file="test.report",
+ tcp="127.0.0.1:9000"):
+
+ self.file = file
+ self.tcp = tcp
+ self.scp = scp
+ self.config = ooni.config.report
+ self.logger = ooni.logger
+
+ if self.config.timestamp:
+ tmp = self.file.split('.')
+ self.file = '.'.join(tmp[:-1]) + "-" + \
+ datetime.now().isoformat('-') + '.' + \
+ tmp[-1]
+ print self.file
+
+ try:
+ import paramiko
+ except:
+ self.scp = None
+ self.logger.warn("Could not import paramiko. SCP will not be disabled")
+
+ def __call__(self, data):
+ """
+ This should be invoked every time you wish to write some
+ data to the reporting system
+ """
+ #print "Writing report(s)"
+ dump = '--- \n'
+ dump += yaml.dump(data)
+ reports = []
+
+ if self.file:
+ reports.append("file")
+
+ if self.tcp:
+ reports.append("tcp")
+
+ if self.scp:
+ reports.append("scp")
+
+ jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports]
+ gevent.joinall(jobs)
+ ret = []
+ for job in jobs:
+ #print job.value
+ ret.append(job.value)
+ return ret
+
+ def file_report(self, data, file=None, mode='a+'):
+ """
+ This reports to a file in YAML format
+ """
+ if not file:
+ file = self.file
+ try:
+ f = open(file, mode)
+ f.write(data)
+ except Exception, e:
+ raise e
+ finally:
+ f.close()
+
+
+ def tcp_report(self, data):
+ """This connect to the specified tcp server
+ and writes the data passed as argument.
+ """
+ host, port = self.tcp.split(":")
+ tcp = socket.getprotobyname('tcp')
+ send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp)
+ try:
+ send_socket.connect((host, int(port)))
+ send_socket.send(data)
+
+ except Exception, e:
+ raise e
+
+ finally:
+ send_socket.close()
+
+
+ def scp_report(self, data, rfile=None, mode='a+'):
+ """Push data to the remote ssh server.
+ :rfile the remote filename to write
+ :data the raw data content that should be written
+ :mode in what mode the file should be created
+ """
+ if not rfile:
+ rfile = self.file
+ host, port = self.scp.split(":")
+ transport = paramiko.Transport((host, port))
+
+ # The remote path of the remote file to write
+ rfpath = os.path.join(self.config.ssh_rpath, rfile)
+
+ try:
+ username = self.config.ssh_username
+ except:
+ raise "No username provided"
+
+ # Load the local known host key file
+ transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts"))
+
+ # We prefer to use an ssh keyfile fo authentication
+ if self.config.ssh_keyfile:
+ keyfile = os.path.expanduser(self.config.ssh_keyfile)
+ key = paramiko.RSAKey.from_private_key_file(keylocfile)
+ try:
+ transport.connect(username=username, pkey=key)
+ except Exception, e:
+ raise e
+
+ # If not even a password is fine
+ elif self.config.ssh_password:
+ try:
+ transport.connect(username=username, password=self.config.ssh_password)
+ except Exception, e:
+ raise e
+
+ # ... but no authentication, that is madness!
+ else:
+ raise "No key or password provided for ssh"
+
+ sftp = paramiko.SFTPClient.from_transport(transport)
+ try:
+ sftp = ssh.open_sftp()
+ remote_file = sftp.file(rfile, mode)
+ remote_file.set_pipelined(True)
+ remote_file.write(data)
+
+ except Exception, e:
+ raise e
+ sftp.close()
+ transport.close()
+
+
+ def send_report(self, data, type):
+ """This sends the report using the
+ specified type.
+ """
+ #print "Reporting %s to %s" % (data, type)
+ self.logger.info("Reporting to %s" % type)
+ getattr(self, type+"_report").__call__(data)
+
+
diff --git a/plugoo/tests.py b/plugoo/tests.py
new file mode 100644
index 0000000..24986b2
--- /dev/null
+++ b/plugoo/tests.py
@@ -0,0 +1,105 @@
+import os
+from datetime import datetime
+import yaml
+
+import logging
+import itertools
+import gevent
+from plugoo.reports import Report
+
+class Test:
+ """
+ This is a ooni probe Test.
+ Also known as a Plugoo!
+ """
+ def __init__(self, ooni):
+ self.config = ooni.config
+ self.logger = ooni.logger
+ self.name = "test"
+ self.report = Report(ooni,
+ scp=ooni.config.report.ssh,
+ file=ooni.config.report.file,
+ tcp=ooni.config.report.tcp)
+
+
+ def control(self, *a, **b):
+ pass
+
+ def experiment(self, *a, **b):
+ """Override this method to write your own
+ Plugoo.
+ """
+ pass
+
+ def load_assets(self, assets):
+ """Takes as input an array of Asset objects and
+ outputs an iterator for the loaded assets.
+ example:
+ assets = [hostlist, portlist, requestlist]
+
+ """
+ asset_count = len(assets)
+ bigsize = 0
+ bigidx = 0
+
+ if asset_count > 1:
+ # If we have more than on asset we try to do some
+ # optimizations as how to iterate through them by
+ # picking the largest asset set as the main iterator
+ # and do a cartesian product on the smaller sets
+ for i, v in enumerate(assets):
+ size = v.len()
+ if size > bigsize:
+ bigidx, bigsize = (i, size)
+
+ smallassets = list(assets)
+ smallassets.pop(bigidx)
+
+ for x in assets[bigidx]:
+ if asset_count > 1:
+ # XXX this will only work in python 2.6, maybe refactor?
+ for comb in itertools.product(*smallassets):
+ yield (x,) + comb
+ else:
+ yield (x)
+
+ def srun(self, assets=None, buffer=10, timeout=2):
+ self.logger.info("Starting %s", self.name)
+ if assets:
+ self.logger.debug("Running through tests")
+ for i, data in enumerate(self.load_assets(assets)):
+ args = {'data': data}
+ ret = self.experiment(**args)
+ print ret
+ self.report(ret)
+
+ def run(self, assets=None, buffer=10, timeout=100000):
+ self.logger.info("Starting %s", self.name)
+ jobs = []
+ if assets:
+ self.logger.debug("Running through tests")
+ for i, data in enumerate(self.load_assets(assets)):
+ args = {'data': data}
+ # Append to the job queue
+ jobs.append(gevent.spawn(self.experiment, **args))
+ # If the buffer is full run the jobs
+ if i % buffer == (buffer-1):
+ # Run the jobs with the selected timeout
+ gevent.joinall(jobs, timeout=timeout)
+ for job in jobs:
+ #print "JOB VAL: %s" % job.value
+ self.logger.info("Writing report(s)")
+ self.report(job.value)
+ job.kill()
+ jobs = []
+
+ if len(jobs) > 0:
+ gevent.joinall(jobs, timeout=timeout)
+ for job in jobs:
+ #print "JOB VAL: %s" % job.value
+ self.logger.info("Writing report(s)")
+ self.report(job.value)
+ job.kill()
+ jobs = []
+
+
diff --git a/tests/bridget.py b/tests/bridget.py
index 0a2cc4f..61fc8b7 100644
--- a/tests/bridget.py
+++ b/tests/bridget.py
@@ -23,11 +23,11 @@ from subprocess import Popen, PIPE
from datetime import datetime
import shutil
-import plugoo
import gevent
from gevent import socket
import fcntl
-from plugoo import Plugoo, Asset, torify
+from plugoo.assets import Asset
+from plugoo.tests import Test
import urllib2
import httplib
import json
@@ -71,7 +71,7 @@ class BridgeTAsset(Asset):
def __init__(self, file=None):
self = Asset.__init__(self, file)
-class BridgeT(Plugoo):
+class BridgeT(Test):
# This is the timeout value after which
# we will give up
timeout = 20
@@ -368,11 +368,11 @@ def run(ooni):
bridges = BridgeTAsset(os.path.join(config.main.assetdir, \
config.tests.tor_bridges))
- assets = [bridges]
+ bridgelist = [bridges]
bridget = BridgeT(ooni)
ooni.logger.info("Starting bridget test")
- bridget.run(assets)
+ bridget.run(bridgelist)
bridget.print_failures()
bridget.clean()
ooni.logger.info("Testing completed!")
diff --git a/tests/dnstamper.py b/tests/dnstamper.py
index 06b3551..68be12d 100644
--- a/tests/dnstamper.py
+++ b/tests/dnstamper.py
@@ -5,16 +5,18 @@ except:
import gevent
import os
import plugoo
-from plugoo import Plugoo, Asset
+from plugoo.assets import Asset
+from plugoo.tests import Test
+
__plugoo__ = "DNST"
__desc__ = "DNS censorship detection test"
class DNSTAsset(Asset):
def __init__(self, file=None):
- self = Asset.__init__(self, file)
+ self = asset.__init__(self, file)
-class DNST(Plugoo):
+class DNST(Test):
def lookup(self, hostname, ns):
res = resolver.Resolver(configure=False)
res.nameservers = [ns]
diff --git a/tests/traceroute.py b/tests/traceroute.py
index f2272c9..891d356 100644
--- a/tests/traceroute.py
+++ b/tests/traceroute.py
@@ -11,7 +11,8 @@ try:
except:
print "Error: traceroute plugin requires scapy to be installed (http://www.secdev.org/projects/scapy)"
-from plugoo import Plugoo, Asset
+from plugoo.assets import Asset
+from plugoo.tests import Test
import socket
@@ -20,10 +21,10 @@ __desc__ = "Performs TTL walking tests"
class TracerouteAsset(Asset):
def __init__(self, file=None):
- self = Asset.__init__(self, file)
+ self = asset.__init__(self, file)
-class Traceroute(Plugoo):
+class Traceroute(Test):
"""A *very* quick and dirty traceroute implementation, UDP and TCP
"""
def traceroute(self, dst, dst_port=3880, src_port=3000, proto="tcp", max_hops=30):
More information about the tor-commits
mailing list