[tor-commits] [ooni-probe/master] Remove all references to the old API

commit 654147c3a92156c35fc1a8ecc0aac8e933f57e59 Author: Arturo Filastò arturo@filasto.net Date: Wed Nov 7 00:47:59 2012 +0100

Remove all references to the old API --- docs/source/index.rst | 40 ++- nettests/bridge_reachability/echo.py | 5 +- nettests/core/echo.py | 1 - ooni/__init__.py | 7 - ooni/custodiet.py | 421 ------------------------------- ooni/lib/Makefile | 36 --- ooni/nodes.py | 176 +++++++++++++ ooni/oonicli.py | 11 +- ooni/plugoo/__init__.py | 47 ---- ooni/plugoo/assets.py | 62 ----- ooni/plugoo/interface.py | 56 ---- ooni/plugoo/nodes.py | 176 ------------- ooni/plugoo/reports.py | 145 ----------- ooni/plugoo/tests.py | 142 ----------- ooni/plugoo/work.py | 148 ----------- ooni/protocols/http.py | 141 ----------- ooni/protocols/scapyproto.py | 55 ---- ooni/reporter.py | 20 +- ooni/runner.py | 61 +---- ooni/scaffolding.py | 78 ------ ooni/utils/legacy.py | 459 ---------------------------------- 21 files changed, 228 insertions(+), 2059 deletions(-)

diff --git a/docs/source/index.rst b/docs/source/index.rst index 5c96dd1..2497a09 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,26 +15,41 @@ about the various types, methods, and amounts of network tampering in the world. Getting started ***************

-If you choose to use virtualenv to setup your development environment you will -need to do the following:: +Requirements:

- virtualenv ENV - source ENV/bin/activate - pip install twisted Scapy pyyaml pyOpenSSL + * Git: http://git-scm.com/book/en/Getting-Started-Installing-Git + * Python >= 2.6: http://www.python.org/download/releases/ + * pip: http://www.pip-installer.org/en/latest/

-To get the latest version of scapy you will need mercurial. You can then install -it with:: +On debian based systems these can be installed with:

- pip install hg+http://hg.secdev.org/scapy + apt-get install git-core python python-pip python-dev

-On debian you can install all the dependecies with apt-get with this command:: +The python dependencies required for running ooniprobe are:

- apt-get install python-twisted python-twisted-names python-yaml python-scapy python-beautifulsoup + * Twisted + * Scapy + * txtorcon

-Once you have installed all the dependencies OONI tests can be run like so:: +They can be installed from the requirements.txt with:

- bin/ooniprobe path/to/test.py --cmd1 foo --cmd2 bar + pip install -r requirements.txt

+You are highly recommended to do so from inside of a virtual environment, since +pip does not download the packages via SSL and you will need to install it +system wide. + +This will require you to have installed virtualenv. + + apt-get install python-virtualenv + +To create a new virtual environment do + + virtualenv env + +Then install OONI with: + + pip install -r requirements.txt

Contents ******** @@ -45,7 +60,6 @@ Contents

oonib install - tutorial writing_tests api/* glossary diff --git a/nettests/bridge_reachability/echo.py b/nettests/bridge_reachability/echo.py index 611970e..0c20a3f 100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@ -50,8 +50,7 @@ class EchoTest(ScapyTest): description = 'A simple ICMP-8 test to see if a host is reachable.' version = '0.0.1' inputFile = ['file', 'f', None, 'File of list of IPs to ping'] - requirements = None - #report = Storage() + requiresRoot = True

optParameters = [ ['interface', 'i', None, 'Network interface to use'], @@ -73,7 +72,7 @@ class EchoTest(ScapyTest): if self.localOptions: log.debug("%s: local_options found" % self.name) for key, value in self.localOptions.items(): - log.debug("%s: setting self.%s = %s" % (key, value)) + log.debug("setting self.%s = %s" % (key, value)) setattr(self, key, value)

## xxx is this now .subOptions? diff --git a/nettests/core/echo.py b/nettests/core/echo.py deleted file mode 120000 index d9926cd..0000000 --- a/nettests/core/echo.py +++ /dev/null @@ -1 +0,0 @@ -../../ooni/bridget/tests/echo.py \ No newline at end of file diff --git a/ooni/__init__.py b/ooni/__init__.py index 659d4af..0c9f297 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -9,13 +9,6 @@ from . import runner from . import templates from . import utils

-# XXX below are legacy related modules -#from . import ooniprobe -#from . import plugoo -#from . import plugins - __all__ = ['oconfig', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', 'runner', 'templates', 'utils'] - # XXX below are legacy related modules - #'ooniprobe', 'plugoo', 'plugins'] diff --git a/ooni/custodiet.py b/ooni/custodiet.py deleted file mode 100755 index 8cbcfce..0000000 --- a/ooni/custodiet.py +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -# -# custodiet -# ********* -# -# "...quis custodiet ipsos custodes?" -# - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.) -# -# "'Hand me the Custodian,' Goodchild demands, inserting the waiflike -# robot into Bambara's opened navel. 'Providing conscience for those who -# have none.' Goodchild and the other Breen government agents disappear -# into the surrounding desert in a vehicle, kicking up cloud of white dust. -# Bambara awakens, and, patting the dust from his clothing, turns to -# greet a one-armed child. 'Hi, my name's Bambara; I'm a -# thirty-six-year-old Virgo and a former killer, who's hobbies include -# performing recreational autopsies, defecating, and drinking rum. I've -# recently been given a conscience, and would very much like to help you.' -# Cut to Bambara and the child, now with one of Bambara's arms, leaving -# a surgical clinic." -# - AeonFlux, "The Purge" (sometime in the late 90s) -# -# :copyright: (c) 2012 Isis Lovecruft -# :license: see LICENSE for more details. -# :version: 0.1.0-beta -# - -# ooniprobe.py imports -import sys -from signal import SIGTERM, signal -from pprint import pprint - -from twisted.python import usage -from twisted.internet import reactor -from twisted.plugin import getPlugins - -from zope.interface.verify import verifyObject -from zope.interface.exceptions import BrokenImplementation -from zope.interface.exceptions import BrokenMethodImplementation - -from ooni.bridget.tests import bridget -from ooni.bridget.utils import log, tests, work, reports -from ooni.bridget.utils.interface import ITest -from ooni.utils.logo import getlogo - -# runner.py imports -import os -import types -import time -import inspect -import yaml - -from twisted.internet import defer, reactor -from twisted.python import reflect, failure, usage -from twisted.python import log as tlog - -from twisted.trial import unittest -from twisted.trial.runner import TrialRunner, TestLoader -from twisted.trial.runner import isPackage, isTestCase, ErrorHolder -from twisted.trial.runner import filenameToModule, _importFromFile - -from ooni import nettest -from ooni.inputunit import InputUnitFactory -from ooni.nettest import InputTestSuite -from ooni.plugoo import tests as oonitests -from ooni.reporter import ReporterFactory -from ooni.utils import log, geodata, date -from ooni.utils.legacy import LegacyOONITest -from ooni.utils.legacy import start_legacy_test, adapt_legacy_test - - -__version__ = "0.1.0-beta" - - -#def retrieve_plugoo(): -# """ -# Get all the plugins that implement the ITest interface and get the data -# associated to them into a dict. -# """ -# interface = ITest -# d = {} -# error = False -# for p in getPlugins(interface, plugins): -# try: -# verifyObject(interface, p) -# d[p.shortName] = p -# except BrokenImplementation, bi: -# print "Plugin Broken" -# print bi -# error = True -# if error != False: -# print "Plugin Loaded!" -# return d -# -#plugoo = retrieve_plugoo() - -""" - -ai to watch over which tests to run - custodiet - - * runTest() or getPrefixMethodNames() to run the tests in order for each - test (esp. the tcp and icmp parts) to be oonicompat we should use the - test_icmp_ping API framework for those. - - * should handle calling - -tests to run: - echo - syn - fin - conn - tls - tor -need fakebridge - canary - -""" - -def runTest(test, options, global_options, reactor=reactor): - """ - Run an OONI probe test by name. - - @param test: a string specifying the test name as specified inside of - shortName. - - @param options: the local options to be passed to the test. - - @param global_options: the global options for OONI - """ - parallelism = int(global_options['parallelism']) - worker = work.Worker(parallelism, reactor=reactor) - test_class = plugoo[test].__class__ - report = reports.Report(test, global_options['output']) - - log_to_stdout = True - if global_options['quiet']: - log_to_stdout = False - - log.start(log_to_stdout, - global_options['log'], - global_options['verbosity']) - - resume = 0 - if not options: - options = {} - if 'resume' in options: - resume = options['resume'] - - test = test_class(options, global_options, report, reactor=reactor) - if test.tool: - test.runTool() - return True - - if test.ended: - print "Ending test" - return None - - wgen = work.WorkGenerator(test, - dict(options), - start=resume) - for x in wgen: - worker.push(x) - -class MainOptions(usage.Options): - tests = [bridget, ] - subCommands = [] - for test in tests: - print test - testopt = getattr(test, 'options') - subCommands.append([test, None, testopt, "Run the %s test" % test]) - - optFlags = [ - ['quiet', 'q', "Don't log to stdout"] - ] - - optParameters = [ - ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], - #['target-node', 't', 'localhost:31415', 'Select target node'], - ['output', 'o', 'bridge.log', "Specify output report file"], - ['reportfile', 'o', 'bridge.log', "Specify output log file"], - ['verbosity', 'v', 1, "Specify the logging level"], - ] - - def opt_version(self): - """ - Display OONI version and exit. - """ - print "OONI version:", __version__ - sys.exit(0) - - def __str__(self): - """ - Hack to get the sweet ascii art into the help output and replace the - strings "Commands" with "Tests". - """ - return getlogo() + '\n' + self.getSynopsis() + '\n' + \ - self.getUsage(width=None).replace("Commands:", "Tests:") - - - -def isTestCase(thing): - try: - return issubclass(thing, unittest.TestCase) - except TypeError: - return False - -def isLegacyTest(obj): - """ - Returns True if the test in question is written using the OONITest legacy - class. - We do this for backward compatibility of the OONIProbe API. - """ - try: - if issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest: - return True - else: - return False - except TypeError: - return False - -def processTest(obj, config): - """ - Process the parameters and :class:`twisted.python.usage.Options` of a - :class:`ooni.nettest.Nettest`. - - :param obj: - An uninstantiated old test, which should be a subclass of - :class:`ooni.plugoo.tests.OONITest`. - :param config: - A configured and instantiated :class:`twisted.python.usage.Options` - class. - """ - - inputFile = obj.inputFile - - if obj.optParameters or inputFile: - if not obj.optParameters: - obj.optParameters = [] - - if inputFile: - obj.optParameters.append(inputFile) - - class Options(usage.Options): - optParameters = obj.optParameters - - options = Options() - options.parseOptions(config['subArgs']) - obj.localOptions = options - - if inputFile: - obj.inputFile = options[inputFile[0]] - try: - tmp_obj = obj() - tmp_obj.getOptions() - except usage.UsageError: - options.opt_help() - - return obj - -def findTestClassesFromConfig(config): - """ - Takes as input the command line config parameters and returns the test - case classes. - If it detects that a certain test class is using the old OONIProbe format, - then it will adapt it to the new testing system. - - :param config: - A configured and instantiated :class:`twisted.python.usage.Options` - class. - :return: - A list of class objects found in a file or module given on the - commandline. - """ - - filename = config['test'] - classes = [] - - module = filenameToModule(filename) - for name, val in inspect.getmembers(module): - if isTestCase(val): - classes.append(processTest(val, config)) - elif isLegacyTest(val): - classes.append(adapt_legacy_test(val, config)) - return classes - -def makeTestCases(klass, tests, methodPrefix): - """ - Takes a class some tests and returns the test cases. methodPrefix is how - the test case functions should be prefixed with. - """ - - cases = [] - for test in tests: - cases.append(klass(methodPrefix+test)) - return cases - -def loadTestsAndOptions(classes, config): - """ - Takes a list of classes and returns their testcases and options. - Legacy tests will be adapted. - """ - - methodPrefix = 'test' - suiteFactory = InputTestSuite - options = [] - testCases = [] - names = [] - - _old_klass_type = LegacyOONITest - - for klass in classes: - if isinstance(klass, _old_klass_type): - try: - cases = start_legacy_test(klass) - #cases.callback() - if cases: - print cases - return [], [] - testCases.append(cases) - except Exception, e: - log.err(e) - else: - try: - opts = klass.local_options - options.append(opts) - except AttributeError, ae: - options.append([]) - log.err(ae) - elif not isinstance(klass, _old_klass_type): - tests = reflect.prefixedMethodNames(klass, methodPrefix) - if tests: - cases = makeTestCases(klass, tests, methodPrefix) - testCases.append(cases) - try: - k = klass() - opts = k.getOptions() - options.append(opts) - except AttributeError, ae: - options.append([]) - log.err(ae) - else: - try: - raise RuntimeError, "Class is some strange type!" - except RuntimeError, re: - log.err(re) - - return testCases, options - -class ORunner(object): - """ - This is a specialized runner used by the ooniprobe command line tool. - I am responsible for reading the inputs from the test files and splitting - them in input units. I also create all the report instances required to run - the tests. - """ - def __init__(self, cases, options=None, config=None, *arg, **kw): - self.baseSuite = InputTestSuite - self.cases = cases - self.options = options - - try: - assert len(options) != 0, "Length of options is zero!" - except AssertionError, ae: - self.inputs = [] - log.err(ae) - else: - try: - first = options.pop(0) - except: - first = {} - if 'inputs' in first: - self.inputs = options['inputs'] - else: - log.msg("Could not find inputs!") - log.msg("options[0] = %s" % first) - self.inputs = [None] - - try: - reportFile = open(config['reportfile'], 'a+') - except: - filename = 'report_'+date.timestamp()+'.yaml' - reportFile = open(filename, 'a+') - self.reporterFactory = ReporterFactory(reportFile, - testSuite=self.baseSuite(self.cases)) - - def runWithInputUnit(self, inputUnit): - idx = 0 - result = self.reporterFactory.create() - - for inputs in inputUnit: - result.reporterFactory = self.reporterFactory - - suite = self.baseSuite(self.cases) - suite.input = inputs - suite(result, idx) - - # XXX refactor all of this index bullshit to avoid having to pass - # this index around. Probably what I want to do is go and make - # changes to report to support the concept of having multiple runs - # of the same test. - # We currently need to do this addition in order to get the number - # of times the test cases that have run inside of the test suite. - idx += (suite._idx - idx) - - result.done() - - def run(self): - self.reporterFactory.options = self.options - for inputUnit in InputUnitFactory(self.inputs): - self.runWithInputUnit(inputUnit) - -if __name__ == "__main__": - config = Options() - config.parseOptions() - - if not config.subCommand: - config.opt_help() - signal(SIGTERM) - #sys.exit(1) - - runTest(config.subCommand, config.subOptions, config) - reactor.run() diff --git a/ooni/lib/Makefile b/ooni/lib/Makefile deleted file mode 100644 index c40b8d2..0000000 --- a/ooni/lib/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -all: txtorcon - -txtraceroute: - echo "Processing dependency txtraceroute..." - git clone https://github.com/hellais/txtraceroute.git txtraceroute.git - mv txtraceroute.git/txtraceroute.py txtraceroute.py - rm -rf txtraceroute.git - -txtorcon: - echo "Processing dependency txtorcon..." - git clone https://github.com/meejah/txtorcon.git txtorcon.git - mv txtorcon.git/txtorcon txtorcon - rm -rf txtorcon.git - -clean: - rm -rf txtorcon -# rm -rf txtraceroute.py -# rm -rf txscapy.py - -cleanall: - rm -rf txtorcon - rm -rf txtraceroute.py - rm -rf txscapy.py - -txscapy: - echo "Processing dependency txscapy" - git clone https://github.com/hellais/txscapy.git txscapy.git - mv txscapy.git/txscapy.py txscapy.py - rm -rf txscapy.git - -#rfc3339: -# echo "Processing RFC3339 dependency" -# hg clone https://bitbucket.org/henry/rfc3339 rfc3339 -# mv rfc3339/rfc3339.py rfc3339.py -# rm -rf rfc3339 - diff --git a/ooni/nodes.py b/ooni/nodes.py new file mode 100644 index 0000000..155f183 --- /dev/null +++ b/ooni/nodes.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 +""" + nodes + ***** + + This contains all the code related to Nodes + both network and code execution. + + :copyright: (c) 2012 by Arturo Filastò, Isis Lovecruft + :license: see LICENSE for more details. + +""" + +import os +from binascii import hexlify + +try: + import paramiko +except: + print "Error: module paramiko is not installed." +from pprint import pprint +import sys +import socks +import xmlrpclib + +class Node(object): + def __init__(self, address, port): + self.address = address + self.port = port + +class LocalNode(object): + def __init__(self): + pass + +""" +[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5") +[]: node_socket = node.wrap_socket() +""" +class NetworkNode(Node): + def __init__(self, address, port, node_type="SOCKS5", auth_creds=None): + self.node = Node(address,port) + + # XXX support for multiple types + # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...) + self.node_type = node_type + # type-specific authentication credentials + self.auth_creds = auth_creds + + def _get_socksipy_socket(self, proxy_type, auth_creds): + import socks + s = socks.socksocket() + # auth_creds[0] -> username + # auth_creds[1] -> password + s.setproxy(proxy_type, self.node.address, self.node.port, + self.auth_creds[0], self.auth_creds[1]) + return s + + def _get_socket_wrapper(self): + if (self.node_type.startswith("SOCKS")): # SOCKS proxies + if (self.node_type != "SOCKS5"): + proxy_type = socks.PROXY_TYPE_SOCKS5 + elif (self.node_type != "SOCKS4"): + proxy_type = socks.PROXY_TYPE_SOCKS4 + else: + print "We don't know this proxy type." + sys.exit(1) + + return self._get_socksipy_socket(proxy_type) + elif (self.node_type == "HTTP"): # HTTP proxies + return self._get_socksipy_socket(PROXY_TYPE_HTTP) + else: # Unknown proxies + print "We don't know this proxy type." + sys.exit(1) + + def wrap_socket(self): + return self._get_socket_wrapper() + +class CodeExecNode(Node): + def __init__(self, address, port, node_type, auth_creds): + self.node = Node(address,port) + + # node type (SSH proxy, etc.) + self.node_type = node_type + # type-specific authentication credentials + self.auth_creds = auth_creds + + def add_unit(self): + pass + + def get_status(self): + pass + +class PlanetLab(CodeExecNode): + def __init__(self, address, auth_creds, ooni): + self.auth_creds = auth_creds + + self.config = ooni.utils.config + self.logger = ooni.logger + self.name = "PlanetLab" + + def _api_auth(self): + api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/') + auth = {} + ## should be changed to separate node.conf file + auth['Username'] = self.config.main.pl_username + auth['AuthString'] = self.config.main.pl_password + auth['AuthMethod'] = "password" + authorized = api_server.AuthCheck(auth) + + if authorized: + print 'We are authorized!' + return auth + else: + print 'Authorization failed. Please check your settings for pl_username and pl_password in the ooni-probe.conf file.' + + def _search_for_nodes(self, node_filter=None): + api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True) + node_filter = {'hostname': '*.cert.org.cn'} + return_fields = ['hostname', 'site_id'] + all_nodes = api_server.GetNodes(self.api_auth(), node_filter, boot_state_filter) + pprint(all_nodes) + return all_nodes + + def _add_nodes_to_slice(self): + api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True) + all_nodes = self.search_for_nodes() + for node in all_nodes: + api_server.AddNode(self.api_auth(), node['site_id'], all_nodes) + print 'Adding nodes %s' % node['hostname'] + + def _auth_login(slicename, machinename): + """Attempt to authenticate to the given PL node, slicename and + machinename, using any of the private keys in ~/.ssh/ """ + + agent = paramiko.Agent() + agent_keys = agent.get_keys() + if len(agent_keys) == 0: + return + + for key in agent_keys: + print 'Trying ssh-agent key %s' % hexlify(key.get_fingerprint()), + try: + paramiko.transport.auth_publickey(machinename, slicename) + print 'Public key authentication to PlanetLab node %s successful.' % machinename, + return + except paramiko.SSHException: + print 'Public key authentication to PlanetLab node %s failed.' % machinename, + + def _get_command(): + pass + + def ssh_and_run_(slicename, machinename, command): + """Attempt to make a standard OpenSSH client to PL node, and run + commands from a .conf file.""" + + ## needs a way to specify 'ssh -l <slicename> <machinename>' + ## with public key authentication. + + command = PlanetLab.get_command() + + client = paramiko.SSHClient() + client.load_system_host_keys() + client.connect(machinename) + + stdin, stdout, stderr = client.exec_command(command) + + def send_files_to_node(directory, files): + """Attempt to rsync a tree to the PL node.""" + pass + + def add_unit(): + pass + + def get_status(): + pass diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 1988652..6def1df 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -32,12 +32,12 @@ from ooni.utils import log

class Options(usage.Options, app.ReactorSelectionMixin): - synopsis = """%s [options] [[file|package|module|TestCase|testmethod]...] + synopsis = """%s [options] [path to test].py """ % (os.path.basename(sys.argv[0]),)

longdesc = ("ooniprobe loads and executes a suite or a set of suites of" - "network tests. These are loaded from modules, packages and" - "files listed on the command line") + " network tests. These are loaded from modules, packages and" + " files listed on the command line")

optFlags = [["help", "h"], ['debug-stacktraces', 'B', @@ -47,8 +47,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): optParameters = [ ["reportfile", "o", None, "report file name"], ["logfile", "l", None, "log file name"], - ['temp-directory', None, '_ooni_temp', - 'Path to use as working directory for tests.'] ]

compData = usage.Completions( @@ -98,12 +96,11 @@ def run(): if config['debug-stacktraces']: defer.setDebugging(True)

- log.start(config['logfile']) - classes = runner.findTestClassesFromConfig(config) casesList, options = runner.loadTestsAndOptions(classes, config)

for idx, cases in enumerate(casesList): orunner = runner.ORunner(cases, options[idx], config) + log.start(config['logfile']) orunner.run()

diff --git a/ooni/plugoo/__init__.py b/ooni/plugoo/__init__.py deleted file mode 100644 index f3a49e9..0000000 --- a/ooni/plugoo/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: UTF-8 -""" - plugoo - ****** - - This contains all of the "goo" necessary for creating - ooni-probe plugoonies. - - :copyright: (c) 2012 by Arturo Filastò. - :license: see LICENSE for more details. - -""" - -__all__ = ['assets', 'nodes', 'reports', 'tests'] - -import os -from datetime import datetime -import yaml - -import logging -import itertools - -def gen_headers(self, options="common"): - """ - Returns a set of headers to be used when generating - HTTP requests. - - :options specify what rules should be used for - generating the headers. - "common": choose a very common header set (default) - "random": make the headers random - """ - if options == "common": - headers = [('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), - ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'), - ('Accept-Encoding', 'gzip,deflate,sdch'), - ('Accept-Language', 'en,en-US;q=0.8,it;q=0.6'), - ('Cache-Control', 'max-age=0') - ('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11')] - elif options == "random": - # XXX not implemented - return False - else: - print "Error, unrecognized header generation options.." - return False - - return headers diff --git a/ooni/plugoo/assets.py b/ooni/plugoo/assets.py deleted file mode 100644 index 205d60d..0000000 --- a/ooni/plugoo/assets.py +++ /dev/null @@ -1,62 +0,0 @@ -class Asset: - """ - This is an ooni-probe asset. It is a python - iterator object, allowing it to be efficiently looped. - To create your own custom asset your should subclass this - and override the next_asset method and the len method for - computing the length of the asset. - """ - def __init__(self, file=None, *args, **argv): - self.fh = None - if file: - self.name = file - self.fh = open(file, 'r') - self.eof = False - - def __iter__(self): - return self - - def len(self): - """ - Returns the length of the asset - """ - for i, l in enumerate(self.fh): - pass - # rewind the file - self.fh.seek(0) - return i + 1 - - def parse_line(self, line): - """ - Override this method if you need line - by line parsing of an Asset. - """ - return line.replace('\n','') - - def next_asset(self): - """ - Return the next asset. - """ - # XXX this is really written with my feet. - # clean me up please... - line = self.fh.readline() - if line: - parsed_line = self.parse_line(line) - if parsed_line: - return parsed_line - else: - self.fh.seek(0) - raise StopIteration - - def next(self): - try: - return self.next_asset() - except: - raise StopIteration - -class MissingAssetException(Exception): - """Raised when an Asset necessary for running the Test is missing.""" - def __init__(self, error_message): - print error_message - import sys - return sys.exit() diff --git a/ooni/plugoo/interface.py b/ooni/plugoo/interface.py deleted file mode 100644 index 6dc83a0..0000000 --- a/ooni/plugoo/interface.py +++ /dev/null @@ -1,56 +0,0 @@ -from zope.interface import implements, Interface, Attribute - -class ITest(Interface): - """ - This interface represents an OONI test. It fires a deferred on completion. - """ - - shortName = Attribute("""A short user facing description for this test""") - description = Attribute("""A string containing a longer description for the test""") - - requirements = Attribute("""What is required to run this this test, for example raw socket access or UDP or TCP""") - - options = Attribute("""These are the arguments to be passed to the test for it's execution""") - - blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""") - - def control(experiment_result, args): - """ - @param experiment_result: The result returned by the experiment method. - - @param args: the keys of this dict are the names of the assets passed in - from load_assets. The value is one item of the asset. - - Must return a dict containing what should be written to the report. - Anything returned by control ends up inside of the YAMLOONI report. - """ - - def experiment(args): - """ - Perform all the operations that are necessary to running a test. - - @param args: the keys of this dict are the names of the assets passed in - from load_assets. The value is one item of the asset. - - Must return a dict containing the values to be passed to control. - """ - - def load_assets(): - """ - Load the assets that should be passed to the Test. These are the inputs - to the OONI test. - Must return a dict that has as keys the asset names and values the - asset contents. - If the test does not have any assets it should return an empty dict. - """ - - def end(): - """ - This can be called at any time to terminate the execution of all of - these test instances. - - What this means is that no more test instances with new parameters will - be created. A report will be written. - """ - - diff --git a/ooni/plugoo/nodes.py b/ooni/plugoo/nodes.py deleted file mode 100644 index 155f183..0000000 --- a/ooni/plugoo/nodes.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -""" - nodes - ***** - - This contains all the code related to Nodes - both network and code execution. - - :copyright: (c) 2012 by Arturo Filastò, Isis Lovecruft - :license: see LICENSE for more details. - -""" - -import os -from binascii import hexlify - -try: - import paramiko -except: - print "Error: module paramiko is not installed." -from pprint import pprint -import sys -import socks -import xmlrpclib - -class Node(object): - def __init__(self, address, port): - self.address = address - self.port = port - -class LocalNode(object): - def __init__(self): - pass - -""" -[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5") -[]: node_socket = node.wrap_socket() -""" -class NetworkNode(Node): - def __init__(self, address, port, node_type="SOCKS5", auth_creds=None): - self.node = Node(address,port) - - # XXX support for multiple types - # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...) - self.node_type = node_type - # type-specific authentication credentials - self.auth_creds = auth_creds - - def _get_socksipy_socket(self, proxy_type, auth_creds): - import socks - s = socks.socksocket() - # auth_creds[0] -> username - # auth_creds[1] -> password - s.setproxy(proxy_type, self.node.address, self.node.port, - self.auth_creds[0], self.auth_creds[1]) - return s - - def _get_socket_wrapper(self): - if (self.node_type.startswith("SOCKS")): # SOCKS proxies - if (self.node_type != "SOCKS5"): - proxy_type = socks.PROXY_TYPE_SOCKS5 - elif (self.node_type != "SOCKS4"): - proxy_type = socks.PROXY_TYPE_SOCKS4 - else: - print "We don't know this proxy type." - sys.exit(1) - - return self._get_socksipy_socket(proxy_type) - elif (self.node_type == "HTTP"): # HTTP proxies - return self._get_socksipy_socket(PROXY_TYPE_HTTP) - else: # Unknown proxies - print "We don't know this proxy type." - sys.exit(1) - - def wrap_socket(self): - return self._get_socket_wrapper() - -class CodeExecNode(Node): - def __init__(self, address, port, node_type, auth_creds): - self.node = Node(address,port) - - # node type (SSH proxy, etc.) - self.node_type = node_type - # type-specific authentication credentials - self.auth_creds = auth_creds - - def add_unit(self): - pass - - def get_status(self): - pass - -class PlanetLab(CodeExecNode): - def __init__(self, address, auth_creds, ooni): - self.auth_creds = auth_creds - - self.config = ooni.utils.config - self.logger = ooni.logger - self.name = "PlanetLab" - - def _api_auth(self): - api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/') - auth = {} - ## should be changed to separate node.conf file - auth['Username'] = self.config.main.pl_username - auth['AuthString'] = self.config.main.pl_password - auth['AuthMethod'] = "password" - authorized = api_server.AuthCheck(auth) - - if authorized: - print 'We are authorized!' - return auth - else: - print 'Authorization failed. Please check your settings for pl_username and pl_password in the ooni-probe.conf file.' - - def _search_for_nodes(self, node_filter=None): - api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True) - node_filter = {'hostname': '*.cert.org.cn'} - return_fields = ['hostname', 'site_id'] - all_nodes = api_server.GetNodes(self.api_auth(), node_filter, boot_state_filter) - pprint(all_nodes) - return all_nodes - - def _add_nodes_to_slice(self): - api_server = xmlrpclib.ServerProxy('https://www.planet-lab.org/PLCAPI/', allow_none=True) - all_nodes = self.search_for_nodes() - for node in all_nodes: - api_server.AddNode(self.api_auth(), node['site_id'], all_nodes) - print 'Adding nodes %s' % node['hostname'] - - def _auth_login(slicename, machinename): - """Attempt to authenticate to the given PL node, slicename and - machinename, using any of the private keys in ~/.ssh/ """ - - agent = paramiko.Agent() - agent_keys = agent.get_keys() - if len(agent_keys) == 0: - return - - for key in agent_keys: - print 'Trying ssh-agent key %s' % hexlify(key.get_fingerprint()), - try: - paramiko.transport.auth_publickey(machinename, slicename) - print 'Public key authentication to PlanetLab node %s successful.' % machinename, - return - except paramiko.SSHException: - print 'Public key authentication to PlanetLab node %s failed.' % machinename, - - def _get_command(): - pass - - def ssh_and_run_(slicename, machinename, command): - """Attempt to make a standard OpenSSH client to PL node, and run - commands from a .conf file.""" - - ## needs a way to specify 'ssh -l <slicename> <machinename>' - ## with public key authentication. - - command = PlanetLab.get_command() - - client = paramiko.SSHClient() - client.load_system_host_keys() - client.connect(machinename) - - stdin, stdout, stderr = client.exec_command(command) - - def send_files_to_node(directory, files): - """Attempt to rsync a tree to the PL node.""" - pass - - def add_unit(): - pass - - def get_status(): - pass diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py deleted file mode 100644 index 1bfdac0..0000000 --- a/ooni/plugoo/reports.py +++ /dev/null @@ -1,145 +0,0 @@ -from __future__ import with_statement - -import os -import yaml - -import itertools -from ooni.utils import log, date, net - -class Report: - """This is the ooni-probe reporting mechanism. It allows - reporting to multiple destinations and file formats. - - :scp the string of <host>:<port> of an ssh server - - :yaml the filename of a the yaml file to write - - :file the filename of a simple txt file to write - - :tcp the <host>:<port> of a TCP server that will just listen for - inbound connection and accept a stream of data (think of it - as a `nc -l -p <port> > filename.txt`) - """ - def __init__(self, testname=None, file="report.log", - scp=None, - tcp=None): - - self.testname = testname - self.file = file - self.tcp = tcp - self.scp = scp - #self.config = ooni.config.report - - #if self.config.timestamp: - # tmp = self.file.split('.') - # self.file = '.'.join(tmp[:-1]) + "-" + \ - # datetime.now().isoformat('-') + '.' + \ - # tmp[-1] - # print self.file - - self.scp = None - self.write_header() - - def write_header(self): - pretty_date = date.pretty_date() - header = "# OONI Probe Report for Test %s\n" % self.testname - header += "# %s\n\n" % pretty_date - self._write_to_report(header) - # XXX replace this with something proper - address = net.getClientAddress() - test_details = {'start_time': str(date.now()), - 'asn': address['asn'], - 'test_name': self.testname, - 'addr': address['ip']} - self(test_details) - - def _write_to_report(self, dump): - reports = [] - - if self.file: - reports.append("file") - - if self.tcp: - reports.append("tcp") - - if self.scp: - reports.append("scp") - - #XXX make this non blocking - for report in reports: - self.send_report(dump, report) - - def __call__(self, data): - """ - This should be invoked every time you wish to write some - data to the reporting system - """ - dump = yaml.dump([data]) - self._write_to_report(dump) - - def file_report(self, data): - """ - This reports to a file in YAML format - """ - with open(self.file, 'a+') as f: - f.write(data) - - def send_report(self, data, type): - """ - This sends the report using the - specified type. - """ - #print "Reporting %s to %s" % (data, type) - log.msg("Reporting to %s" % type) - getattr(self, type+"_report").__call__(data) - -class NewReport(object): - filename = 'report.log' - startTime = None - endTime = None - testName = None - ipAddr = None - asnAddr = None - - def _open(): - self.fp = open(self.filename, 'a+') - - @property - def header(): - pretty_date = date.pretty_date() - report_header = "# OONI Probe Report for Test %s\n" % self.testName - report_header += "# %s\n\n" % pretty_date - test_details = {'start_time': self.startTime, - 'asn': asnAddr, - 'test_name': self.testName, - 'addr': ipAddr} - report_header += yaml.dump([test_details]) - return report_header - - def create(): - """ - Create a new report by writing it's header. - """ - self.fp = open(self.filename, 'w+') - self.fp.write(self.header) - - def exists(): - """ - Returns False if the file does not exists. - """ - return os.path.exists(self.filename) - - def write(data): - """ - Write a report to the file. - - :data: python data structure to be written to report. - """ - if not self.exists(): - self.create() - else: - self._open() - yaml_encoded_data = yaml.dump([data]) - self.fp.write(yaml_encoded_data) - self.fp.close() - diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py deleted file mode 100644 index 653dd67..0000000 --- a/ooni/plugoo/tests.py +++ /dev/null @@ -1,142 +0,0 @@ -import os -import yaml -from zope.interface import Interface, Attribute - -import logging -import itertools -from twisted.internet import reactor, defer, threads -## XXX why is this imported and not used? -from twisted.python import failure - -from ooni.utils import log, date -from ooni.plugoo import assets, work -from ooni.plugoo.reports import Report -from ooni.plugoo.interface import ITest - - -class OONITest(object): - """ - This is the base class for writing OONI Tests. - - It should be used in conjunction with the ITest Interface. It allows the - developer to benefit from OONIs reporting system and command line argument - parsing system. - """ - name = "oonitest" - # By default we set this to False, meaning that we don't block - blocking = False - reactor = reactor - tool = False - ended = False - - def __init__(self, local_options, global_options, report, ooninet=None, - reactor=reactor): - # These are the options that are read through the tests suboptions - self.local_options = local_options - # These are the options global to all of OONI - self.global_options = global_options - self.report = report - #self.ooninet = ooninet - self.reactor = reactor - self.result = {} - self.initialize() - self.assets = self.load_assets() - - def initialize(self): - """ - Override this method if you are interested in having some extra - behavior when your test class is instantiated. - """ - pass - - def load_assets(self): - """ - This method should be overriden by the test writer to provide the - logic for loading their assets. - """ - return {} - - def __repr__(self): - return "<OONITest %s %s %s>" % (self.local_options, - self.global_options, - self.assets) - - def end(self): - """ - State that the current test should finish. - """ - self.ended = True - - def finished(self, return_value): - """ - The Test has finished running, we must now calculate the test runtime - and add all time data to the report. - """ - #self.ooninet.report(result) - self.end_time = date.now() - result = self.result - result['start_time'] = str(self.start_time) - result['end_time'] = str(self.end_time) - result['run_time'] = str(self.end_time - self.start_time) - result['return_value'] = return_value - log.msg("FINISHED %s" % result) - self.report(result) - return result - - def _do_experiment(self, args): - """ - A wrapper around the launch of experiment. - If we are running a blocking test experiment will be run in a thread if - not we expect it to return a Deferred. - - @param args: the asset line(s) that we are working on. - - returns a deferred. - """ - if self.blocking: - self.d = threads.deferToThread(self.experiment, args) - else: - self.d = self.experiment(args) - - self.d.addCallback(self.control, args) - self.d.addCallback(self.finished) - self.d.addErrback(self.finished) - return self.d - - def control(self, result, args): - """ - Run the control. - - @param result: what was returned by experiment. - - @param args: the asset(s) lines that we are working on. - """ - log.msg("Doing control") - return result - - def experiment(self, args): - """ - Run the experiment. This sample implementation returns a deferred, - making it a non-blocking test. - - @param args: the asset(s) lines that we are working on. - """ - log.msg("Doing experiment") - d = defer.Deferred() - return d - - def startTest(self, args): - """ - This method is invoked by the worker to start the test with one line of - the asset file. - - @param args: the asset(s) lines that we are working on. - """ - self.start_time = date.now() - - if self.shortName: - log.msg("Starting test %s" % self.shortName) - else: - log.msg("Starting test %s" % self.__class__) - - return self._do_experiment(args) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py deleted file mode 100644 index db88fbf..0000000 --- a/ooni/plugoo/work.py +++ /dev/null @@ -1,148 +0,0 @@ -# -*- coding: UTF-8 -""" - work.py - ********** - - This contains all code related to generating - Units of Work and processing it. - - :copyright: (c) 2012 by Arturo Filastò. - :license: see LICENSE for more details. - -""" -import itertools -import yaml -from datetime import datetime - -from zope.interface import Interface, Attribute - -from twisted.python import failure -from twisted.internet import reactor, defer - -class Worker(object): - """ - This is the core of OONI. It takes as input Work Units and - runs them concurrently. - """ - def __init__(self, maxconcurrent=10, reactor=reactor): - """ - @param maxconcurrent: how many test instances should be run - concurrently. - """ - self.reactor = reactor - self.maxconcurrent = maxconcurrent - self._running = 0 - self._queued = [] - - def _run(self, r): - """ - Check if we should start another test because we are below maximum - concurrency. - - This function is called every time a test finishes running. - - @param r: the return value of a previous test. - """ - if self._running > 0: - self._running -= 1 - - if self._running < self.maxconcurrent and self._queued: - workunit, d = self._queued.pop(0) - asset, test, idx = workunit - while test.ended and workunit: - try: - workunit, d = self._queued.pop(0) - asset, test, idx = workunit - except: - workunit = None - - if not test.ended: - self._running += 1 - actuald = test.startTest(asset).addBoth(self._run) - - if isinstance(r, failure.Failure): - # XXX probably we should be doing something to retry test running - r.trap() - - if self._running == 0 and not self._queued: - self.reactor.stop() - - return r - - def push(self, workunit): - """ - Add a test to the test queue and run it if we are not maxed out on - concurrency. - - @param workunit: a tuple containing the (asset, test, idx), where asset - is the line of the asset(s) we are working on, test - is an instantiated test and idx is the index we are - currently at. - """ - if self._running < self.maxconcurrent: - asset, test, idx = workunit - if not test.ended: - self._running += 1 - return test.startTest(asset).addBoth(self._run) - - d = defer.Deferred() - self._queued.append((workunit, d)) - return d - -class WorkGenerator(object): - """ - Factory responsible for creating units of work. - - This shall be run on the machine running OONI-cli. The returned WorkUnits - can either be run locally or on a remote OONI Node or Network Node. - """ - size = 10 - - def __init__(self, test, arguments=None, start=None): - self.Test = test - - if self.Test.assets and self.Test.assets.values()[0]: - self.assetGenerator = itertools.product(*self.Test.assets.values()) - else: - self.assetGenerator = None - - self.assetNames = self.Test.assets.keys() - - self.idx = 0 - self.end = False - if start: - self.skip(start) - - def __iter__(self): - return self - - def skip(self, start): - """ - Skip the first x number of lines of the asset. - - @param start: int how many items we should skip. - """ - for j in xrange(0, start-1): - for i in xrange(0, self.size): - self.assetGenerator.next() - self.idx += 1 - - def next(self): - if self.end: - raise StopIteration - - if not self.assetGenerator: - self.end = True - return ({}, self.Test, self.idx) - - try: - asset = self.assetGenerator.next() - ret = {} - for i, v in enumerate(asset): - ret[self.assetNames[i]] = v - except StopIteration: - raise StopIteration - - self.idx += 1 - return (ret, self.Test, self.idx) - diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py deleted file mode 100644 index 569c382..0000000 --- a/ooni/protocols/http.py +++ /dev/null @@ -1,141 +0,0 @@ -import random -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import protocol, defer -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset -from ooni.utils import log - -useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), - ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), - ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), - ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] - -class BodyReceiver(protocol.Protocol): - def __init__(self, finished): - self.finished = finished - self.data = "" - - def dataReceived(self, bytes): - self.data += bytes - - def connectionLost(self, reason): - self.finished.callback(self.data) - -from twisted.web.http_headers import Headers -class HTTPTest(OONITest): - """ - A utility class for dealing with HTTP based testing. It provides methods to - be overriden for dealing with HTTP based testing. - The main functions to look at are processResponseBody and - processResponseHeader that are invoked once the headers have been received - and once the request body has been received. - """ - randomize_ua = True - follow_redirects = False - - def initialize(self): - from twisted.web.client import Agent - import yaml - - self.agent = Agent(self.reactor) - if self.follow_redirects: - from twisted.web.client import RedirectAgent - self.agent = RedirectAgent(self.agent) - - self.request = {} - self.response = {} - - def _processResponseBody(self, data): - self.response['body'] = data - self.result['response'] = self.response - self.processResponseBody(data) - - def processResponseBody(self, data): - """ - This should handle all the response body smushing for getting it ready - to be passed onto the control. - - @param data: The content of the body returned. - """ - pass - - def processResponseHeaders(self, headers): - """ - This should take care of dealing with the returned HTTP headers. - - @param headers: The content of the returned headers. - """ - pass - - def processRedirect(self, location): - """ - Handle a redirection via a 3XX HTTP status code. - - @param location: the url that is being redirected to. - """ - pass - - def doRequest(self, url): - d = self.build_request(url) - def finished(data): - return data - - d.addCallback(self._cbResponse) - d.addCallback(finished) - return d - - def experiment(self, args): - log.msg("Running experiment") - url = self.local_options['url'] if 'url' not in args else args['url'] - - d = self.doRequest(url) - return d - - def _cbResponse(self, response): - self.response['headers'] = list(response.headers.getAllRawHeaders()) - self.response['code'] = response.code - self.response['length'] = response.length - self.response['version'] = response.length - - if str(self.response['code']).startswith('3'): - self.processRedirect(response.headers.getRawHeaders('Location')[0]) - self.processResponseHeaders(self.response['headers']) - #self.result['response'] = self.response - - finished = defer.Deferred() - response.deliverBody(BodyReceiver(finished)) - finished.addCallback(self._processResponseBody) - - def randomize_useragent(self): - user_agent = random.choice(useragents) - self.request['headers']['User-Agent'] = [user_agent] - - def build_request(self, url, method="GET", headers=None, body=None): - self.request['method'] = method - self.request['url'] = url - self.request['headers'] = headers if headers else {} - self.request['body'] = body - if self.randomize_ua: - self.randomize_useragent() - - self.result['request'] = self.request - self.result['url'] = url - return self.agent.request(self.request['method'], self.request['url'], - Headers(self.request['headers']), - self.request['body']) - - def load_assets(self): - if self.local_options: - return {'url': Asset(self.local_options['asset'])} - else: - return {} - diff --git a/ooni/protocols/scapyproto.py b/ooni/protocols/scapyproto.py deleted file mode 100644 index 4166146..0000000 --- a/ooni/protocols/scapyproto.py +++ /dev/null @@ -1,55 +0,0 @@ -import random -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import protocol, defer -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset -from ooni.utils import log - -from ooni.lib.txscapy import txsr, txsend - -class ScapyTest(OONITest): - """ - A utility class for writing scapy driven OONI tests. - """ - - receive = True - timeout = None - pcapfile = 'scapytest.pcap' - def initialize(self, reactor=None): - - if not self.reactor: - from twisted.internet import reactor - self.reactor = reactor - - self.request = {} - self.response = {} - - def experiment(self, args): - log.msg("Running experiment") - if self.receive: - log.msg("Sending and receiving packets.") - d = txsr(self.build_packets(), pcapfile=self.pcapfile, - timeout=self.timeout) - else: - log.msg("Sending packets.") - d = txsend(self.build_packets()) - - def finished(data): - log.msg("Finished sending") - return data - - d.addCallback(finished) - return d - - def build_packets(self): - """ - Override this method to build scapy packets. - """ - from scapy.all import IP, TCP - return IP()/TCP() - - def load_assets(self): - return {} - diff --git a/ooni/reporter.py b/ooni/reporter.py index ad7956d..a02e5a9 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -273,18 +273,14 @@ class OONIReporter(OReporter):

idx = self.getTestIndex(test)

- self._tests[idx]['last_time'] = self._getTime() - self._tests[idx]['test_started'] - # This is here for allowing reporting of legacy tests. - # XXX In the future this should be removed. - try: - report = list(test.legacy_report) - log.debug("Set the report to be a list") - except: - # XXX I put a dict() here so that the object is re-instantiated and I - # actually end up with the report I want. This could either be a - # python bug or a yaml bug. - report = dict(test.report) - log.debug("Set the report to be a dict") + self._tests[idx]['last_time'] = self._getTime() - \ + self._tests[idx]['test_started'] + + # XXX I put a dict() here so that the object is re-instantiated and I + # actually end up with the report I want. This could either be a + # python bug or a yaml bug. + report = dict(test.report) + log.debug("Set the report to be a dict")

log.debug("Adding to report %s" % report) self._tests[idx]['report'] = report diff --git a/ooni/runner.py b/ooni/runner.py index fa50840..102ff29 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -20,25 +20,9 @@ from twisted.trial.runner import filenameToModule from ooni.inputunit import InputUnitFactory from ooni.nettest import InputTestSuite

-from ooni.plugoo import tests as oonitests - from ooni.reporter import ReporterFactory from ooni.utils import log, date

-from ooni.utils.legacy import LegacyOONITest -from ooni.utils.legacy import start_legacy_test, adapt_legacy_test - -def isLegacyTest(obj): - """ - Returns True if the test in question is written using the OONITest legacy - class. - We do this for backward compatibility of the OONIProbe API. - """ - try: - return issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest - except TypeError: - return False - def processTest(obj, config): """ Process the parameters and :class:`twisted.python.usage.Options` of a @@ -115,9 +99,6 @@ def findTestClassesFromConfig(config): if isTestCase(val): log.debug("Detected TestCase %s" % val) classes.append(processTest(val, config)) - elif isLegacyTest(val): - log.debug("Detected Legacy Test %s" % val) - classes.append(adapt_legacy_test(val, config)) return classes

def makeTestCases(klass, tests, method_prefix): @@ -141,38 +122,18 @@ def loadTestsAndOptions(classes, config): options = [] test_cases = []

- _old_klass_type = LegacyOONITest - for klass in classes: - if isinstance(klass, _old_klass_type): - try: - cases = start_legacy_test(klass) - if cases: - log.debug("Processing cases") - log.debug(str(cases)) - return [], [] - test_cases.append(cases) - except Exception, e: - log.err(e) - else: - try: - opts = klass.local_options - options.append(opts) - except AttributeError, ae: - options.append([]) - log.err(ae) - else: - tests = reflect.prefixedMethodNames(klass, method_prefix) - if tests: - cases = makeTestCases(klass, tests, method_prefix) - test_cases.append(cases) - try: - k = klass() - opts = k.getOptions() - options.append(opts) - except AttributeError, ae: - options.append([]) - log.err(ae) + tests = reflect.prefixedMethodNames(klass, method_prefix) + if tests: + cases = makeTestCases(klass, tests, method_prefix) + test_cases.append(cases) + try: + k = klass() + opts = k.getOptions() + options.append(opts) + except AttributeError, ae: + options.append([]) + log.err(ae)

return test_cases, options

diff --git a/ooni/scaffolding.py b/ooni/scaffolding.py deleted file mode 100644 index dffe342..0000000 --- a/ooni/scaffolding.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python - -""" -This script should be used for creating the scaffolding for a test. -""" -from __future__ import print_function -import os -import sys -from ooni.utils import log - -print("!!!! This test writing strategy is now deprecated !!!") -print("visit: https://ooni.readthedocs.org/en/latest/writing_tests.html " - "for new instructions") -sys.exit(1) - -test_template = """""" -This is a self genrated test created by scaffolding.py. -you will need to fill it up with all your necessities. -Safe hacking :). -""" -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset -from ooni.utils import log - -class %(testShortname)sArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['resume', 'r', 0, 'Resume at this index']] - -class %(testShortname)sTest(OONITest): - implements(IPlugin, ITest) - - shortName = "%(testSNlower)s" - description = "%(testName)s" - requirements = None - options = %(testShortname)sArgs - blocking = True - - def control(self, experiment_result, args): - # What you return here ends up inside of the report. - log.msg("Running control") - return {} - - def experiment(self, args): - # What you return here gets handed as input to control - log.msg("Running experiment") - return {} - - def load_assets(self): - if self.local_options: - return {'asset': Asset(self.local_options['asset'])} - else: - return {} - -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -%(testShortname)s = %(testShortname)sTest(None, None, None) -""" - -test_vars = {'testName': None, 'testShortname': None} -test_vars['testName'] = raw_input('Test Name: ') -test_vars['testShortname'] = raw_input("Test Short Name: ") -test_vars['testSNlower'] = test_vars['testShortname'].lower() - -fname = os.path.join('plugins', test_vars['testSNlower']+'.py') - -if os.path.exists(fname): - print('WARNING! File named "%s" already exists.' % fname) - if raw_input("Do you wish to continue (y/N)? ").lower() != 'y': - print("gotcha! Dying..") - sys.exit(0) - -fp = open(fname, 'w') -fp.write(test_template % test_vars) -fp.close() - diff --git a/ooni/utils/legacy.py b/ooni/utils/legacy.py deleted file mode 100755 index 3e21d97..0000000 --- a/ooni/utils/legacy.py +++ /dev/null @@ -1,459 +0,0 @@ -# -*- coding: UTF-8 -# -# Open Observatory of Network Interference -# **************************************** -# -# legacy.py -# --------- -# Utilities for working with legacy OONI tests, i.e. tests which were created -# before the transition to the new twisted.trial based API. -# -# "The Net interprets censorship as damage and routes around it." -# - John Gilmore; TIME magazine (6 December 1993) -# -# :authors: Isis Lovecruft, Arturo Filasto -# :license: see included LICENSE file -# :copyright: (c) 2012 Isis Lovecruft, Arturo Filasto, The Tor Project, Inc. -# :version: 0.1.0-pre-alpha -# -# This API has been deprecated and is merely for API purposes. -# - -__VERSION__="0.0.1-pre-alpha" - -import inspect -import os -import yaml - -from twisted.internet import defer, reactor -from twisted.python import usage -from twisted.python import log as tplog -from twisted.python.usage import Options as tpOptions - -from ooni import nettest -from ooni.plugoo.tests import OONITest -from ooni.plugoo import work, reports -from ooni.utils import log, date -from ooni.utils.logo import getlogo - - -def runTest(test, options, global_options, reactor=reactor): - """ - Run an OONI test by name. - - @param test: - a string specifying the test name as specified inside - of shortName. - @param options: - the local options to be passed to the test. - @param global_options: the global options for OONI. - """ - parallelism = int(global_options['parallelism']) - worker = work.Worker(parallelism, reactor=reactor) - test_class = test.__class__ - report = reports.Report(test, global_options['output']) - - log_to_stdout = True - if global_options['quiet']: - log_to_stdout = False - - log.start(log_to_stdout, - global_options['log'], - global_options['verbosity']) - - resume = 0 - if not options: - options = {} - if 'resume' in options: - resume = options['resume'] - - test = test_class(options, global_options, report, reactor=reactor) - if test.tool: - test.runTool() - return True - - if test.ended: - print "Ending test" - return None - - wgen = work.WorkGenerator(test, dict(options), start=resume) - for x in wgen: - worker.push(x) - - -class LegacyOptions(usage.Options): - """Deprecated.""" - def __init__(test_classes=[], sub_commands=[]): - self.test_classes = test_classes - self.sub_commands = sub_commands - for cls in self.test_classes: - sub_commands.append([cls, None, cls.options, - "Run the %s test" % cls]) - optParameters = [ - ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], - ['output', 'o', 'report.log', "Specify output report file"], - ['log', 'l', 'oonicli.log', "Specify output log file"], - ['verbosity', 'v', 1, "Specify the logging level"]] - optFlags = [['quiet', 'q', "Don't log to stdout"]] - - def opt_version(self): - """ - This API has been deprecated; please use the new alpha-release API. - See /nettests in the top directory, as well as the /docs folder for - further information. - """ - print "OONI version: %s\n\n%s" % (__VERSION__, __doc__) - sys.exit(0) - - def __str__(self): - """ - Hack to get the sweet ascii art into the help output and replace the - strings "Commands" with "Tests". - """ - return getlogo() + '\n' + self.getSynopsis() + '\n' + \ - self.getUsage(width=None).replace("Commands:", "Tests:") - -def run_ooniprobe_py(*args): - log.start() - if not args: - args = "--help" - old_api = usage.Options() - try: - old_api.parseOptions() - except: - log.msg("Use of this API is deprecated. Please use /bin/ooniprobe.") - runTest(old_api.sub_command, old_api.subOptions, old_api) - reactor.run() - -class LegacyReporter(object): - """ - Backwards compatibility class for creating a report object for results - from a :class:`ooni.runner.LegacyTest`. A - :class:`ooni.runner.LegacyReporter` object will eventually get wrapped in - a list when :mod:`ooni.oonicli` calls - :meth:`ooni.reporter.OONIReporter.stopTest`. - - :param report_target: - The type of object to write results to, by default a list. - """ - def __init__(self, report_target=[]): - self.report_target = report_target - if isinstance(self.report_target, dict): - self._type = dict - elif isinstance(self.report_target, list): - self._type = list - else: - self._type = type(self.report_target) - - def __call__(self, info): - if self._type is dict: - self.report_target.update(info) - elif self._type is list: - self.report_target.append(info) - else: - log.debug("ADD A NEW REPORT_TARGET TYPE!!") - -class LegacyOONITest(nettest.NetTestCase): - """ - Converts an old test, which should be a subclass of - :class:`ooni.plugoo.tests.OONITest`, to an :mod:`ooni.oonicli` - compatible class. - - :param obj: - An uninstantiated old test, which should be a subclass of - :class:`ooni.plugoo.tests.OONITest`. - :param config: - A configured and instantiated :class:`twisted.python.usage.Options` - class. - :meth start_legacy_test: - Handler for calling :meth:`ooni.plugoo.tests.OONITest.startTest`. - """ - - ## we need __bases__ because inspect.getmro() as well as - ## zope.interface.implements() both expect it: - from ooni.plugoo.tests import OONITest - __bases__ = (OONITest, ) - - def __getattr__(self, name): - """ - Override of builtin getattr for :class:`ooni.runner.LegacyTest` so - that method calls to a LegacyTest instance or its parent class - OONITest do not return unhandled errors, but rather report that the - method is unknown. - """ - def __unknown_method__(*a): - log.msg("Call to unknown method %s.%s" % (self.originalTest, name)) - if a: - log.msg("Unknown method %s parameters: %s" % str(a)) - return __unknown_method__ - - def find_missing_options(self): - """ - In the case that our test is actually a class within a module named - after itself, i.e. 'ooni.plugins.bridget.bridget', we want dynamic - method discover so that we can search for the test's Options class. - - Example: - Let's say we want the Options class, which is at - ``ooni.plugins.bridget.bridget.options``. But in this case, our - original_test variable isn't going to have an attribute named - 'options', because original_test is actually the *first* occurence of - 'bridget'. - - In other words, our original_test is actually the module, so we need - to find the test, which is done with: - - getattr(original_test.__class__, test_class) - - After that, we've got our test stored as something like - ``ooni.plugins.bridget.bridget`` and we need to find 'options' as an - attribute under that, which is what - - options_finder = inspect.attrgetter('options') - - is used for. And the namespace stuff is just used for debugging edge - cases where we totally can't find the options. - - :ivar original_class: - The original subclass of OONITest, except that in this case, - because our test is a module, what we have here is - 'ooni.plugins.bridget.BridgeTest', while we actually need - something like 'ooni.plugins.bridget.bridget.BridgeTest' instead. - :ivar class_string: - The :ivar:`original_class` converted to a string. - :ivar from_module: - The parent module of :ivar:`original_class`, i.e. - `ooni.plugins.bridget`. - :ivar test_class: - The last part of :ivar:`from_module`, ie. 'bridget'. - :ivar options_finder: - An instance of :meth:`inspect.attrgetter` which searches for - methods within a test class named 'options'. - """ - original_test = self.originalTest - original_class = original_test.__class__ - class_string = str(original_class) - from_module = inspect.getmodule(original_class) - test_class = class_string.rsplit('.', 1)[1] - options_finder = inspect.attrgetter('options') - - if self.was_named is False or self.name != test_class: - log.msg("Discovered legacy test named %s ..." % test_class) - setattr(self, 'name', test_class) - - try: - namespace = globals()[class_string] - log.debug("orginal namespace: %s" % namespace) - except KeyError, keyerr: - log.debug(keyerr) - - options = tpOptions - try: - options = options_finder(getattr(original_class, test_class)) - except AttributeError: - self.__getattr__(test_class) - except Exception, e: - log.err(e) - finally: - return options() - - def __init__(self, obj, config): - """ - xxx fill me in - - :param obj: - An uninstantiated old test, which should be a subclass of - :class:`ooni.plugoo.tests.OONITest`. - :param config: - A configured and instantiated - :class:`twisted.python.usage.Options` class. - :attr originalTest: - :attr subArgs: - :attr name: - :ivar was_named: - :attr subOptions: - """ - super(LegacyOONITest, self).__init__() - self.originalTest = obj - self.start_time = date.now() - self.name = 'LegacyOONITest' - self.was_named = False - try: - self.name = self.originalTest.shortName - self.was_named = True - except AttributeError: - if self.originalTest.name and self.originalTest.name != 'oonitest': - self.name = self.originalTest.name - self.was_named = True - - if 'subArgs' in config: - self.subArgs = config['subArgs'] - else: - self.subArgs = (None, ) - log.msg("No suboptions to test %s found; continuing..."% self.name) - - try: - self.subOptions = self.originalTest.options() - except AttributeError: - if self.was_named is False: - self.subOptions = self.find_missing_options() - else: - self.subOptions = None - log.msg("That test appears to have a name, but no options!") - - if self.subOptions is not None: - if len(self.subArgs) > 0: - self.subOptions.parseOptions(self.subArgs) - self.local_options = self.subOptions - else: - print self.subOptions - - if 'reportfile' in config: - self.reporter_file = config['reportfile'] - else: - filename = str(self.name) + "-" + str(date.timestamp()) + ".yaml" - self.reporter_file = os.path.join(os.getcwd(), filename) - self.reporter = [] - self.report = LegacyReporter(report_target=self.reporter) - - self.legacy_test = self.originalTest(None, self.local_options, - None, self.report) - setattr(self.legacy_test, 'name', self.name) - setattr(self.legacy_test, 'start_time', self.start_time) - - self.inputs = {} - for keys, values in self.legacy_test.assets.items(): - self.inputs[keys] = values - setattr(self.legacy_test, 'inputs', self.inputs) - - @defer.inlineCallbacks - def run_with_args(self, args): - """ - Handler for calling :meth:`ooni.plugoo.tests.OONITest.startTest` with - each :param:`args` that, in the old framework, would have been - generated one line at a time by - :class:`ooni.plugoo.assets.Asset`. This function is wrapped with - :meth:`twisted.internet.defer.inlineCallbacks` so that the result of - each call to :meth:`ooni.plugoo.tests.OONITest.experiment` is returned - immediately as :ivar:`returned`. - """ - result = yield self.legacy_test.startTest(args) - defer.returnValue(result) - -def adapt_legacy_test(obj, config): - """ - Wrapper function for taking a legacy OONITest class and converting it into - a :class:`LegacyTest`, which is a variant of the new - :class:`ooni.nettest.TestCase` and is compatible with - :mod:`ooni.oonicli`. This allows for backward compatibility of old OONI - tests. - - :param obj: - An uninstantiated old test, which should be a subclass of - :class:`ooni.plugoo.tests.OONITest`. - :param config: - A configured and instantiated :class:`twisted.python.usage.Options` - class. - :return: - A :class:`LegacyOONITest`. - """ - return LegacyOONITest(obj, config) - -def report_legacy_test_to_file(legacy_test, file=None): - """ - xxx this function current does not get used, and could easily be handled - by ooni.runner.loadTestsAndOptions, or some other function in - ooni.runner. - - xxx fill me in - """ - reporter_file = legacy_test.reporter_file - - if file is not None: - base = os.path.dirname(os.path.abspath(file)) - if base.endswith("ooni") or base == os.getcwd(): - reporter_file = file - else: - log.msg("Writing to %s not allowed, using default file %s." - % (base, reporter_file)) - - yams = yaml.safe_dump(legacy_test.reporter) - with open(reporter_file, 'a') as rosemary: - rosemary.write(yams) - rosemary.flush() - log.msg("Finished reporting.") - -def log_legacy_test_results(result, legacy_test, args): - """ - Callback function for deferreds in :func:`start_legacy_test` which - handles updating the legacy_test's :class:`legacy_test.report`. - - :param result: - The possible result of a deferred which has been returned from - :meth:`ooni.plugoo.test.OONITest.experiment` and - :meth:`ooni.plugoo.test.OONITest.control`. - :param legacy_test: - The :class:`LegacyOONITest` which we're processing. - :param args: - The current inputs which we're giving to legacy_test.startTest(). - :return: - The :param:`legacy_test`. - """ - if result: - legacy_test.report({args: result}) - log.debug("Legacy test %s with args:\n%s\nreturned result:\n%s" - % (legacy_test.name, args, result)) - else: - legacy_test.report({args: None}) - log.debug("No results return for %s with args:\n%s" - % (legacy_test.name, args)) - return legacy_test - -def start_legacy_test(legacy_test): - """ - This is the main function which should be used to call a legacy test, it - handles parsing the deprecated :class:`ooni.plugoo.assets.Asset` items as - inputs, and calls back to a custom, backwards-compatible Reporter. - - For each input to the legacy_test, this function creates a - :class:`twisted.internet.defer.Deferred` which has already received its - :meth:`callback`. The end result is a - :class:`twisted.internet.defer.gatherResults` of all the outcomes of - :param:`legacy_test` for each of the inputs. - - :param legacy_test: - A :class:`LegacyOONITest` to process. - :ivar results: - A list of :class:`twisted.internet.defer.Deferred`s which gets - processed as a :class:`twisted.internet.defer.DeferredList`. - :ivar current_input: - The input we are current working on, i.e. what would have been 'args' - (as in, 'experiment(args)') in the old design. - :return: - A :class:`twisted.internet.defer.gatherResults`. - """ - results = [] - current_input = {} - - if len(legacy_test.inputs) > 0: - for keys, values in legacy_test.inputs: - for value in values: - current_input[keys] = value - log.debug("Running %s with args: %s" - % (legacy_test.name, current_input)) - d = legacy_test.run_with_args(current_input) - d.addCallback(log_legacy_test_results, legacy_test, - current_input) - d.addErrback(tplog.err) - results.append(d) - else: - current_input['zero_input_test'] = True - log.debug("Running %s with current input: %s" - % (legacy_test.name, current_input)) - d = legacy_test.run_with_args(current_input) - d.addCallback(log_legacy_test_results, legacy_test, current_input) - d.addErrback(tplog.err) - results.append(d) - - dlist = defer.gatherResults(results) - return dlist