November 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] * Moved old ooniprobe.py script as the logic it used for running what
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit b32acfe8c63ec9f6bbbf8b5020c6534ce0157da5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 1 09:05:48 2012 +0000 * Moved old ooniprobe.py script as the logic it used for running what are not legacy tests was mostly reimplemented in ooni/utils/legacy.py. * Cleaned up the code a lot. * Removed more unused imports... --- ooni/utils/legacy.py | 104 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 103 insertions(+), 1 deletions(-) diff --git a/ooni/utils/legacy.py b/ooni/utils/legacy.py index aa415a9..56a3ca6 100755 --- a/ooni/utils/legacy.py +++ b/ooni/utils/legacy.py @@ -1,27 +1,129 @@ -#-*- coding: utf-8 -*- +# -*- coding: UTF-8 +# +# Open Observatory of Network Interference +# **************************************** # # legacy.py # --------- # Utilities for working with legacy OONI tests, i.e. tests which were created # before the transition to the new twisted.trial based API. # +# "The Net interprets censorship as damage and routes around it." +# - John Gilmore; TIME magazine (6 December 1993) +# # :authors: Isis Lovecruft, Arturo Filasto # :license: see included LICENSE file # :copyright: (c) 2012 Isis Lovecruft, Arturo Filasto, The Tor Project, Inc. # :version: 0.1.0-pre-alpha +# +# This API has been deprecated and is merely for API purposes. +# +__VERSION__="0.0.1-pre-alpha" import inspect import os import yaml from twisted.internet import defer, reactor +from twisted.python import usage from twisted.python import log as tplog from twisted.python.usage import Options as tpOptions from ooni import nettest from ooni.plugoo.tests import OONITest +from ooni.plugoo import work, reports from ooni.utils import log, date +from ooni.utils.logo import getLogo + + +def runTest(test, options, global_options, reactor=reactor): + """ + Run an OONI test by name. + + @param test: + a string specifying the test name as specified inside + of shortName. + @param options: + the local options to be passed to the test. + @param global_options: the global options for OONI. + """ + parallelism = int(global_options['parallelism']) + worker = work.Worker(parallelism, reactor=reactor) + test_class = test.__class__ + report = reports.Report(test, global_options['output']) + + log_to_stdout = True + if global_options['quiet']: + log_to_stdout = False + + log.start(log_to_stdout, + global_options['log'], + global_options['verbosity']) + + resume = 0 + if not options: + options = {} + if 'resume' in options: + resume = options['resume'] + + test = test_class(options, global_options, report, reactor=reactor) + if test.tool: + test.runTool() + return True + + if test.ended: + print "Ending test" + return None + + wgen = work.WorkGenerator(test, dict(options), start=resume) + for x in wgen: + worker.push(x) + + +class LegacyOptions(usage.Options): + """Deprecated.""" + def __init__(test_classes=[], sub_commands=[]): + self.test_classes = test_classes + self.sub_commands = sub_commands + for cls in self.test_classes: + sub_commands.append([cls, None, cls.options, + "Run the %s test" % cls]) + optParameters = [ + ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], + ['output', 'o', 'report.log', "Specify output report file"], + ['log', 'l', 'oonicli.log', "Specify output log file"], + ['verbosity', 'v', 1, "Specify the logging level"]] + optFlags = [['quiet', 'q', "Don't log to stdout"]] + + def opt_version(self): + """ + This API has been deprecated; please use the new alpha-release API. + See /nettests in the top directory, as well as the /docs folder for + further information. + """ + print "OONI version: %s\n\n%s" % (__VERSION__, __doc__) + sys.exit(0) + + def __str__(self): + """ + Hack to get the sweet ascii art into the help output and replace the + strings "Commands" with "Tests". + """ + return getlogo() + '\n' + self.getSynopsis() + '\n' + \ + self.getUsage(width=None).replace("Commands:", "Tests:") + +def run_ooniprobe_py(*args): + log.start() + if not args: + args = "--help" + old_api = usage.Options() + try: + old_api.parseOptions() + except: + log.msg("Use of this API is deprecated. Please use /bin/ooniprobe." + runTest(old_api.sub_command, old_api.subOptions, old_api) + reactor.run() class LegacyReporter(object): """

1 0

[ooni-probe/master] * Refactoring and moving the unported tests.
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 35c25e794b389cf399a431459ccef93f131660d1 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 1 09:58:19 2012 +0000 * Refactoring and moving the unported tests. --- ooni/__init__.py | 2 +- ooni/oonitests/TO_BE_PORTED.txt | 9 -- ooni/oonitests/dnstamper.py | 200 --------------------------------------- ooni/oonitests/tcpscan.py | 86 ----------------- ooni/oonitests/traceroute.py | 110 --------------------- ooni/plugins/__init__.py | 3 - 6 files changed, 1 insertions(+), 409 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index e85436a..2f7c19f 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -20,4 +20,4 @@ from . import utils ## XXX below are legacy related modules #from . import ooniprobe from . import plugoo -from . import plugins +#from . import plugins diff --git a/ooni/oonitests/TO_BE_PORTED.txt b/ooni/oonitests/TO_BE_PORTED.txt deleted file mode 100644 index 02447cb..0000000 --- a/ooni/oonitests/TO_BE_PORTED.txt +++ /dev/null @@ -1,9 +0,0 @@ - -The tests in this directory are very old, and have neither been ported to -Twisted, nor to the new twisted.trial API framework. Although, they are not -old in the sense of the *seriously old* OONI code which was written two years -ago. - -These tests should be updated at least to use Twisted. - --isis diff --git a/ooni/oonitests/dnstamper.py b/ooni/oonitests/dnstamper.py deleted file mode 100644 index f73f494..0000000 --- a/ooni/oonitests/dnstamper.py +++ /dev/null @@ -1,200 +0,0 @@ -# -*- coding: utf-8 -*- -""" - dnstamper - ********* - - This test resolves DNS for a list of domain names, one per line, in the - file specified in the ooni-config under the setting "dns_experiment". If - the file is top-1m.txt, the test will be run using Amazon's list of top - one million domains. The experimental dns servers to query should - be specified one per line in assets/dns_servers.txt. - - The test reports censorship if the cardinality of the intersection of - the query result set from the control server and the query result set - from the experimental server is zero, which is to say, if the two sets - have no matching results whatsoever. - - NOTE: This test frequently results in false positives due to GeoIP-based - load balancing on major global sites such as google, facebook, and - youtube, etc. - - :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft - :license: see LICENSE for more details - - TODO: - * Switch to using Twisted's DNS builtins instead of dnspython - * -""" - -import os - -from twisted.names import client -from twisted.internet import reactor -from twisted.internet.protocol import Factory, Protocol -from twisted.python import usage -from twisted.plugin import IPlugin -from zope.interface import implements - -from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, OONITest -from ooni import log - -class Top1MAsset(Asset): - """ - Class for parsing the Alexa top-1m.txt as an asset. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - self = Asset.parse_line(self, line) - return line.split(',')[1].replace('\n','') - -class DNSTamperAsset(Asset): - """ - Creates DNS testing specific Assets. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - -class DNSTamperArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file of hostnames to resolve'], - ['controlserver', 'c', '8.8.8.8', 'Known good DNS server'], - ['testservers', 't', None, 'Asset file of the DNS servers to test'], - ['resume', 'r', 0, 'Resume at this index in the asset file']] -''' - def control(self, experiment_result, args): - print "Experiment Result:", experiment_result - print "Args", args - return experiment_result - - def experiment(self, args): -''' - -class DNSTamperTest(OONITest): - implements(IPlugin, ITest) - - shortName = "DNSTamper" - description = "DNS censorship detection test" - requirements = None - options = DNSTamperArgs - blocking = False - - def load_assets(self): - if self.local_options: - if self.local_options['asset']: - assetf = self.local_options['asset'] - if assetf == 'top-1m.txt': - return {'asset': Top1MAsset(assetf)} - else: - return {'asset': DNSTamperAsset(assetf)} - else: - return {} - - def lookup(self, hostname, nameserver): - """ - Resolves a hostname through a DNS nameserver to the corresponding - IP addresses. - """ - def got_result(result): - #self.logger.log(result) - print result - reactor.stop() - - def got_failure(failure): - failure.printTraceback() - reactor.stop() - - res = client.createResolver(servers=[(nameserver, 53)]) - d = res.getHostByName(hostname) - d.addCallbacks(got_result, got_failure) - - ## XXX MAY ALSO BE: - #answer = res.getAddress(servers=[('nameserver', 53)]) - - ret = [] - - for data in answer: - ret.append(data.address) - - return ret - - def reverse_lookup(self, ip, nameserver): - """ - Attempt to do a reverse DNS lookup to determine if the control and exp - sets from a positive result resolve to the same domain, in order to - remove false positives due to GeoIP load balancing. - """ - res = client.createResolver(servers=nameserver) - n = reversename.from_address(ip) - revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] - - return revn - - def experiment(self, *a, **kw): - """ - Compares the lookup() sets of the control and experiment groups. - """ - # this is just a dirty hack - address = kw['data'][0] - ns = kw['data'][1] - - config = self.config - ctrl_ns = config.tests.dns_control_server - - print "ADDRESS: %s" % address - print "NAMESERVER: %s" % ns - - exp = self.lookup(address, ns) - control = self.lookup(address, ctrl_ns) - - result = [] - - if len(set(exp) & set(control)) > 0: - print "Address %s has not tampered with on DNS server %s\n" % (address, ns) - result = (address, ns, exp, control, False) - return result - else: - print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control) - result = (address, ns, exp, control, True) - - if config.tests.dns_reverse_lookup: - - exprevn = [self.reverse_lookup(ip, ns) for ip in exp] - ctrlrevn = [self.reverse_lookup(ip, ctrl_ns) - for ip in control] - - if len(set(exprevn) & set(ctrlrevn)) > 0: - print "Further testing has eliminated this as a false positive." - else: - print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn) - return result - - else: - print "\n" - return result - -#def run(ooni): -# """ -# Run the test. -# """ -# config = ooni.config -# urls = [] -# -# if (config.tests.dns_experiment == "top-1m.txt"): -# dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, -# config.tests.dns_experiment)) -# else: -# dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, -# config.tests.dns_experiment)) -# dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, -# config.tests.dns_experiment_dns)) -# -# assets = [dns_experiment, dns_experiment_dns] -# -# dnstest = DNST(ooni) -# ooni.logger.info("Beginning dnstamper test...") -# dnstest.run(assets, {'index': 1}) -# ooni.logger.info("Dnstamper test completed!") - -dnstamper = DNSTamperTest(None, None, None) diff --git a/ooni/oonitests/tcpscan.py b/ooni/oonitests/tcpscan.py deleted file mode 100644 index 26e854b..0000000 --- a/ooni/oonitests/tcpscan.py +++ /dev/null @@ -1,86 +0,0 @@ -""" - TCP Port Scanner - **************** - - Does a TCP connect scan on the IP:port pairs. - -""" -import os -from gevent import socket -from datetime import datetime -import socks - -from plugoo.assets import Asset -from plugoo.tests import Test - -__plugoo__ = "TCP Port Scanner" -__desc__ = "This a test template to be used to build your own tests" - -class TCPScanAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - -class TCPScan(Test): - """ - The main Test class - """ - - def experiment(self, *a, **kw): - """ - Fill this up with the tasks that should be performed - on the "dirty" network and should be compared with the - control. - """ - addr = kw['data'] - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - res = False - try: - self.logger.debug('Doing a connection to %s' % addr) - s.connect((addr.split(':')[0], int(addr.split(':')[1]))) - res = True - except socket.error, msg: - self.logger.debug('Connection failed to %s: %s' % (addr, msg)) - - finally: - s.close() - - return {'Time': datetime.now(), - 'Address': addr, - 'Status': res} - - def control(self): - """ - Fill this up with the control related code. - """ - return True - -def run(ooni, asset=None): - """ - This is the function that will be called by OONI - and it is responsible for instantiating and passing - the arguments to the Test class. - """ - config = ooni.config - - # This the assets array to be passed to the run function of - # the test - if asset: - assets = [TCPScanAsset(asset)] - else: - assets = [TCPScanAsset(os.path.join(config.main.assetdir, \ - "tcpscan.txt"))] - - # Instantiate the Test - thetest = TCPScan(ooni) - ooni.logger.info("starting TCP Scan...") - # Run the test with argument assets - thetest.run(assets) - ooni.logger.info("finished.") - - diff --git a/ooni/oonitests/traceroute.py b/ooni/oonitests/traceroute.py deleted file mode 100644 index d84bf25..0000000 --- a/ooni/oonitests/traceroute.py +++ /dev/null @@ -1,110 +0,0 @@ -try: - from dns import resolver -except: - print "Error: dnspython is not installed (http://www.dnspython.org/)" -import gevent -import os -import plugoo - -try: - import scapy -except: - print "Error: traceroute plugin requires scapy to be installed (http://www.secdev.org/projects/scapy)" - -from plugoo.assets import Asset -from plugoo.tests import Test - -import socket - -__plugoo__ = "Traceroute" -__desc__ = "Performs TTL walking tests" - -class TracerouteAsset(Asset): - def __init__(self, file=None): - self = Asset.__init__(self, file) - - -class Traceroute(Test): - """A *very* quick and dirty traceroute implementation, UDP and TCP - """ - def traceroute(self, dst, dst_port=3880, src_port=3000, proto="tcp", max_hops=30): - dest_addr = socket.gethostbyname(dst) - print "Doing traceroute on %s" % dst - - recv = socket.getprotobyname('icmp') - send = socket.getprotobyname(proto) - ttl = 1 - while True: - recv_sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, recv) - if proto == "tcp": - send_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, send) - else: - send_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, send) - recv_sock.settimeout(10) - send_sock.settimeout(10) - - send_sock.setsockopt(socket.SOL_IP, socket.IP_TTL, ttl) - recv_sock.bind(("", src_port)) - if proto == "tcp": - try: - send_sock.settimeout(2) - send_sock.connect((dst, dst_port)) - except socket.timeout: - pass - - except Exception, e: - print "Error doing connect %s" % e - else: - send_sock.sendto("", (dst, dst_port)) - - curr_addr = None - try: - print "receiving data..." - _, curr_addr = recv_sock.recvfrom(512) - curr_addr = curr_addr[0] - - except socket.error, e: - print "SOCKET ERROR: %s" % e - - except Exception, e: - print "ERROR: %s" % e - - finally: - send_sock.close() - recv_sock.close() - - if curr_addr is not None: - curr_host = "%s" % curr_addr - else: - curr_host = "*" - - print "%d\t%s" % (ttl, curr_host) - - if curr_addr == dest_addr or ttl > max_hops: - break - - ttl += 1 - - - def experiment(self, *a, **kw): - # this is just a dirty hack - address = kw['data'][0] - - self.traceroute(address) - -def run(ooni): - """Run the test""" - config = ooni.config - urls = [] - - traceroute_experiment = TracerouteAsset(os.path.join(config.main.assetdir, \ - config.tests.traceroute)) - - assets = [traceroute_experiment] - - traceroute = Traceroute(ooni) - ooni.logger.info("starting traceroute test") - traceroute.run(assets) - ooni.logger.info("finished") - - diff --git a/ooni/plugins/__init__.py b/ooni/plugins/__init__.py deleted file mode 100644 index 129108c..0000000 --- a/ooni/plugins/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -#from twisted.plugin import pluginPackagePaths -#__path__.extend(pluginPackagePaths(__name__)) -#__all__ = []

1 0

[ooni-probe/master] * Worked out the last bit of the bug with tests not getting all the
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 9f0304b59b86d8d3be3e6d1564111b92fc18a1e8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 1 11:59:45 2012 +0000 * Worked out the last bit of the bug with tests not getting all the different available inputs, i.e. a test would take things from an input file, and then not from commandline of from hardcoded parameters. I think it's done. A couple of the tests are still giving errors. --- ooni/runner.py | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index f10f9ca..ae086c1 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -244,10 +244,10 @@ def loadTestsAndOptions(classes, config): if hasattr(klass, 'optParameters') or hasattr(klass, 'inputFile'): try: - opts = processTestOptions(klass, config) - print opts, config + opt = processTestOptions(klass, config) + print opt, config except: - opts = {} + opt = {} finally: instance = klass() try: @@ -255,11 +255,11 @@ def loadTestsAndOptions(classes, config): except Exception, e: inputs = []; log.err(e) else: - opts.update({'inputs': inputs}) - finally: - options.append(opts) - log.debug("loadTestsAndOptions(): type(opts)=%s" % type(opts)) - + if opt and opt is not None: + opt.update({'inputs':inputs}) + else: pass + finally: options.append(opt) + log.debug("loadTestsAndOptions(): type(opt)=%s" % type(opt)) return test_cases, options class ORunner(object):

1 0

[ooni-probe/master] * Remainer of the TestCase changes, now that the other problems have been
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 9ad8e44a9a430521eb664b3bc508caad1e28e9f8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 1 10:03:36 2012 +0000 * Remainer of the TestCase changes, now that the other problems have been sorted out. Restructured most of the logic for test creation, and removed some assertion which are actually incapable of being False. --- ooni/nettest.py | 78 +++++++++++++++++++++++++++++++-------- ooni/runner.py | 100 +++++++++++++++++++++++++++++++++++-------------- ooni/utils/legacy.py | 4 +- 3 files changed, 135 insertions(+), 47 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index ff16f0a..e3c7800 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -112,6 +112,29 @@ class TestCase(unittest.TestCase): optParameters = None requiresRoot = False + def setUpClass(self, *args, **kwargs): + """ + Create a TestCase instance. This function is equivalent to '__init__'. + To add futher setup steps before a set of tests in a TestCase instance + run, create a function called 'setUp'. + + Class attributes, such as `report`, `optParameters`, `name`, and + `author` should be overriden statically as class attributes in any + subclass of :class:`ooni.nettest.TestCase`, so that the calling + functions in ooni.runner can handle them correctly. + """ + methodName = 'runTest' + if kwargs: + if 'methodName' in kwargs: + methodName = kwargs['methodName'] + + super(TestCase, self).setUpClass() + + #for key, value in kwargs.items(): + # setattr(self.__class__, key, value) + # + #self.inputs = self.getInputs() + def deferSetUp(self, ignored, result): """ If we have the reporterFactory set we need to write the header. If @@ -128,30 +151,53 @@ class TestCase(unittest.TestCase): log.debug("Not first run. Running test setup directly") return unittest.TestCase.deferSetUp(self, ignored, result) - def inputProcessor(self, fp): - log.debug("Running default input processor") - for x in fp.readlines(): - yield x.strip() - fp.close() + @staticmethod + def inputParser(inputs): + """Replace me with a custom function for parsing inputs.""" + log.debug("Running custom input processor") + return inputs + + def _getInputs(self): + """ + I am called from the ooni.runner and you probably should not override + me. I gather the internal inputs from an instantiated test class and + pass them to the rest of the runner. + + If you are looking for a way to parse inputs from inputFile, see + :meth:`inputParser`. + I open :attr:inputFile if there is one, and return inputs one by one + after stripping them of whitespace and running them through the parser + :meth:`inputParser`. + """ + + ## don't burn cycles on testing null inputs: + if len(self.inputs) == 1 and self.inputs[0] == None: + self.inputs = [] + processor = [] + else: + log.msg("Received direct inputs:\n%s" % self.inputs) + processor = [i for i in self.inputProcessor(self.inputs)] - def getOptions(self): - log.debug("Getting options for test") if self.inputFile: try: - fp = open(self.inputFile) ## xxx fixme: - except Exception, e: ## bad news to leave file - log.err(e) ## descriptors open + fp = open(self.inputFile) + except Exception, e: + log.err(e) + fp.close() else: - from_file = self.__input_file_processor__(fp) - self.inputs = itertools.chain(processor, from_file) + log.debug("Running input file processor") + lines = [line.strip() for line in fp.readlines()] + fp.close() + parsed = [self.inputParser(ln) for ln in lines] + both = itertools.chain(processor, parsed) elif self.inputFile is False: log.debug("%s specified that it doesn't need inputFile." - % self.__class__.__name__) - self.inputs = processed + % self.name) + both = processor else: - raise BrokenImplementation + both = processor + return both - return self.inputs def getOptions(self): ''' diff --git a/ooni/runner.py b/ooni/runner.py index 36f5b6e..0e5dbba 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -134,8 +134,12 @@ def processTestOptions(cls, config): if cls.inputFile: cls.optParameters.append(cls.inputFile) + if not hasattr(cls, subCommands): + cls.subCommands = [] + class Options(usage.Options): optParameters = cls.optParameters + parseArgs = lambda a: cls.subCommands.append(a) opts = Options() opts.parseOptions(config['subArgs']) @@ -157,63 +161,101 @@ def processTestOptions(cls, config): except usage.UsageError: options = opts.opt_help() else: - return cls, options """ - - return cls, cls.localOptions + return cls.localOptions def loadTestsAndOptions(classes, config): """ Takes a list of test classes and returns their testcases and options. Legacy tests will be adapted. """ + from inspect import isclass method_prefix = 'test' options = [] test_cases = [] - _old_class_type = LegacyOONITest + DEPRECATED = LegacyOONITest for klass in classes: - if isinstance(klass, _old_class_type): + if isinstance(klass, DEPRECATED) \ + and not issubclass(klass, TestCase): + log.msg("Processing cases and options for legacy test %s" + % ( klass.shortName if hasattr(klass, shortName) + else 'oonitest' )) + if hasattr(klass, description): + log.msg("%s" % klass.description) + + subcmds = [] + if hasattr(klass, options): ## an unitiated Legacy test + log.debug("%s.options found: %s " % (klass, klass.options)) + try: + assert isclass(klass.options), \ + "%s is not class" % klass.qoptions + except AssertionError, ae: + log.debug(ae) + else: + ok = klass.options + ok.parseArgs = lambda x: subcmds.append(x) + try: + opts = ok() + opts.parseOptions(config['subArgs']) + except Exception, e: + log.debug(e) + opts = {} + finally: + opts.append(opts) + + if hasattr(klass, local_options): ## we've been initialized already + log.debug("%s.local_options found" % klass) + try: + assert klass.local_options is not None + opts = klass.local_options + except AttributeError, ae: + opts = {}; log.debug(ae) + finally: + log.debug("type(opts) = %s" % type(opts)) + options.append(opts) try: cases = start_legacy_test(klass) - if cases: - log.debug("Processing cases") - log.debug(str(cases)) - return [], [] + #if cases: ## why are these empty lists here? + # return [], [] ## were nettests having issues due + except Exception, e: ## to legacy tests? + cases = []; log.err(e) + finally: + log.debug(str(cases)) test_cases.append(cases) + + elif issubclass(klass, TestCase): + log.debug("Processing cases and options for OONI %s test" + % ( klass.name if hasattr( klass, 'name' ) \ + else TestCase.name )) + try: + tests = reflect.prefixedMethodNames(klass, method_prefix) except Exception, e: - log.err(e) + tests = []; log.debug(e) else: try: - opts = klass.local_options - options.append(opts) - except AttributeError, ae: - options.append([]) - log.err(ae) - if cases: - print cases - return [], [] - else: - tests = reflect.prefixedMethodNames(klass, method_prefix) - if tests: - cases = makeTestCases(klass, tests, method_prefix) - test_cases.append(cases) + cases = makeTestCases(klass, tests, method_prefix) + except Exception, e: + cases = []; log.err(e) + log.debug("loadTestsAndOptions(): test %s found cases=%s" + % (tests, cases)) + try: opts = processTestOptions(klass, config) except AttributeError, ae: - options.append([]) - log.err(ae) - else: + opts = {}; log.err(ae) + finally: try: instance = klass() - inputs = instance.__get_inputs__() + inputs = instance._getInputs() except Exception, e: - log.err(e) + inputs = []; log.err(e) else: - opts.update(inputs) + opts.update({'inputs': inputs}) options.append(opts) + log.debug("loadTestsAndOptions(): type(opts)=%s" % type(opts)) return test_cases, options diff --git a/ooni/utils/legacy.py b/ooni/utils/legacy.py index 56a3ca6..61fbe2f 100755 --- a/ooni/utils/legacy.py +++ b/ooni/utils/legacy.py @@ -34,7 +34,7 @@ from ooni import nettest from ooni.plugoo.tests import OONITest from ooni.plugoo import work, reports from ooni.utils import log, date -from ooni.utils.logo import getLogo +from ooni.utils.logo import getlogo def runTest(test, options, global_options, reactor=reactor): @@ -121,7 +121,7 @@ def run_ooniprobe_py(*args): try: old_api.parseOptions() except: - log.msg("Use of this API is deprecated. Please use /bin/ooniprobe." + log.msg("Use of this API is deprecated. Please use /bin/ooniprobe.") runTest(old_api.sub_command, old_api.subOptions, old_api) reactor.run()

1 0

[ooni-probe/master] * Deleted the old ooniprobe.py script, except for a few parts which may still
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 2d31c729d25280f415dab515e684662e14402f4b Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 1 09:08:38 2012 +0000 * Deleted the old ooniprobe.py script, except for a few parts which may still be useful -- those are now in ooni.utils.legacy. --- ooni/ooniprobe.py | 151 ----------------------------------------------------- 1 files changed, 0 insertions(+), 151 deletions(-) diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py deleted file mode 100755 index 5793eee..0000000 --- a/ooni/ooniprobe.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -# -# ooniprobe -# ********* -# -# Open Observatory of Network Interference -# -# "The Net interprets censorship as damage and routes around it." -# - John Gilmore; TIME magazine (6 December 1993) -# -# The goal of ooni-probe is to collect data about censorship around -# the world. -# -# :copyright: (c) 2012 by Arturo Filastò, Isis Lovecruft -# :license: see LICENSE for more details. -# :version: 0.0.1-pre-alpha -# - -import sys -from pprint import pprint - -from twisted.python import usage -from twisted.internet import reactor -from twisted.plugin import getPlugins - -from zope.interface.verify import verifyObject -from zope.interface.exceptions import BrokenImplementation -from zope.interface.exceptions import BrokenMethodImplementation - -from plugoo import tests, work, assets, reports -from plugoo.interface import ITest -from utils.logo import getlogo -from utils import log -import plugins - -__version__ = "0.0.1-prealpha" - -def retrieve_plugoo(): - """ - Get all the plugins that implement the ITest interface and get the data - associated to them into a dict. - """ - interface = ITest - d = {} - error = False - for p in getPlugins(interface, plugins): - try: - verifyObject(interface, p) - d[p.shortName] = p - except BrokenImplementation, bi: - print "Plugin Broken" - print bi - error = True - if error != False: - print "Plugin Loaded!" - return d - -plugoo = retrieve_plugoo() - -def runTest(test, options, global_options, reactor=reactor): - """ - Run an OONI probe test by name. - - @param test: a string specifying the test name as specified inside of - shortName. - - @param options: the local options to be passed to the test. - - @param global_options: the global options for OONI - """ - parallelism = int(global_options['parallelism']) - worker = work.Worker(parallelism, reactor=reactor) - test_class = plugoo[test].__class__ - report = reports.Report(test, global_options['output']) - - log_to_stdout = True - if global_options['quiet']: - log_to_stdout = False - - log.start(log_to_stdout, - global_options['log'], - global_options['verbosity']) - - resume = 0 - if not options: - options = {} - if 'resume' in options: - resume = options['resume'] - - test = test_class(options, global_options, report, reactor=reactor) - if test.tool: - test.runTool() - return True - - if test.ended: - print "Ending test" - return None - - wgen = work.WorkGenerator(test, - dict(options), - start=resume) - for x in wgen: - worker.push(x) - -class Options(usage.Options): - tests = plugoo.keys() - subCommands = [] - for test in tests: - subCommands.append([test, None, plugoo[test].options, "Run the %s test" % test]) - - optFlags = [ - #['remote', 'r', "If the test should be run remotely (not supported)"], - ['quiet', 'q', "Don't log to stdout"] - ] - - optParameters = [ - ['parallelism', 'n', 10, "Specify the number of parallel tests to run"], - #['target-node', 't', 'localhost:31415', 'Select target node'], - ['output', 'o', 'report.log', "Specify output report file"], - ['log', 'l', 'oonicli.log', "Specify output log file"], - ['verbosity', 'v', 1, "Specify the logging level"], - #['password', 'p', 'opennetwork', "Specify the password for authentication"], - ] - - def opt_version(self): - """ - Display OONI version and exit. - """ - print "OONI version:", __version__ - sys.exit(0) - - def __str__(self): - """ - Hack to get the sweet ascii art into the help output and replace the - strings "Commands" with "Tests". - """ - return getlogo() + '\n' + self.getSynopsis() + '\n' + \ - self.getUsage(width=None).replace("Commands:", "Tests:") - -if __name__ == "__main__": - config = Options() - config.parseOptions() - - if not config.subCommand: - config.opt_help() - sys.exit(1) - - runTest(config.subCommand, config.subOptions, config) - reactor.run() -

1 0

[ooni-probe/master] * The Makefile is a quick and dirty way to run all the tests simultaneously,
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 1e5d1aeea8115f5d0b8c04c2308ef001a3538481 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 1 12:10:18 2012 +0000 * The Makefile is a quick and dirty way to run all the tests simultaneously, to check that your haxorzingz haven't fubar'd all of the things. * That Makefile still needs more tests added to it. * There is a short INSTRUCTIONS file in /bin so that people don't do that thing where they are like, "uh...i don't know which greenword thing to run." --- bin/INSTRUCTIONS | 15 +++++++++++++++ bin/Makefile | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 0 deletions(-) diff --git a/bin/INSTRUCTIONS b/bin/INSTRUCTIONS new file mode 100644 index 0000000..a624ae7 --- /dev/null +++ b/bin/INSTRUCTIONS @@ -0,0 +1,15 @@ +That Makefile is for testing that the tests work. However, not +all of the tests have a line in the Makefile yet. If you would +like to add things to it, feel free to. Or even to write OONI +unittests for realsies. + +The 'old_ooniprobe' script used to call the old script for +starting up tests, ooni/ooniprobe.py, but I have just removed +ooniprobe.py and kept the remaining useful pieces in +ooni/utils/legacy.py where the rest of the new API for running +old code lives. I think they are happier if we keep them together. + +tl;dr: the thing you actually want to use is the script called + 'ooniprobe'. + + --isis diff --git a/bin/Makefile b/bin/Makefile new file mode 100644 index 0000000..039f596 --- /dev/null +++ b/bin/Makefile @@ -0,0 +1,42 @@ +######################################################### +# This janky Makefile is Isis' halfassed solution for # +# a "one-click" zomg-are-we-there-yet oracle. # +# # +# Obviously, you just do '$ make all'. <(A)3 # +######################################################### + +# +# you'll also need a file of things to test things on. +# for all of the things. +# + +## all tests, without debugging +## ---------------------------- +all: echot simplet captivet dnst httphostt squidt + +## all tests, with debugging +## ------------------------- +## note: you will be doing "n-Ret-n-Ret-n-Ret-s-Ret-n-Ret..." +## for a *REALLY* long time +all_debug: echod simpled captived dnsd httphostd squidd + +simplet: + ../bin/ooniprobe ../nettests/simpletest.py -a ../ooni/assets/hostnames.txt + +simpled: + python -m pdb ../bin/ooniprobe ../nettests/simpletest.py -a ../ooni/assets/hostnames.txt + +echot: + ../bin/ooniprobe ../ooni/echo.py -f ../ooni/assets/hostnames.txt + +echod: + python -m pdb ../bin/ooniprobe ../ooni/echo.py -f ../ooni/assets/hostnames.txt + +captivet: + ../bin/ooniprobe ../nettests/core/captiveportal.py -f ../ooni/assets/hostnames.txt + +captived: + python -m pdb ../bin/ooniprobe --spew ../nettests/core/captiveportal.py -f ../ooni/assets/hostnames.txt + +mvreports: + for $report in `find ../ -name "report*"`; do mv $report test-results ; done

1 0

[ooni-probe/master] * Fixed a stupid bug where only half the occurrences of a method
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit a365349b8ff58233968e844e386ac67bcfda090d Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 1 11:32:25 2012 +0000 * Fixed a stupid bug where only half the occurrences of a method were renamed. --- nettests/core/echo.py | 205 +++++++++++++++++++++++++++++++++++++++++++++++ nettests/simpletest.py | 1 - ooni/echo.py | 206 ------------------------------------------------ ooni/nettest.py | 28 ++++--- ooni/runner.py | 25 +++--- 5 files changed, 236 insertions(+), 229 deletions(-) diff --git a/nettests/core/echo.py b/nettests/core/echo.py new file mode 100644 index 0000000..a0826b6 --- /dev/null +++ b/nettests/core/echo.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# +---------+ +# | echo.py | +# +---------+ +# A simple ICMP-8 ping test. +# +# :author: Isis Lovecruft +# :version: 0.0.1-pre-alpha +# :license: (c) 2012 Isis Lovecruft +# see attached LICENCE file +# + +import os +import sys + +from pprint import pprint + +from twisted.internet import reactor +from twisted.plugin import IPlugin +from twisted.python import usage +from ooni.nettest import TestCase +from ooni.utils import log, Storage +from ooni.utils.net import PermissionsError, IfaceError + +try: + from scapy.all import sr1, IP, ICMP ## XXX v4/v6? + from ooni.lib import txscapy + from ooni.lib.txscapy import txsr, txsend + from ooni.templates.scapyt import ScapyTest +except: + log.msg("This test requires scapy, see www.secdev.org/projects/scapy") + +## xxx TODO: move these to a utility function for determining OSes +LINUX=sys.platform.startswith("linux") +OPENBSD=sys.platform.startswith("openbsd") +FREEBSD=sys.platform.startswith("freebsd") +NETBSD=sys.platform.startswith("netbsd") +DARWIN=sys.platform.startswith("darwin") +SOLARIS=sys.platform.startswith("sunos") +WINDOWS=sys.platform.startswith("win32") + +class EchoTest(ScapyTest): + """ + xxx fill me in + """ + name = 'echo' + author = 'Isis Lovecruft <isis(a)torproject.org>' + description = 'A simple ICMP-8 test to see if a host is reachable.' + version = '0.0.1' + inputFile = ['file', 'f', None, 'File of list of IPs to ping'] + requirements = None + report = Storage() + + optParameters = [ + ['interface', 'i', None, 'Network interface to use'], + ['count', 'c', 5, 'Number of packets to send', int], + ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], + ['ttl', 'l', 25, 'Set the IP Time to Live', int], + ['timeout', 't', 2, 'Seconds until timeout if no response', int], + ['pcap', 'p', None, 'Save pcap to this file'], + ['receive', 'r', True, 'Receive response packets'] + ] + + def setUpClass(self, *a, **kw): + ''' + :ivar ifaces: + Struct returned from getifaddrs(3) and turned into a tuple in the + form (*ifa_name, AF_FAMILY, *ifa_addr) + ''' + super(EchoTest, self).__init__(*a, **kw) + + ## allow subclasses which register/implement external classes + ## to define their own reactor without overrides: + if not hasattr(super(EchoTest, self), 'reactor'): + log.debug("%s test: Didn't find reactor!" % self.name) + self.reactor = reactor + + if self.localOptions: + log.debug("%s localOptions found" % self.name) + log.debug("%s test options: %s" % (self.name, self.subOptions)) + self.local_options = self.localOptions.parseOptions(self.subOptions) + for key, value in self.local_options: + log.debug("Set attribute %s[%s] = %s" % (self.name, key, value)) + setattr(self, key, value) + + ## xxx is this now .subOptions? + #self.inputFile = self.localOptions['file'] + self.timeout *= 1000 ## convert to milliseconds + + if not self.interface: + log.msg("No network interface specified!") + log.debug("OS detected: %s" % sys.platform) + if LINUX or OPENBSD or NETBSD or FREEBSD or DARWIN or SOLARIS: + from twisted.internet.test import _posixifaces + log.msg("Attempting to discover network interfaces...") + ifaces = _posixifaces._interfaces() + elif WINDOWS: + from twisted.internet.test import _win32ifaces + log.msg("Attempting to discover network interfaces...") + ifaces = _win32ifaces._interfaces() + else: + log.debug("Client OS %s not accounted for!" % sys.platform) + log.debug("Unable to discover network interfaces...") + ifaces = [('lo', '')] + + ## found = {'eth0': '1.1.1.1'} + found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] + log.info("Found interfaces:\n%s" % pprint(found)) + self.interfaces = self.tryInterfaces(found) + else: + ## xxx need a way to check that iface exists, is up, and + ## we have permissions on it + log.debug("Our interface has been set to %s" % self.interface) + + if self.pcap: + try: + self.pcapfile = open(self.pcap, 'a+') + except: + log.msg("Unable to write to pcap file %s" % self.pcap) + self.pcapfile = None + + try: + assert os.path.isfile(self.file) + fp = open(self.file, 'r') + except Exception, e: + hosts = ['8.8.8.8', '38.229.72.14'] + log.err(e) + else: + self.inputs = self.inputProcessor(fp) + self.removePorts(hosts) + + log.debug("Initialization of %s test completed with:\n%s" + % (self.name, ''.join(self.__dict__))) + + @staticmethod + def inputParser(inputs): + log.debug("Removing possible ports from host addresses...") + log.debug("Initial inputs:\n%s" % pprint(inputs)) + + assert isinstance(inputs, list) + hosts = [h.rsplit(':', 1)[0] for h in inputs] + log.debug("Inputs converted to:\n%s" % hosts) + + return hosts + + def tryInterfaces(self, ifaces): + try: + from scapy.all import sr1 ## we want this check to be blocking + except: + log.msg("This test requires scapy: www.secdev.org/projects/scapy") + raise SystemExit + + ifup = {} + while ifaces: + for ifname, ifaddr in ifaces: + log.debug("Currently testing network capabilities of interface" + + "%s by sending a packet to our address %s" + % (ifname, ifaddr)) + try: + pkt = IP(dst=ifaddr)/ICMP() + ans, unans = sr(pkt, iface=ifname, timeout=self.timeout) + except Exception, e: + raise PermissionsError if e.find("Errno 1") else log.err(e) + else: + ## xxx i think this logic might be wrong + log.debug("Interface test packet\n%s\n\n%s" + % (pkt.summary(), pkt.show2())) + if ans.summary(): + log.info("Received answer for test packet on interface" + +"%s :\n%s" % (ifname, ans.summary())) + ifup.update(ifname, ifaddr) + else: + log.info("Our interface test packet was unanswered:\n%s" + % unans.summary()) + + if len(ifup) > 0: + log.msg("Discovered the following working network interfaces: %s" + % ifup) + return ifup + else: + raise IfaceError("Could not find a working network interface.") + + def buildPackets(self): + log.debug("self.input is %s" % self.input) + log.debug("self.hosts is %s" % self.hosts) + for addr in self.input: + packet = IP(dst=self.input)/ICMP() + self.request.append(packet) + return packet + + def test_icmp(self): + if self.recieve: + self.buildPackets() + all = [] + for packet in self.request: + d = self.sendReceivePackets(packets=packet) + all.append(d) + self.response.update({packet: d}) + d_list = defer.DeferredList(all) + return d_list + else: + d = self.sendPackets() + return d diff --git a/nettests/simpletest.py b/nettests/simpletest.py index 7cc9318..c599d68 100644 --- a/nettests/simpletest.py +++ b/nettests/simpletest.py @@ -5,7 +5,6 @@ class SimpleTest(nettest.TestCase): ['controlserver', 'c', 'google.com', 'Specify the control server'], ['resume', 'r', 0, 'Resume at this index'], ['other', 'o', None, 'Other arguments']] - def test_foo(self, *arg, **kw): print "Running %s with %s" % ("test_foo", self.input) self.report['test_foo'] = 'Antani' diff --git a/ooni/echo.py b/ooni/echo.py deleted file mode 100644 index c4ef588..0000000 --- a/ooni/echo.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# +---------+ -# | echo.py | -# +---------+ -# A simple ICMP-8 ping test. -# -# :author: Isis Lovecruft -# :version: 0.0.1-pre-alpha -# :license: (c) 2012 Isis Lovecruft -# see attached LICENCE file -# - -import os -import sys - -from pprint import pprint - -from twisted.internet import reactor -from twisted.plugin import IPlugin -from twisted.python import usage -from ooni.nettest import TestCase -from ooni.utils import log, Storage -from ooni.utils.net import PermissionsError, IfaceError - -try: - from scapy.all import sr1, IP, ICMP ## XXX v4/v6? - from ooni.lib import txscapy - from ooni.lib.txscapy import txsr, txsend - from ooni.templates.scapyt import ScapyTest -except: - log.msg("This test requires scapy, see www.secdev.org/projects/scapy") - -## xxx TODO: move these to a utility function for determining OSes -LINUX=sys.platform.startswith("linux") -OPENBSD=sys.platform.startswith("openbsd") -FREEBSD=sys.platform.startswith("freebsd") -NETBSD=sys.platform.startswith("netbsd") -DARWIN=sys.platform.startswith("darwin") -SOLARIS=sys.platform.startswith("sunos") -WINDOWS=sys.platform.startswith("win32") - -class EchoTest(ScapyTest): - """ - xxx fill me in - """ - name = 'echo' - author = 'Isis Lovecruft <isis(a)torproject.org>' - description = 'A simple ICMP-8 test to see if a host is reachable.' - version = '0.0.1' - inputFile = ['file', 'f', None, 'File of list of IPs to ping'] - requirements = None - report = Storage() - - optParameters = [ - ['interface', 'i', None, 'Network interface to use'], - ['count', 'c', 5, 'Number of packets to send', int], - ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], - ['ttl', 'l', 25, 'Set the IP Time to Live', int], - ['timeout', 't', 2, 'Seconds until timeout if no response', int], - ['pcap', 'p', None, 'Save pcap to this file'], - ['receive', 'r', True, 'Receive response packets'] - ] - - def setUpClass(self, *a, **kw): - ''' - :ivar ifaces: - Struct returned from getifaddrs(3) and turned into a tuple in the - form (*ifa_name, AF_FAMILY, *ifa_addr) - ''' - super(EchoTest, self).__init__(*a, **kw) - - ## allow subclasses which register/implement external classes - ## to define their own reactor without overrides: - if not hasattr(super(EchoTest, self), 'reactor'): - log.debug("%s test: Didn't find reactor!" % self.name) - self.reactor = reactor - - if self.localOptions: - log.debug("%s localOptions found" % self.name) - log.debug("%s test options: %s" % (self.name, self.subOptions)) - self.local_options = self.localOptions.parseOptions(self.subOptions) - for key, value in self.local_options: - log.debug("Set attribute %s[%s] = %s" % (self.name, key, value)) - setattr(self, key, value) - - ## xxx is this now .subOptions? - #self.inputFile = self.localOptions['file'] - self.timeout *= 1000 ## convert to milliseconds - - if not self.interface: - log.msg("No network interface specified!") - log.debug("OS detected: %s" % sys.platform) - if LINUX or OPENBSD or NETBSD or FREEBSD or DARWIN or SOLARIS: - from twisted.internet.test import _posixifaces - log.msg("Attempting to discover network interfaces...") - ifaces = _posixifaces._interfaces() - elif WINDOWS: - from twisted.internet.test import _win32ifaces - log.msg("Attempting to discover network interfaces...") - ifaces = _win32ifaces._interfaces() - else: - log.debug("Client OS %s not accounted for!" % sys.platform) - log.debug("Unable to discover network interfaces...") - ifaces = [('lo', '')] - - ## found = {'eth0': '1.1.1.1'} - found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] - log.info("Found interfaces:\n%s" % pprint(found)) - self.interfaces = self.tryInterfaces(found) - else: - ## xxx need a way to check that iface exists, is up, and - ## we have permissions on it - log.debug("Our interface has been set to %s" % self.interface) - - if self.pcap: - try: - self.pcapfile = open(self.pcap, 'a+') - except: - log.msg("Unable to write to pcap file %s" % self.pcap) - self.pcapfile = None - - try: - assert os.path.isfile(self.file) - fp = open(self.file, 'r') - except Exception, e: - hosts = ['8.8.8.8', '38.229.72.14'] - log.err(e) - else: - self.inputs = self.inputProcessor(fp) - self.removePorts(hosts) - - log.debug("Initialization of %s test completed with:\n%s" - % (self.name, ''.join(self.__dict__))) - - def inputProcessor(self, fp): - inputs = [x.strip() for x in fp.readlines() if not x.startswith('#')] - fp.close() - log.debug("Removing possible ports from host addresses...") - log.debug("Initial inputs:\n%s" % pprint(inputs)) - - assert isinstance(inputs, list) - hosts = [h.rsplit(':', 1)[0] for h in inputs] - log.debug("Inputs converted to:\n%s" % hosts) - - return hosts - - def tryInterfaces(self, ifaces): - try: - from scapy.all import sr1 ## we want this check to be blocking - except: - log.msg("This test requires scapy: www.secdev.org/projects/scapy") - raise SystemExit - - ifup = {} - while ifaces: - for ifname, ifaddr in ifaces: - log.debug("Currently testing network capabilities of interface" - + "%s by sending a packet to our address %s" - % (ifname, ifaddr)) - try: - pkt = IP(dst=ifaddr)/ICMP() - ans, unans = sr(pkt, iface=ifname, timeout=self.timeout) - except Exception, e: - raise PermissionsError if e.find("Errno 1") else log.err(e) - else: - ## xxx i think this logic might be wrong - log.debug("Interface test packet\n%s\n\n%s" - % (pkt.summary(), pkt.show2())) - if ans.summary(): - log.info("Received answer for test packet on interface" - +"%s :\n%s" % (ifname, ans.summary())) - ifup.update(ifname, ifaddr) - else: - log.info("Our interface test packet was unanswered:\n%s" - % unans.summary()) - - if len(ifup) > 0: - log.msg("Discovered the following working network interfaces: %s" - % ifup) - return ifup - else: - raise IfaceError("Could not find a working network interface.") - - def buildPackets(self): - log.debug("self.input is %s" % self.input) - log.debug("self.hosts is %s" % self.hosts) - for addr in self.input: - packet = IP(dst=self.input)/ICMP() - self.request.append(packet) - return packet - - def test_icmp(self): - if self.recieve: - self.buildPackets() - all = [] - for packet in self.request: - d = self.sendReceivePackets(packets=packet) - all.append(d) - self.response.update({packet: d}) - d_list = defer.DeferredList(all) - return d_list - else: - d = self.sendPackets() - return d diff --git a/ooni/nettest.py b/ooni/nettest.py index e3c7800..541d65b 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -1,7 +1,15 @@ # -*- encoding: utf-8 -*- # -# :authors: Arturo "hellais" Filastò <art(a)fuffa.org> +# nettest.py +# -------------------> +# +# :authors: Arturo "hellais" Filastò <art(a)fuffa.org>, +# Isis Lovecruft <isis(a)torproject.org> # :licence: see LICENSE +# :copyright: 2012 Arturo Filasto, Isis Lovecruft +# :version: 0.1.0-alpha +# +# <------------------- import sys import os @@ -84,9 +92,9 @@ class TestCase(unittest.TestCase): ``ooniprobe mytest.py -c path/to/file.txt`` - * inputProcessor: should be set to a function that takes as argument an - open file descriptor and it will yield the input to be passed to the - test instance. + * inputParser: should be set to a function that takes as argument an + iterable containing test inputs and it simply parses those inputs and + returns them back to test instance. * name: should be set to the name of the test. @@ -123,12 +131,10 @@ class TestCase(unittest.TestCase): subclass of :class:`ooni.nettest.TestCase`, so that the calling functions in ooni.runner can handle them correctly. """ - methodName = 'runTest' - if kwargs: - if 'methodName' in kwargs: - methodName = kwargs['methodName'] - - super(TestCase, self).setUpClass() + if kwargs and 'methodName' in kwargs: + return super( TestCase, self ).setUpClass( + methodName=kwargs['methodName'] ) + return super( TestCase, self ).setUpClass( ) #for key, value in kwargs.items(): # setattr(self.__class__, key, value) @@ -176,7 +182,7 @@ class TestCase(unittest.TestCase): processor = [] else: log.msg("Received direct inputs:\n%s" % self.inputs) - processor = [i for i in self.inputProcessor(self.inputs)] + processor = [i for i in self.inputParser(self.inputs)] if self.inputFile: try: diff --git a/ooni/runner.py b/ooni/runner.py index 0e5dbba..f10f9ca 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -242,19 +242,22 @@ def loadTestsAndOptions(classes, config): log.debug("loadTestsAndOptions(): test %s found cases=%s" % (tests, cases)) - try: - opts = processTestOptions(klass, config) - except AttributeError, ae: - opts = {}; log.err(ae) - finally: + if hasattr(klass, 'optParameters') or hasattr(klass, 'inputFile'): try: + opts = processTestOptions(klass, config) + print opts, config + except: + opts = {} + finally: instance = klass() - inputs = instance._getInputs() - except Exception, e: - inputs = []; log.err(e) - else: - opts.update({'inputs': inputs}) - options.append(opts) + try: + inputs = instance._getInputs() + except Exception, e: + inputs = []; log.err(e) + else: + opts.update({'inputs': inputs}) + finally: + options.append(opts) log.debug("loadTestsAndOptions(): type(opts)=%s" % type(opts)) return test_cases, options

1 0

[ooni-probe/master] * Cleaned up a statement printed on ImportError when dnspython is not
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 7ef7b51b1d76878d33767876ff4e9c0a4dd70fc2 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Oct 23 11:45:08 2012 +0000 * Cleaned up a statement printed on ImportError when dnspython is not installed. * TODO Fix the Resolver in twisted.python.names so that we can use it instead of dnspython. --- nettests/core/captiveportal.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/nettests/core/captiveportal.py b/nettests/core/captiveportal.py index 6eed04d..46f0856 100644 --- a/nettests/core/captiveportal.py +++ b/nettests/core/captiveportal.py @@ -25,7 +25,8 @@ from ooni.utils import log try: from dns import resolver except ImportError: - print "The dnspython module was not found. https://crate.io/packages/dnspython/" + print "The dnspython module was not found:" + print "See https://crate.io/packages/dnspython/" resolver = None __plugoo__ = "captiveportal"

1 0

[ooni-probe/master] * Added a basic ping (ICMP8) test.
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit aba350538e34fbbc8aa6be91bc95eca21dfc301c Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Oct 23 11:50:08 2012 +0000 * Added a basic ping (ICMP8) test. --- ooni/echo.py | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 129 insertions(+), 0 deletions(-) diff --git a/ooni/echo.py b/ooni/echo.py new file mode 100644 index 0000000..1417622 --- /dev/null +++ b/ooni/echo.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# +---------+ +# | echo.py | +# +---------+ +# A simply ICMP-8 ping test. +# +# :author: Isis Lovecruft +# :version: 0.0.1-pre-alpha +# :license: (c) 2012 Isis Lovecruft +# see attached LICENCE file +# + +import os +import sys + +from twisted.plugin import IPlugin +from twisted.python import usage +from zope.interface import implements + +from ooni.lib import txscapy +from ooni.utils import log +from ooni.plugoo.assets import Asset +from ooni.plugoo.interface import ITest +from ooni.protocols.scapyproto import ScapyTest + +from ooni import nettest + +class EchoOptions(nettest.): + optParameters = [ + ['interface', 'i', None, 'Network interface to use'], + ['destination', 'd', None, 'File of hosts to ping'], + ['count', 'c', 5, 'Number of packets to send', int], + ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], + ['ttl', 't', 25, 'Set the IP Time to Live', int], + ] + optFlags = [] + +class EchoAsset(Asset): + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + if line.startswith('#'): + return + else: + return line.replace('\n', '') + +class EchoTest(ScapyTest): + implements(IPlugin, ITest) + + shortName = 'echo' + description = 'A simple ICMP-8 test to check if a host is reachable' + options = EchoOptions + requirements = None + blocking = False + + pcap_file = 'echo.pcap' + receive = True + + def initialize(self): + self.request = {} + self.response = {} + + if self.local_options: + + options = self.local_options + + if options['interface']: + self.interface = options['interface'] + + if options['count']: + ## there's a Counter() somewhere, use it + self.count = options['count'] + + if options['size']: + self.size = options['size'] + + if options['ttl']: + self.ttl = options['ttl'] + + def load_assets(self): + assets = {} + option = self.local_options + + if option and option['destination']: + + try: + from scapy.all import IP + except: + log.err() + + if os.path.isfile(option['destination']): + with open(option['destination']) as hosts: + for line in hosts.readlines(): + assets.update({'host': EchoAsset(line)}) + else: + while type(options['destination']) is str: + try: + IP(options['destination']) + except: + log.err() + break + assets.update({'host': options['destination']}) + else: + log.msg("Couldn't understand destination option...") + log.msg("Give one IPv4 address, or a file with one address per line.") + return assets + + def experiment(self, args): + if len(args) == 0: + log.err("Error: We're Echo, not Narcissus!") + log.err(" Provide a list of hosts to ping...") + d = sys.exit(1) + return d + + ## XXX v4 / v6 + from scapy.all import ICMP, IP, sr + ping = sr(IP(dst=args)/ICMP()) + if ping: + self.response.update(ping.show()) + else: + log.msg('No response received from %s' % args) + + def control(self, *args): + pass + +echo = EchoTest(None, None, None)

1 0

[ooni-probe/master] * Added a couple basic Exception classes to ooni.utils.net
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 187356a5ff61985b8691615334fa124ecd61ceab Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 16:00:39 2012 +0000 * Added a couple basic Exception classes to ooni.utils.net --- ooni/utils/net.py | 25 +++++++++++++++++++++++++ 1 files changed, 25 insertions(+), 0 deletions(-) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 218a74a..46c29e5 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -1,4 +1,29 @@ +# -*- encoding: utf-8 -*- +# +# net.py +# -------- +# OONI utilities for network infrastructure and hardware. +# +# :authors: Isis Lovecruft, Arturo Filasto +# :version: 0.0.1-pre-alpha +# :license: (c) 2012 Isis Lovecruft, Arturo Filasto +# see attached LICENCE file + + def getClientAddress(): address = {'asn': 'REPLACE_ME', 'ip': 'REPLACE_ME'} return address + + +class PermissionsError(SystemExit): + def __init__(self, *args, **kwargs): + if not args and not kwargs: + pe = "This test requires admin or root privileges to run. Exiting..." + super(PermissionsError, self).__init__(pe, *args, **kwargs) + else: + super(PermissionsError, self).__init__(*args, **kwargs) + +class IfaceError(SystemExit): + def __init__(self, *args, **kwargs): + super(IfaceError, self).__init__(*args, **kwargs)

1 0