December 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Updated .gitignore from copy of old repo
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit ac95afc153037a804dbcaf8a510176b7517e76fe Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Dec 6 17:59:26 2012 +0000 Updated .gitignore from copy of old repo --- .gitignore | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore index d1015ab..4cb0775 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ pcaps/ report*.yaml docs/build/* data/*.dat +ooniprobe.conf

1 0

[ooni-probe/master] Fixed error with reporting results for tcp flags tests, and cleaned up test
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 9688f83110a7d8986e20d665b93f4acbf6ba40f6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Dec 9 23:13:42 2012 +0000 Fixed error with reporting results for tcp flags tests, and cleaned up test abort code. --- nettests/bridge_reachability/tcpsyn.py | 156 ++++++++++++++------------------ ooni/nettest.py | 77 +++++++--------- ooni/reporter.py | 3 +- ooni/runner.py | 62 ++++++++----- 4 files changed, 140 insertions(+), 158 deletions(-) diff --git a/nettests/bridge_reachability/tcpsyn.py b/nettests/bridge_reachability/tcpsyn.py index 6a4b8db..79b8e52 100644 --- a/nettests/bridge_reachability/tcpsyn.py +++ b/nettests/bridge_reachability/tcpsyn.py @@ -16,32 +16,29 @@ import os import sys -from ipaddr import IPAddress from twisted.python import usage from twisted.python.failure import Failure -from twisted.internet import reactor, defer, address +from twisted.internet import reactor, defer from ooni import nettest, config from ooni.utils import net, log from ooni.utils.otime import timestamp try: - from scapy.all import TCP, IP + from scapy.all import TCP, IP, sr from ooni.utils import txscapy except: log.msg("This test requires scapy, see www.secdev.org/projects/scapy") -class TCPFlagOptions(usage.Options): +class TCPFlagsOptions(usage.Options): """Options for TCPTest.""" optParameters = [ ['dst', 'd', None, 'Host IP to ping'], ['port', 'p', None, 'Host port'], - ['flags', 's', None, 'Comma separated flags to set [S|A|F]'], + ['flags', 's', 'S', 'Comma separated flags to set, eg. "SA"'], ['count', 'c', 3, 'Number of SYN packets to send', int], ['interface', 'i', None, 'Network interface to use'], - ['hexdump', 'x', False, 'Show hexdump of responses'], - ['pdf', 'y', False, - 'Create pdf of visual representation of packet conversations']] + ['hexdump', 'x', False, 'Show hexdump of responses']] class TCPFlagsTest(nettest.NetTestCase): """ @@ -57,29 +54,28 @@ class TCPFlagsTest(nettest.NetTestCase): version = '0.1.1' requiresRoot = True - usageOptions = TCPFlagOptions + usageOptions = TCPFlagsOptions inputFile = ['file', 'f', None, 'File of list of IP:PORTs to ping'] + destinations = {} + def setUp(self, *a, **kw): """Configure commandline parameters for TCPSynTest.""" - self.report = {} - self.packets = {'results': [], 'unanswered': []} - if self.localOptions: for key, value in self.localOptions.items(): setattr(self, key, value) if not self.interface: try: iface = net.getDefaultIface() - except net.IfaceError, ie: - log.warn("Could not find a working network interface!") - log.fail(ie) - else: self.interface = iface + except net.IfaceError: + self.abortClass("Could not find a working network interface!") + if self.flags: + self.flags = self.flags.split(',') if config.advanced.debug: defer.setDebugging('on') - def addToDestinations(self, addr='0.0.0.0', port='443'): + def addToDestinations(self, addr=None, port='443'): """ Validate and add an IP address and port to the dictionary of destinations to send to. If the host's IP is already in the @@ -89,12 +85,16 @@ class TCPFlagsTest(nettest.NetTestCase): @param port: A string representing a port number. @returns: A 2-tuple containing the address and port. """ + if addr is None: + return (None, None) # do we want to return SkipTest? + dst, dport = net.checkIPandPort(addr, port) - if not dst in self.report.keys(): - self.report[dst] = {'dst': dst, 'dport': [dport]} + if not dst in self.destinations.keys(): + self.destinations[dst] = {'dst': dst, + 'dport': [dport]} else: log.debug("Got additional port for destination.") - self.report[dst]['dport'].append(dport) + self.destinations[dst]['dport'].append(dport) return (dst, dport) def inputProcessor(self, input_file=None): @@ -118,7 +118,7 @@ class TCPFlagsTest(nettest.NetTestCase): raw_ip, raw_port = one.rsplit(':', 1) yield self.addToDestinations(raw_ip, raw_port) - def tcp_flags(self, flags=None): + def test_tcp_flags(self): """ Generate, send, and listen for responses to, a list of TCP/IP packets to an address and port pair taken from the current input, and a string @@ -128,11 +128,14 @@ class TCPFlagsTest(nettest.NetTestCase): A string representing the TCP flags to be set, i.e. "SA" or "F". Defaults to "S". """ - def build_packets(addr, port, flags=None, count=3): + def build_packets(addr, port): """Construct a list of packets to send out.""" packets = [] - for x in xrange(count): - packets.append( IP(dst=addr)/TCP(dport=port, flags=flags) ) + for flag in self.flags: + log.debug("Generating packets with %s flags for %s:%d..." + % (flag, addr, port)) + for x in xrange(self.count): + packets.append( IP(dst=addr)/TCP(dport=port, flags=flag) ) return packets def process_packets(packet_list): @@ -144,72 +147,45 @@ class TCPFlagsTest(nettest.NetTestCase): @param packet_list: A :class:scapy.plist.PacketList """ + log.msg("Processing received packets...") results, unanswered = packet_list - self.packets['results'].append([r for r in results]) - self.packets['unanswered'].append([u for u in unanswered]) - + for (q, r) in results: - request_data = {'dst': q.dst, - 'dport': q.dport, - 'summary': q.summary(), - 'command': q.command(), - 'hexdump': None, - 'sent_time': q.time} - response_data = {'src': r['IP'].src, - 'flags': r['IP'].flags, - 'summary': r.summary(), - 'command': r.command(), - 'hexdump': None, - 'recv_time': r.time, - 'delay': r.time - q.time} + request = {'dst': q.dst, + 'dport': q.dport, + 'summary': q.summary(), + 'hexdump': None, + 'sent_time': q.time} + response = {'src': r['IP'].src, + 'flags': r['IP'].flags, + 'summary': r.summary(), + 'hexdump': None, + 'recv_time': r.time, + 'delay': r.time - q.time} if self.hexdump: - request_data.update('hexdump', q.hexdump()) - response_data.update('hexdump', r.hexdump()) - - for dest, data in self.report.items(): - if data['dst'] == response_data['src']: - if not 'reachable' in data: - if self.hexdump: - log.msg("%s\n%s" % (q.hexdump(), r.hexdump())) - else: - log.msg(" Received response:\n%s ==> %s" - % (q.mysummary(), r.mysummary())) - data.update( {'reachable': True, - 'request': request_data, - 'response': response_data} ) - self.report[response_data['src']['data'].update(data) - - if unanswered is not None and len(unanswered) > 0: - log.msg("Waiting on responses from\n%s" % - '\n'.join( [unans.summary() for unans in unanswered] )) - log.msg("Writing response packet information to report...") - - (addr, port) = self.input - packets = build_packets(addr, port, str(flags), self.count) - d = txscapy.sr(packets, iface=self.interface) - #d.addCallbacks(process_packets, log.exception) - #d.addCallbacks(process_unanswered, log.exception) - d.addCallback(process_packets) - d.addErrback(process_unanswered) - - return d - - @log.catch - def createPDF(self): - pdfname = self.name + '_' + timestamp() - self.packets['results'].pdfdump(pdfname) - log.msg("Visual packet conversation saved to %s.pdf" % pdfname) - - def test_tcp_flags(self): - """Send packets with given TCP flags to an address:port pair.""" - flag_list = self.flags.split(',') - - dl = [] - for flag in flag_list: - dl.append(self.tcp_flags(flag)) - d = defer.DeferredList(dl) - - if self.pdf: - d.addCallback(self.createPDF) - - return d + request['hexdump'] = q.hexdump() + response['hexdump'] = r.hexdump() + + for dest, data in self.destinations.items(): + if response['src'] == data['dst']: + log.msg(" Received response from %s:\n%s ==> %s" % ( + response['src'], q.mysummary(), r.mysummary())) + if self.hexdump: + log.msg("%s\n%s" % (q.hexdump(), r.hexdump())) + + self.report['request'] = request + self.report['response'] = response + + if unanswered is not None: + unans = [un.summary() for un in unanswered] + log.msg(" Waiting on responses from:\n%s" % '\n'.join(unans)) + self.report['unanswered'] = unans + + try: + self.report = {} + (addr, port) = self.input + pkts = build_packets(addr, port) + d = process_packets(sr(pkts, iface=self.interface, timeout=5)) + return d + except Exception, ex: + log.exception(ex) diff --git a/ooni/nettest.py b/ooni/nettest.py index bd6ef9b..29ced70 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -12,9 +12,11 @@ import sys import os import itertools import traceback +import inspect from twisted.trial import unittest, itrial from twisted.trial import util as txtrutil +from twisted.trial.test import skipping from twisted.internet import defer, utils from twisted.python import usage @@ -171,59 +173,46 @@ class NetTestCase(object): def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) - def _abort(self, reason): - """ - - Abort running the current input. Raises - :class:`twisted.trial.test.skipping.SkipTest <SkipTest>` test_method, - or test_class. If called with only one argument, assume we're going to - ignore the current input. Otherwise, the name of the method or class - in relation to the test_instance, i.e. "self" should be given as value - for the keyword argument "obj". + def _abortMethod(self, reason, method=None): + if method is None: + test_method = self._testMethod + else: + test_method = getattr(self.__class__, method, False) - XXX call oreporter.allDone() from parent stack frame - """ - reason = str(reason) - raise SkipTest("%s\n%s" % (str(reason), str(self.input)) ) - - def _abortMethod(self, reason, method): - if inspect.ismethod(method): - abort = getattr(self.__class__, method, False) - log.debug("Aborting remaining inputs for %s" % str(abort.func_name)) - setattr(abort, 'skip', reason) + if inspect.ismethod(test_method): + method_name = test_method.im_func.func_name + setattr(test_method, 'skip', reason) + raise skipping.SkipTest("Aborting %s for reason: %s" + % (method_name, reason) ) else: - log.debug("abortMethod(): could not find method %s" % str(method)) + log.debug("_abortMethod(): could not find method %s" % test_method) - @log.catch - def _abortClass(self, reason, cls): - if not inspect.isclass(obj) or not isTestCase(obj): - log.debug("_abortClass() could not find class %s" % str(cls)) - return - abort = getattr(obj, '__class__', self.__class__) - log.debug("Aborting %s test" % str(abort.name)) - setattr(abort, 'skip', reason) - - def abortCurrentInput(self, reason): + def abortInput(self, reason): """ Abort the current input. @param reason: A string explaining why this test is being skipped. + @raises: A :class:`twisted.trial.test.skipping.SkipTest <SkipTest>` """ - return self._abort(reason) - - def abortInput(self, reason): - return self._abort(reason) + raise skipping.SkipTest(" Reason: %s\nCurrent input: %s" + % (reason, self.input)) + def abortMethod(self, reason, test_method=None): + """ + Abort all remaining inputs for the current test method. -# This needs to be here so that NetTestCase.abort() can call it, since we -# cannot import runner because runner imports NetTestCase. -def isTestCase(obj): - """ - Return True if obj is a subclass of NetTestCase, false if otherwise. - """ - try: - return issubclass(obj, NetTestCase) - except TypeError: - return False + @param reason: A string explaining why the current test_method is + being skipped. + @param test_method: (optional) The test_method to skip, defaults to + the currently running test_method. + """ + return self._abortMethod(reason, test_method) + def abortClass(self, reason='unspecified'): + """ + Abort the entire NetTestCase class. + @param reason: A string explaining why the class is being skipped. + """ + log.msg("Aborting %s: %s" % (self.__class__.name, reason)) + setattr(self.__class__, 'skip', reason) diff --git a/ooni/reporter.py b/ooni/reporter.py index 6fdc142..133b98d 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -218,7 +218,8 @@ class OONIBReporter(OReporter): response = yield self.agent.request("PUT", url, bodyProducer=bodyProducer) except: - # XXX we must trap this in the runner and make sure to report the data later. + # XXX we must trap this in the runner and make sure to report the + # data later. raise OONIBReportUpdateFailed #parsed_response = json.loads(backend_response) diff --git a/ooni/runner.py b/ooni/runner.py index 4214360..5969dd5 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -16,7 +16,7 @@ import traceback import itertools from twisted.python import reflect, usage, failure -from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.trial.runner import filenameToModule from twisted.trial import reporter as txreporter from twisted.trial import util as txtrutil @@ -28,6 +28,15 @@ from ooni.inputunit import InputUnitFactory from ooni import reporter, nettest from ooni.utils import log, checkForRoot, PermissionsError +def isTestCase(obj): + """ + Return True if obj is a subclass of NetTestCase, false if otherwise. + """ + try: + return issubclass(obj, nettest.NetTestCase) + except TypeError: + return False + def processTest(obj, cmd_line_options): """ Process the parameters and :class:`twisted.python.usage.Options` of a @@ -112,7 +121,7 @@ def findTestClassesFromConfig(cmd_line_options): module = filenameToModule(filename) for name, val in inspect.getmembers(module): - if nettest.isTestCase(val): + if isTestCase(val): classes.append(processTest(val, cmd_line_options)) return classes @@ -185,13 +194,19 @@ def runTestWithInput(test_class, test_method, test_input, oreporter): d.errback(timeout_fail) except defer.AlreadyCalledError: # if the deferred has already been called but the *back chain is - # still unfinished, crash the reactor and report the timeout + # still unfinished, safely crash the reactor and report the timeout reactor.crash() test_instance._timedOut = True # see test_instance._wait test_instance._test_result.addExpectedFailure(test_instance, fail) test_timeout = txtrutils.suppressWarnings( test_timeout, txtrutil.suppress(category=DeprecationWarning)) + def test_skip_class(reason): + try: + d.errback(failure.Failure(SkipTest("%s" % reason))) + except defer.AlreadyCalledError: + pass # XXX not sure what to do here... + def test_done(result, test_instance, test_name): log.debug("Concluded %s with inputs %s" % (test_name, test_instance.input)) @@ -199,10 +214,7 @@ def runTestWithInput(test_class, test_method, test_input, oreporter): def test_error(error, test_instance, test_name): if isinstance(error, SkipTest): - if len(error.args) > 0: - skip_what = error.args[1] - # XXX we'll need to handle methods and classes - log.info("%s" % error.message) + log.warn("%s" % error.message) else: log.exception(error) @@ -219,26 +231,30 @@ def runTestWithInput(test_class, test_method, test_input, oreporter): test_instance._setUp() test_instance.setUp() - test_skip = txtrutil.acquireAttribute( - test_instance._parents, 'skip', None) - if test_skip is not None: - # XXX we'll need to do something more than warn + test_skip = txtrutil.acquireAttribute(test_instance._parents, 'skip', None) + if test_skip: log.warn("%s marked these tests to be skipped: %s" - % (test_instance.name, test_skip)) + % (test_instance.name, test_skip)) skip_list = [test_skip] - if not test_method in skip_list: - test = getattr(test_instance, test_method) - d = defer.maybeDeferred(test) + test = getattr(test_instance, test_method) + test_instance._testMethod = test + + d = defer.maybeDeferred(test) + + # register the timer with the reactor + call_timeout = reactor.callLater(test_instance.timeout, test_timeout, d) + d.addBoth(lambda x: call_timeout.active() and call_timeout.cancel() or x) + + # check if the class has been aborted + if hasattr(test_instance.__class__, 'skip'): + reason = getattr(test_instance.__class__, 'skip') + call_skip = reactor.callLater(0, test_skip_class, reason) + d.addBoth(lambda x: call_skip.active() and call_skip.cancel() or x) + + d.addCallback(test_done, test_instance, test_method) + d.addErrback(test_error, test_instance, test_method) - # register the timer with the reactor - call = reactor.callLater(test_instance.timeout, test_timeout, d) - d.addBoth(lambda x: call.active() and call.cancel() or x) - - d.addCallback(test_done, test_instance, test_method) - d.addErrback(test_error, test_instance, test_method) - else: - d = defer.Deferred() return d def runTestWithInputUnit(test_class, test_method, input_unit, oreporter):

1 0

[ooni-probe/master] Merge remote-tracking branch 'tpo-common/master'
by isis＠torproject.org 18 Dec '12

18 Dec '12

1 0

[ooni-probe/master] Cleaned up a couple of the comments in ooniprobe.conf.sample explaining the
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 252988c853ca5fd90a50efec7de539f84f184297 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 12 15:52:07 2012 +0000 Cleaned up a couple of the comments in ooniprobe.conf.sample explaining the config options. --- ooniprobe.conf.sample | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooniprobe.conf.sample b/ooniprobe.conf.sample index 7031ba1..67da610 100644 --- a/ooniprobe.conf.sample +++ b/ooniprobe.conf.sample @@ -22,9 +22,9 @@ advanced: # /path/to/ooni-probe/data/ geoip_data_dir: /usr/share/GeoIP/ debug: true - # If we're including the client's IP address, how long should we wait for - # the connection to complete before timing out? + # How many seconds should we wait for connections before timing out?: checktimeout: 15 + # Location where Tor is installed: tor_binary: '/usr/sbin/tor' # For auto detection interface: auto

1 0

[ooni-probe/master] Fixed a bug where ooni.runner.processTest() expected an unnecessary
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit fef56442ae930470c7160006dac72339c4af98fa Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 12 19:14:45 2012 +0000 Fixed a bug where ooni.runner.processTest() expected an unnecessary parameter. Also fixed a TypeError due to string formatting in runTestCasesWithInputUnit(), and a couple typos for method calls. * Fixed an "except NoPostProcessor" which should have been "except nettest.NoPostProcessor". --- ooni/runner.py | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 4724d3d..6f03e56 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -21,7 +21,7 @@ from twisted.internet import defer, reactor, threads from twisted.trial import reporter as txreporter from twisted.trial import util as txutil from twisted.trial.runner import filenameToModule -from twisted.trial.unittest import utils as txtrutils +from twisted.trial.unittest import utils as txutils from twisted.trial.unittest import SkipTest from txtorcon import TorProtocolFactory, TorConfig @@ -70,7 +70,7 @@ def checkRequiredOptions(test_instance): if not test_instance.localOptions[required_option]: raise usage.UsageError("%s not specified!" % required_option) -def processTest(obj, cmd_line_options): +def processTest(obj): """ Process the parameters and :class:`twisted.python.usage.Options` of a :class:`ooni.nettest.Nettest`. @@ -249,8 +249,8 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, reactor.crash() test_instance._timedOut = True # see test_instance._wait test_instance._test_result.addExpectedFailure(test_instance, fail) - test_timeout = txtrutils.suppressWarnings( - test_timeout, txtrutil.suppress(category=DeprecationWarning)) + test_timeout = txutils.suppressWarnings( + test_timeout, txutil.suppress(category=DeprecationWarning)) def test_skip_class(reason): try: @@ -289,7 +289,7 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, d1 = oonib_reporter.testDone(test_instance, 'summary') d2 = yaml_reporter.testDone(test_instance, 'summary') return defer.DeferredList([d1, d2]) - except NoPostProcessor: + except nettest.NoPostProcessor: log.debug("No post processor configured") return @@ -329,7 +329,7 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, else: reason = txutil.acquireAttribute(test_instance._parents, 'skip', None) log.warn("%s marked some tests to be skipped. Reason: %s" - % (test_instance.name, test_skip)) + % (test_instance.name, reason)) if reason is not None: call_skip = reactor.callLater(0, test_skip_class, reason) d.addBoth(lambda x: call_skip.active() and call_skip.cancel() or x) @@ -361,7 +361,7 @@ def runTestCasesWithInputUnit(test_cases, input_unit, yaml_reporter, """ dl = [] for test_input in input_unit: - log.debug("Running test with this input %s" % test_input) + log.debug("Running test with this input %s" % str(test_input)) d = runTestCasesWithInput(test_cases, test_input, yaml_reporter, oonib_reporter) dl.append(d)

1 0

[ooni-probe/master] Cleaned up logic for the reactor to callback to test_skip_class() in
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit e5d1a84030ca5267015e3c8e33d6b3a419da2727 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 12 18:08:16 2012 +0000 Cleaned up logic for the reactor to callback to test_skip_class() in ooni.runner.runTestCaseWithInput(). * Moved Exceptions classes to beginning of file. * Fixed a raise statement which called an Exception that doesn't exist anymore. * Cleaned up documentation, imports, code style, and indentation. --- ooni/runner.py | 145 ++++++++++++++++++++++++++++--------------------------- 1 files changed, 74 insertions(+), 71 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 58ec188..306c1d9 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -18,9 +18,9 @@ import yaml from twisted.python import reflect, usage, failure from twisted.internet import defer, reactor, threads -from twisted.trial.runner import filenameToModule from twisted.trial import reporter as txreporter -from twisted.trial import util as txtrutil +from twisted.trial import util as txutil +from twisted.trial.runner import filenameToModule from twisted.trial.unittest import utils as txtrutils from twisted.trial.unittest import SkipTest @@ -28,16 +28,28 @@ from txtorcon import TorProtocolFactory, TorConfig from txtorcon import TorState, launch_tor from ooni import config, nettest, reporter -from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError from ooni.inputunit import InputUnitFactory +from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError from ooni.utils import log, checkForRoot from ooni.utils import PermissionsError, Storage from ooni.utils.net import randomFreePort + +class NoTestCasesFound(Exception): + pass + +class InvalidResumeFile(Exception): + pass + +class noResumeSession(Exception): + pass + +class InvalidConfigFile(Exception): + message = "Invalid setting in ooniprobe.conf: " + + def isTestCase(obj): - """ - Return True if obj is a subclass of NetTestCase, false if otherwise. - """ + """Return True if obj is a subclass of NetTestCase, False otherwise.""" try: return issubclass(obj, nettest.NetTestCase) except TypeError: @@ -55,14 +67,7 @@ def processTest(obj, cmd_line_options): :param cmd_line_options: A configured and instantiated :class:`twisted.python.usage.Options` class. - """ - if not hasattr(obj.usageOptions, 'optParameters'): - obj.usageOptions.optParameters = [] - - if obj.inputFile: - obj.usageOptions.optParameters.append(obj.inputFile) - if obj.requiresRoot: try: checkForRoot() @@ -70,37 +75,40 @@ def processTest(obj, cmd_line_options): log.err("%s requires root to run" % obj.name) sys.exit(1) + if not hasattr(obj.usageOptions, 'optParameters'): + obj.usageOptions.optParameters = [] + if obj.baseParameters: for parameter in obj.baseParameters: obj.usageOptions.optParameters.append(parameter) - if obj.baseFlags: if not hasattr(obj.usageOptions, 'optFlags'): obj.usageOptions.optFlags = [] for flag in obj.baseFlags: obj.usageOptions.optFlags.append(flag) + if obj.inputFile: # inputFile is the optParameters list + obj.usageOptions.optParameters.append(obj.inputFile) options = obj.usageOptions() - options.parseOptions(config.cmd_line_options['subargs']) + obj.localOptions = options - if obj.inputFile: + if obj.inputFile: # inputFilename is the actual filename obj.inputFilename = options[obj.inputFile[0]] try: - log.debug("processing options") + log.debug("Parsing commandline options") tmp_test_case_object = obj() tmp_test_case_object._checkRequiredOptions() - - except usage.UsageError, e: - test_name = tmp_test_case_object.name - log.err("There was an error in running %s!" % test_name) - log.err("%s" % e) + except usage.UsageError, ue: + log.err("%s" % ue) options.opt_help() - raise usage.UsageError("Error in parsing command line args for %s" % test_name) - return obj + raise usage.UsageError("Error parsing command line args for %s" + % tmp_test_case_object.name) + else: + return obj def findTestClassesFromFile(filename): """ @@ -131,9 +139,6 @@ def makeTestCases(klass, tests, method_prefix): cases.append((klass, method_prefix+test)) return cases -class NoTestCasesFound(Exception): - pass - def loadTestsAndOptions(classes, cmd_line_options): """ Takes a list of test classes and returns their testcases and options. @@ -155,31 +160,43 @@ def loadTestsAndOptions(classes, cmd_line_options): return test_cases, options -def getTimeout(test_instance, test_method): +def getTestTimeout(test_instance, test_method): """ + Returns the timeout value set on this test. Check on the instance first, the the class, then the module, then package. As soon as it finds - something with a timeout attribute, returns that. Returns - twisted.trial.util.DEFAULT_TIMEOUT_DURATION if it cannot find anything. + something with a timeout attribute, returns that. Returns the value set in + ooniprobe.conf, :attr:`ooni.config.advanced.default_timeout + <default_timeout>` if it cannot find anything. See twisted.trial.unittest.TestCase docstring for more details. + + @param test_instance: + The instance of a :class:`ooni.nettest.NetTestCase` currently running. + @param test_method: + The test_instance.test_method currently being processed. """ + default = config.advanced.default_timeout + try: - testMethod = getattr(test_instance, test_method) + tm = getattr(test_instance, test_method) except: - log.debug("_getTimeout couldn't find self.methodName!") - return txtrutil.DEFAULT_TIMEOUT_DURATION + log.debug("runner.getTestTimeout() couldn't find %s.%s!" + % (test_instance, test_method)) + try: + return float(default) + except (ValueError, TypeError): + raise InvalidConfigFile("'default_timeout' must be a number!") else: - test_instance._parents = [testMethod, test_instance] - test_instance._parents.extend(txtrutil.getPythonContainers(testMethod)) - timeout = txtrutil.acquireAttribute(test_instance._parents, 'timeout', - txtrutil.DEFAULT_TIMEOUT_DURATION) + test_instance._parents = [tm, test_instance] + test_instance._parents.extend(txutil.getPythonContainers(tm)) + timeout = txutil.acquireAttribute( + test_instance._parents, 'timeout', default) try: return float(timeout) except (ValueError, TypeError): - warnings.warn("'timeout' attribute needs to be a number.", - category=DeprecationWarning) - return txtrutil.DEFAULT_TIMEOUT_DURATION + log.warn("'timeout' attribute must be a number!") + return float(default_timeout) def runTestCasesWithInput(test_cases, test_input, yaml_reporter, oonib_reporter=None): @@ -204,7 +221,6 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, this is set to none then we will only report to the YAML reporter. """ - # This is used to store a copy of all the test reports tests_report = {} @@ -267,31 +283,24 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, dl = [] for test_case in test_cases: - log.debug("Processing %s" % test_case[1]) test_class = test_case[0] test_method = test_case[1] + log.debug("%s: Setting up: %s" % (test_class.name, test_method)) + test_instance = test_class() test_instance.input = test_input test_instance.report = {} - # XXX TODO the twisted.trial.reporter.TestResult is expected by - # test_timeout(), but we should eventually replace it with a stub class + # XXX txreporter.TestResult is expected by test_timeout(), but we + # should eventually replace it with a stub class test_instance._test_result = txreporter.TestResult() - # use this to keep track of the test runtime test_instance._start_time = time.time() - test_instance.timeout = getTimeout(test_instance, test_method) - # call setups on the test test_instance._setUp() test_instance.setUp() - - # check if we're inherited from anything marked to be skipped - test_skip = txtrutil.acquireAttribute(test_instance._parents, 'skip', None) - if test_skip: - log.warn("%s marked these tests to be skipped: %s" - % (test_instance.name, test_skip)) - skip_list = [test_skip] + # get the timeout and _parents, in case it was set in setUp() + test_instance.timeout = getTestTimeout(test_instance, test_method) test = getattr(test_instance, test_method) test_instance._testMethod = test @@ -302,9 +311,14 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, call_timeout = reactor.callLater(test_instance.timeout, test_timeout, d) d.addBoth(lambda x: call_timeout.active() and call_timeout.cancel() or x) - # check if the class has been aborted + # check if anything has been aborted or marked as 'skip' if hasattr(test_instance.__class__, 'skip'): reason = getattr(test_instance.__class__, 'skip') + else: + reason = txutil.acquireAttribute(test_instance._parents, 'skip', None) + log.warn("%s marked some tests to be skipped. Reason: %s" + % (test_instance.name, test_skip)) + if reason is not None: call_skip = reactor.callLater(0, test_skip_class, reason) d.addBoth(lambda x: call_skip.active() and call_skip.cancel() or x) @@ -316,8 +330,8 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, test_methods_d.addCallback(tests_done, test_cases[0][0]) return test_methods_d -def runTestCasesWithInputUnit(test_cases, input_unit, yaml_reporter, - oonib_reporter): +def runTestCasesWithInputUnit(test_cases, input_unit, yaml_reporter, + oonib_reporter): """ Runs the Test Cases that are given as input parallely. A Test Case is a subclass of ooni.nettest.NetTestCase and a list of @@ -341,12 +355,6 @@ def runTestCasesWithInputUnit(test_cases, input_unit, yaml_reporter, dl.append(d) return defer.DeferredList(dl) -class InvalidResumeFile(Exception): - pass - -class noResumeSession(Exception): - pass - def loadResumeFile(): """ Sets the singleton stateDict object to the content of the resume file. @@ -355,7 +363,6 @@ def loadResumeFile(): Raises: :class:ooni.runner.InvalidResumeFile if the resume file is not valid - """ if not config.stateDict: try: @@ -391,7 +398,6 @@ def resumeTest(test_filename, input_unit_factory): :class:ooni.inputunit.InputUnitFactory that is at the index of the previous test run. - """ try: idx = config.stateDict[test_filename] @@ -412,7 +418,7 @@ def resumeTest(test_filename, input_unit_factory): @defer.inlineCallbacks def updateResumeFile(test_filename): """ - update the resume file with the current stateDict state. + Update the resume file with the current stateDict state. """ log.debug("Acquiring lock for %s" % test_filename) yield config.resume_lock.acquire() @@ -434,16 +440,13 @@ def increaseInputUnitIdx(test_filename): including the .py extension. input_unit_idx (int): the current input unit index for the test. - """ config.stateDict[test_filename] += 1 yield updateResumeFile(test_filename) def updateProgressMeters(test_filename, input_unit_factory, - test_case_number): - """ - Update the progress meters for keeping track of test state. - """ + test_case_number): + """Update the progress meters for keeping track of test state.""" log.msg("Setting up progress meters") if not config.state.test_filename: config.state[test_filename] = Storage()

1 0

[ooni-probe/master] Started fixing a bug in oonicli.updateStatusBar() and runner.runTestList()
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit a6bc4d53d3288b85b9d4a2712f94f4a4c3e4862d Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Dec 13 02:20:01 2012 +0000 Started fixing a bug in oonicli.updateStatusBar() and runner.runTestList() where the status bar hangs at 100%. It needs to be changed to use twisted.internet.task.Cooperator and t.i.t.CooperativeTask, which means that oonicli.updateStatusBar() still needs to be changed to be an iterator. --- ooni/oonicli.py | 54 ++++++++++++++++++++++++++++++++++++++++-------------- ooni/runner.py | 21 +++++++++------------ 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 9d4f783..0bf4d55 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -42,13 +42,14 @@ class Options(usage.Options): optFlags = [["help", "h"], ["resume", "r"]] - optParameters = [["reportfile", "o", None, "report file name"], - ["testdeck", "i", None, - "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], - ["collector", "c", None, - "Address of the collector of test results. (example: http://127.0.0.1:8888)"], - ["logfile", "l", None, "log file name"], - ["pcapfile", "p", None, "pcap file name"]] + optParameters = [ + ["reportfile", "o", None, "report file name"], + ["testdeck", "i", None, + "Specify a test deck: a yaml file containing tests and their arguments"], + ["collector", "c", None, + "Address of the collector of test results. (e.g.: http://127.0.0.1:8888)"], + ["logfile", "l", None, "log file name"], + ["pcapfile", "p", None, "pcap file name"]] compData = usage.Completions( extraActions=[usage.CompleteFiles( @@ -81,14 +82,34 @@ class Options(usage.Options): except: raise usage.UsageError("No test filename specified!") -def updateStatusBar(): +class CooperativeTimer(object): + """ + A simple timer for the callback to functions on + :class:`twisted.internet.task.Cooperator <t.i.t.Cooperator>`. see + :meth:`oonicli.runTestList <runTestList>`. + + @param seconds: + An integer specifying the second to wait in between updating the + status and ETA bars. + """ + def __init__(self, seconds=5): + self.max_timer_interval = float(seconds) + self.end = time.time() + self.max_timer_interval + + def __call__(self): + return time.time() >= self.end + +def updateStatusBar(stop_func): for test_filename in config.state.keys(): # The ETA is not updated so we we will not print it out for the # moment. eta = config.state[test_filename].eta() progress = config.state[test_filename].progress() - progress_bar_frmt = "[%s] %s%%" % (test_filename, progress) - print progress_bar_frmt + while progress is not None: + print "[%s] %s%%" % (test_filename, progress) + else: + print "[%s] All tests in file completed." % test_filename + stop_func() def testsEnded(*arg, **kw): """You can place here all the post shutdown tasks.""" @@ -102,7 +123,7 @@ def startSniffing(): from ooni.utils.txscapy import ScapyFactory, ScapySniffer try: checkForRoot() - except NotRootError: + except PermissionsError: print "[!] Includepcap options requires root priviledges to run" print " you should run ooniprobe as root or disable the options in ooniprobe.conf" sys.exit(1) @@ -130,9 +151,14 @@ def runTestList(none, test_list): d2 = defer.DeferredList(deck_dl) d2.addBoth(testsEnded) - # Print every 5 second the list of current tests running - l = task.LoopingCall(updateStatusBar) - l.start(5.0) + try: + # Print every 5 second the list of current tests running + coop = task.Cooperator(started=False) + coop.cooperate(updateStatusBar) #this will need a .next() method + coop.start() + except StopIteration: + return d2 + return d2 def errorRunningTests(failure): diff --git a/ooni/runner.py b/ooni/runner.py index 6f03e56..19dc171 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -328,9 +328,9 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, reason = getattr(test_instance.__class__, 'skip') else: reason = txutil.acquireAttribute(test_instance._parents, 'skip', None) + if reason is not None: log.warn("%s marked some tests to be skipped. Reason: %s" % (test_instance.name, reason)) - if reason is not None: call_skip = reactor.callLater(0, test_skip_class, reason) d.addBoth(lambda x: call_skip.active() and call_skip.cancel() or x) @@ -456,14 +456,14 @@ def increaseInputUnitIdx(test_filename): config.stateDict[test_filename] += 1 yield updateResumeFile(test_filename) -def updateProgressMeters(test_filename, input_unit_factory, - test_case_number): +def updateProgressMeters(test_filename, input_unit_factory, test_case_number): """Update the progress meters for keeping track of test state.""" log.msg("Setting up progress meters") if not config.state.test_filename: config.state[test_filename] = Storage() - config.state[test_filename].per_item_average = 2.0 + per_item_avg = float(2) + config.state[test_filename].per_item_average = per_item_avg input_unit_idx = float(config.stateDict[test_filename]) input_unit_items = float(len(input_unit_factory) + 1) @@ -471,27 +471,24 @@ def updateProgressMeters(test_filename, input_unit_factory, total_iterations = input_unit_items * test_case_number current_iteration = input_unit_idx * test_case_number - log.debug("input_unit_items: %s" % input_unit_items) - log.debug("test_case_number: %s" % test_case_number) - + log.debug("Total InputUnits: %s" % input_unit_items) log.debug("Test case number: %s" % test_case_number) log.debug("Total iterations: %s" % total_iterations) log.debug("Current iteration: %s" % current_iteration) def progress(): - return (current_iteration / total_iterations) * 100.0 - + current_progress = (current_iteration / total_iterations) * 100.0 + while float(current_progress) < float(100): + return current_progress config.state[test_filename].progress = progress def eta(): - return (total_iterations - current_iteration) \ - * config.state[test_filename].per_item_average + return (total_iterations - current_iteration) * per_item_avg config.state[test_filename].eta = eta config.state[test_filename].input_unit_idx = input_unit_idx config.state[test_filename].input_unit_items = input_unit_items - @defer.inlineCallbacks def runTestCases(test_cases, options, cmd_line_options): """

1 0

[ooni-probe/master] Removed an import line from ooni/reporter.py which tried to import classes
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 017f249efc57650510ca6e3eba2c9aa12c3bc7a5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 12 15:56:16 2012 +0000 Removed an import line from ooni/reporter.py which tried to import classes which are now contained in ooni/reporter.py. --- ooni/reporter.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index fc41960..37cf9d1 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -27,7 +27,6 @@ from twisted.internet.error import ConnectionRefusedError from ooni import config, otime from ooni.utils import log, geodata -from ooni.utils.hacks import OSafeRepresenter, OSafeDumper from ooni.utils.net import BodyReceiver, StringProducer, userAgents try:

1 0

[ooni-probe/master] Moved method NetTestCase._checkRequiredOptions() logic into a function in
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit d476d8c9c25c432170a0542bb4c3b3e43ad081e7 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 12 18:29:03 2012 +0000 Moved method NetTestCase._checkRequiredOptions() logic into a function in ooni/runner.py, because it was unnecessary and also more lines of code to put in nettest.py. * Also removed an extraneous call to _checkRequiredOptions. * Moved another exception to the top of ooni/runner.py. --- ooni/nettest.py | 14 +++----------- ooni/runner.py | 29 +++++++++++++++++++---------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 8fc29c3..cff5c62 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -180,12 +180,6 @@ class NetTestCase(object): else: pass - def _checkRequiredOptions(self): - for required_option in self.requiredOptions: - log.debug("Checking if %s is present" % required_option) - if not self.localOptions[required_option]: - raise usage.UsageError("%s not specified!" % required_option) - def _processOptions(self): if self.inputFilename: inputProcessor = self.inputProcessor @@ -199,11 +193,9 @@ class NetTestCase(object): return inputProcessor(inputFilename) self.inputs = inputProcessorIterator() - self._checkRequiredOptions() - - return {'inputs': self.inputs, - 'name': self.name, 'version': self.version - } + return {'inputs': self.inputs, + 'name': self.name, + 'version': self.version} def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) diff --git a/ooni/runner.py b/ooni/runner.py index 306c1d9..4724d3d 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -47,6 +47,9 @@ class noResumeSession(Exception): class InvalidConfigFile(Exception): message = "Invalid setting in ooniprobe.conf: " +class UnableToStartTor(Exception): + pass + def isTestCase(obj): """Return True if obj is a subclass of NetTestCase, False otherwise.""" @@ -55,6 +58,18 @@ def isTestCase(obj): except TypeError: return False +def checkRequiredOptions(test_instance): + """ + If test_instance has an attribute 'requiredOptions', then check that + those options were utilised on the commandline. + """ + required = getattr(test_instance, 'requiredOptions', None) + if required: + for required_option in required: + log.debug("Checking if %s is present" % required_option) + if not test_instance.localOptions[required_option]: + raise usage.UsageError("%s not specified!" % required_option) + def processTest(obj, cmd_line_options): """ Process the parameters and :class:`twisted.python.usage.Options` of a @@ -91,7 +106,6 @@ def processTest(obj, cmd_line_options): options = obj.usageOptions() options.parseOptions(config.cmd_line_options['subargs']) - obj.localOptions = options if obj.inputFile: # inputFilename is the actual filename @@ -99,12 +113,11 @@ def processTest(obj, cmd_line_options): try: log.debug("Parsing commandline options") - tmp_test_case_object = obj() - tmp_test_case_object._checkRequiredOptions() + tmp_test_instance = obj() + checkRequiredOptions(tmp_test_instance) except usage.UsageError, ue: log.err("%s" % ue) options.opt_help() - raise usage.UsageError("Error parsing command line args for %s" % tmp_test_case_object.name) else: @@ -162,7 +175,6 @@ def loadTestsAndOptions(classes, cmd_line_options): def getTestTimeout(test_instance, test_method): """ - Returns the timeout value set on this test. Check on the instance first, the the class, then the module, then package. As soon as it finds something with a timeout attribute, returns that. Returns the value set in @@ -199,7 +211,7 @@ def getTestTimeout(test_instance, test_method): return float(default_timeout) def runTestCasesWithInput(test_cases, test_input, yaml_reporter, - oonib_reporter=None): + oonib_reporter=None): """ Runs in parallel all the test methods that are inside of the specified test case. Reporting happens every time a Test Method has concluded running. @@ -552,9 +564,6 @@ def runTestCases(test_cases, options, cmd_line_options): except Exception: log.exception("Problem in running test") -class UnableToStartTor(Exception): - pass - def startTor(): @defer.inlineCallbacks def state_complete(state): @@ -617,7 +626,7 @@ def startSniffing(): from ooni.utils.txscapy import ScapyFactory, ScapySniffer try: checkForRoot() - except NotRootError: + except PermissionsError: print "[!] Includepcap options requires root priviledges to run" print " you should run ooniprobe as root or disable the options in ooniprobe.conf" sys.exit(1)

1 0

[ooni-probe/master] Merge branch 'master' into tcp-flags
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 20cd1df9de76b54a77aac22e44f457272354c0cb Merge: 4142c7f 9688f83 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Dec 12 01:39:33 2012 +0000 Merge branch 'master' into tcp-flags Conflicts: ooni/inputunit.py ooni/nettest.py ooni/oonicli.py ooni/reporter.py ooni/runner.py ooni/utils/hacks.py ooni/utils/net.py ooni/utils/txscapy.py ooniprobe.conf.sample nettests/bridge_reachability/tcpsyn.py | 191 +++++++++++++++++++++ nettests/experimental/bridge_reachability/echo.py | 150 ++++++++++++----- ooni/inputunit.py | 20 ++- ooni/nettest.py | 60 ++++++- ooni/oonicli.py | 40 +++-- ooni/reporter.py | 45 +++--- ooni/runner.py | 157 +++++++++++++---- ooni/utils/__init__.py | 19 +- ooni/utils/geodata.py | 16 ++- ooni/utils/hacks.py | 7 +- ooni/utils/log.py | 27 +++ ooni/utils/net.py | 113 +++++++++++-- ooni/utils/txscapy.py | 15 +- ooniprobe.conf.sample | 3 + 14 files changed, 712 insertions(+), 151 deletions(-) diff --cc nettests/experimental/bridge_reachability/echo.py index d4033dd,0000000..0f422ca mode 100644,000000..100644 --- a/nettests/experimental/bridge_reachability/echo.py +++ b/nettests/experimental/bridge_reachability/echo.py @@@ -1,132 -1,0 +1,200 @@@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# +---------+ +# | echo.py | +# +---------+ +# A simple ICMP-8 ping test. +# +# @authors: Isis Lovecruft, <isis(a)torproject.org> +# @version: 0.0.2-pre-alpha +# @license: copyright (c) 2012 Isis Lovecruft +# see attached LICENCE file +# + +import os +import sys + +from twisted.python import usage - from twisted.internet import reactor, defer ++from twisted.internet import reactor, defer, address +from ooni import nettest +from ooni.utils import log, net, Storage, txscapy + +try: - from scapy.all import IP, ICMP - from scapy.all import sr1 - from ooni.lib import txscapy - from ooni.lib.txscapy import txsr, txsend - from ooni.templates.scapyt import BaseScapyTest - except: ++ from scapy.all import IP, ICMP ++ from scapy.all import sr1 ++ from ooni.utils import txscapy ++except Exception, e: + log.msg("This test requires scapy, see www.secdev.org/projects/scapy") ++ log.exception(e) + +class UsageOptions(usage.Options): ++ """ ++ Options for EchoTest. ++ ++ Note: 'count', 'size', and 'ttl' have yet to be implemented. ++ """ + optParameters = [ + ['dst', 'd', None, 'Host IP to ping'], + ['file', 'f', None, 'File of list of IPs to ping'], ++ ['pcap', 'p', None, 'Save pcap to this file'], + ['interface', 'i', None, 'Network interface to use'], ++ ['receive', 'r', True, 'Receive response packets'], ++ ['timeout', 't', 2, 'Seconds to wait if no response', int], + ['count', 'c', 1, 'Number of packets to send', int], - ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], - ['ttl', 'l', 25, 'Set the IP Time to Live', int], - ['timeout', 't', 2, 'Seconds until timeout if no response', int], - ['pcap', 'p', None, 'Save pcap to this file'], - ['receive', 'r', True, 'Receive response packets']] ++ ['size', 's', 56, 'Bytes to send in ICMP data field', int], ++ ['ttl', 'l', 25, 'Set the IP Time to Live', int]] + +class EchoTest(nettest.NetTestCase): + """ - xxx fill me in ++ Basic ping test. This takes an input file containing one IP or hostname ++ per line. + """ + name = 'echo' + author = 'Isis Lovecruft <isis(a)torproject.org>' + description = 'A simple ping test to see if a host is reachable.' + version = '0.0.2' + requiresRoot = True + + usageOptions = UsageOptions + #requiredOptions = ['dst'] + + def setUp(self, *a, **kw): ++ """ ++ Send an ICMP-8 packet to a host IP, and process the response. ++ ++ @param timeout: ++ Seconds after sending the last packet to timeout. ++ @param interface: ++ The interface to restrict listening to. ++ @param dst: ++ A single host to ping. ++ @param file: ++ A file of hosts to ping, one per line. ++ @param receive: ++ Whether or not to receive replies. Defaults to True. ++ @param pcap: ++ The file to save packet captures to. ++ """ + self.destinations = {} + + if self.localOptions: + for key, value in self.localOptions.items(): + log.debug("setting self.%s = %s" % (key, value)) + setattr(self, key, value) + + self.timeout *= 1000 ## convert to milliseconds + + if not self.interface: + try: + iface = txscapy.getDefaultIface() + except Exception, e: + log.msg("No network interface specified!") - log.err(e) ++ log.exception(e) + else: + log.msg("Using system default interface: %s" % iface) + self.interface = iface + + if self.pcap: + try: + self.pcapfile = open(self.pcap, 'a+') + except: + log.msg("Unable to write to pcap file %s" % self.pcap) + else: + self.pcap = net.capturePacket(self.pcapfile) + + if not self.dst: + if self.file: + self.dstProcessor(self.file) - for key, value in self.destinations.items(): - for label, data in value.items(): - if not 'ans' in data: - self.dst = label ++ for address, details in self.destinations.items(): ++ for labels, data in details.items(): ++ if not 'response' in labels: ++ self.dst = details['dst_ip'] + else: + self.addDest(self.dst) - log.debug("self.dst is now: %s" % self.dst) + + log.debug("Initialization of %s test completed." % self.name) + + def addDest(self, dest): + d = dest.strip() + self.destinations[d] = {'dst_ip': d} + + def dstProcessor(self, inputfile): - from ipaddr import IPAddress - + if os.path.isfile(inputfile): + with open(inputfile) as f: + for line in f.readlines(): + if line.startswith('#'): + continue + self.addDest(line) + ++ def build_packets(self): ++ """ ++ Construct a list of packets to send out. ++ """ ++ packets = [] ++ for dest, data in self.destinations.items(): ++ pkt = IP(dst=dest)/ICMP() ++ packets.append(pkt) ++ ## XXX if a domain was specified, we need a way to check that ++ ## its IP matches the one we're seeing in pkt.src ++ #try: ++ # address.IPAddress(dest) ++ #except: ++ # data['dst_ip'] = pkt.dst ++ return packets ++ + def test_icmp(self): - def process_response(echo_reply, dest): - ans, unans = echo_reply - if ans: - log.msg("Recieved echo reply from %s: %s" % (dest, ans)) - else: - log.msg("No reply was received from %s. Possible censorship event." % dest) - log.debug("Unanswered packets: %s" % unans) - self.report[dest] = echo_reply - - for label, data in self.destinations.items(): - reply = sr1(IP(dst=lebal)/ICMP()) - process = process_reponse(reply, label) - - #(ans, unans) = ping - #self.destinations[self.dst].update({'ans': ans, - # 'unans': unans, - # 'response_packet': ping}) - #return ping - - #return reply ++ """ ++ Send the list of ICMP packets. ++ ++ TODO: add end summary progress report for % answered, etc. ++ """ ++ try: ++ def nicely(packets): ++ """Print scapy summary nicely.""" ++ return list([x.summary() for x in packets]) ++ ++ def process_answered((answered, sent)): ++ """Callback function for txscapy.sr().""" ++ self.report['sent'] = nicely(sent) ++ self.report['answered'] = [nicely(ans) for ans in answered] ++ ++ for req, resp in answered: ++ log.msg("Received echo-reply:\n%s" % resp.summary()) ++ for dest, data in self.destinations.items(): ++ if data['dst_ip'] == resp.src: ++ data['response'] = resp.summary() ++ data['censored'] = False ++ for snd in sent: ++ if snd.dst == resp.src: ++ answered.remove((req, resp)) ++ return (answered, sent) ++ ++ def process_unanswered((unanswered, sent)): ++ """ ++ Callback function for remaining packets and destinations which ++ do not have an associated response. ++ """ ++ if len(unanswered) > 0: ++ nicer = [nicely(unans) for unans in unanswered] ++ log.msg("Unanswered/remaining packets:\n%s" ++ % nicer) ++ self.report['unanswered'] = nicer ++ for dest, data in self.destinations.items(): ++ if not 'response' in data: ++ log.msg("No reply from %s. Possible censorship event." ++ % dest) ++ data['response'] = None ++ data['censored'] = True ++ return (unanswered, sent) ++ ++ packets = self.build_packets() ++ d = txscapy.sr(packets, iface=self.interface, multi=True) ++ d.addCallback(process_answered) ++ d.addErrback(log.exception) ++ d.addCallback(process_unanswered) ++ d.addErrback(log.exception) ++ self.report['destinations'] = self.destinations ++ return d ++ except Exception, e: ++ log.exception(e) diff --cc ooni/inputunit.py index 2ef89d8,4a5c792..0b4377f --- a/ooni/inputunit.py +++ b/ooni/inputunit.py @@@ -6,9 -6,10 +6,10 @@@ # units. Input units are how the inputs to be fed to tests are # split up into. # --# :authors: Arturo Filastò ++# :authors: Arturo Filastò, Isis Lovecruft # :license: see included LICENSE file + class InputUnitFactory(object): """ This is a factory that takes the size of input units to be generated a set @@@ -61,19 -50,31 +62,30 @@@ class InputUnit(object): """ This is a python iterable object that contains the input elements to be - passed onto a TestCase. + passed onto a :class:`ooni.nettest.NetTestCase`. """ def __init__(self, inputs=[]): + """ + Create an iterable from a list of inputs, which can be given to a NetTestCase. + + @param inputs: A list of inputs for a NetTestCase. + """ self._inputs = iter(inputs) + # _inputs_copy is to avoid stealing things from + # the iterator when __repr__ is called: - _inputs_copy = inputs - self.__contains__ = _inputs_copy ++ self._inputs_copy = inputs - def __repr__(self): + def __str__(self): - return "<%s inputs=%s>" % (self.__class__, self._inputs) + """Prints the original input list.""" - return "<%s inputs=%s>" % (self.__class__, self.__contains__) ++ return "<%s inputs=%s>" % (self.__class__, self._inputs_copy) def __add__(self, inputs): + """Add a list of inputs to the iterator.""" - for input in inputs: - self._inputs.append(input) + for i in inputs: + self._inputs.append(i) def __iter__(self): + """Self explanatory.""" return self def next(self): @@@ -80,5 -82,5 +93,6 @@@ return self._inputs.next() def append(self, input): + """Add an item to the end of the InputUnit iterator.""" self._inputs.append(input) + diff --cc ooni/nettest.py index 8374db1,29ced70..8fc29c3 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@@ -134,21 -119,9 +137,19 @@@ class NetTestCase(object) pass def setUp(self): - """ - Place here your logic to be executed when the test is being setup. - """ + """Place your logic to be executed when the test is being setup here.""" pass + def postProcessor(self, report): + """ + Subclass this to do post processing tasks that are to occur once all + the test methods have been called. Once per input. + postProcessing works exactly like test methods, in the sense that + anything that gets written to the object self.report[] will be added to + the final test report. + """ + raise NoPostProcessor + def inputProcessor(self, filename=None): """ You may replace this with your own custom input processor. It takes as @@@ -185,22 -158,17 +186,24 @@@ if not self.localOptions[required_option]: raise usage.UsageError("%s not specified!" % required_option) - def _processOptions(self, options=None): - if self.inputFile: - self.inputs = self.inputProcessor(self.inputFile) + def _processOptions(self): + if self.inputFilename: + inputProcessor = self.inputProcessor + inputFilename = self.inputFilename + class inputProcessorIterator(object): + """ + Here we convert the input processor generator into an iterator + so that we can run it twice. + """ + def __iter__(self): + return inputProcessor(inputFilename) + self.inputs = inputProcessorIterator() + self._checkRequiredOptions() + - # XXX perhaps we may want to name and version to be inside of a - # different method that is not called options. return {'inputs': self.inputs, - 'name': self.name, - 'version': self.version} + 'name': self.name, 'version': self.version + } def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) diff --cc ooni/oonicli.py index f706ee3,c64e445..9d4f783 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@@ -20,15 -19,19 +20,18 @@@ from twisted.application import ap from twisted.python import usage, failure from twisted.python.util import spewer - from ooni import nettest, runner, reporter, config -## Get rid of the annoying "No route found for -## IPv6 destination warnings": ++# Supress scapy's "No route found for IPv6 destination" warnings: + import logging as pylogging + pylogging.getLogger("scapy.runtime").setLevel(pylogging.ERROR) + from ooni import nettest, runner, reporter, config +from ooni.inputunit import InputUnitFactory - from ooni.utils import net - from ooni.utils import checkForRoot, NotRootError + from ooni.utils import checkForRoot, PermissionsError from ooni.utils import log -from ooni.inputunit import InputUnitFactory + -class Options(usage.Options, app.ReactorSelectionMixin): +class Options(usage.Options): synopsis = """%s [options] [path to test].py """ % (os.path.basename(sys.argv[0]),) @@@ -78,68 -79,13 +81,66 @@@ except: raise usage.UsageError("No test filename specified!") +def updateStatusBar(): + for test_filename in config.state.keys(): + # The ETA is not updated so we we will not print it out for the + # moment. + eta = config.state[test_filename].eta() + progress = config.state[test_filename].progress() + progress_bar_frmt = "[%s] %s%%" % (test_filename, progress) + print progress_bar_frmt + def testsEnded(*arg, **kw): - """ - You can place here all the post shutdown tasks. - """ - log.debug("testsEnded: Finished running all tests") + """You can place here all the post shutdown tasks.""" + log.debug("Finished running all tests") + config.start_reactor = False - try: reactor.stop() - except: pass ++ if not reactor.running: ++ try: reactor.stop() ++ except: reactor.runUntilCurrent() + +def startSniffing(): + from ooni.utils.txscapy import ScapyFactory, ScapySniffer + try: + checkForRoot() + except NotRootError: + print "[!] Includepcap options requires root priviledges to run" + print " you should run ooniprobe as root or disable the options in ooniprobe.conf" + sys.exit(1) + + print "Starting sniffer" + config.scapyFactory = ScapyFactory(config.advanced.interface) + + sniffer = ScapySniffer(config.reports.pcap) + config.scapyFactory.registerProtocol(sniffer) + +def runTestList(none, test_list): + """ + none: is always None. + + test_list (list): a list of tuples containing (test_cases, options, + cmd_line_options) + """ + deck_dl = [] + + for test in test_list: + test_cases, options, cmd_line_options = test + d1 = runner.runTestCases(test_cases, options, cmd_line_options) + deck_dl.append(d1) + + d2 = defer.DeferredList(deck_dl) + d2.addBoth(testsEnded) + + # Print every 5 second the list of current tests running + l = task.LoopingCall(updateStatusBar) + l.start(5.0) + return d2 + +def errorRunningTests(failure): + failure.printTraceback() def run(): - """ - Parses command line arguments of test. - """ + """Call me to begin testing from a file.""" + cmd_line_options = Options() if len(sys.argv) == 1: cmd_line_options.getUsage() @@@ -148,41 -94,42 +149,52 @@@ except usage.UsageError, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) - if cmd_line_options['debug-stacktraces']: - defer.setDebugging(True) - log.start(cmd_line_options['logfile']) - test_file_name = os.path.basename(cmd_line_options['test']) - log.debug("Running script %s" % test_file_name) - - yamloo_filename, pcap_filename = config.oreport_filenames(test_file_name) - - if cmd_line_options['reportfile']: - yamloo_filename = cmd_line_options['reportfile'] - pcap_filename = yamloo_filename+".pcap" - - if os.path.exists(yamloo_filename): - log.msg("Report already exists with filename %s" % yamloo_filename) - log.msg("Renaming it to %s" % yamloo_filename+'.old') - os.rename(yamloo_filename, yamloo_filename+'.old') - if os.path.exists(pcap_filename): - log.msg("Report already exists with filename %s" % pcap_filename) - log.msg("Renaming it to %s" % pcap_filename+'.old') - os.rename(pcap_filename, pcap_filename+'.old') - - classes = runner.findTestClassesFromConfig(cmd_line_options) - test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options) if config.privacy.includepcap: - log.msg("Starting") - runner.startSniffing() + try: + checkForRoot() + except PermissionsError, pe: - log.err(str("'includepcap' option requires administrator or root ", - "privileges to run. Run ooniprobe as root or disable ", - "the includepcap option in ooniprobe.conf ")) ++ m = ("Capturing packets requires administrator/root privileges. ", ++ "Run ooniprobe as root or set 'includepcap = false' in ", ++ "ooniprobe.conf .") ++ log.warn("%s" % m) + sys.exit(1) - log.debug("Starting sniffer") - sniffer_d = net.capturePackets(pcap_filename) ++ else: ++ log.msg("Starting packet capture") ++ runner.startSniffing() + + resume = cmd_line_options['resume'] + + # contains (test_cases, options, cmd_line_options) + test_list = [] + + if cmd_line_options['testdeck']: + test_deck = yaml.safe_load(open(cmd_line_options['testdeck'])) + for test in test_deck: + del cmd_line_options + cmd_line_options = test['options'] + if resume: + cmd_line_options['resume'] = True + else: + cmd_line_options['resume'] = False + test_list.append(runner.loadTest(cmd_line_options)) + else: + log.msg("No test deck detected") + del cmd_line_options['testdeck'] + test_list.append(runner.loadTest(cmd_line_options)) + + if config.advanced.start_tor: + log.msg("Starting Tor...") + d = runner.startTor() + d.addCallback(runTestList, test_list) + d.addErrback(errorRunningTests) + else: + # We need to pass None as first argument because when the callback is + # fired it will pass it's result to runTestCase. + d = runTestList(None, test_list) + d.addErrback(errorRunningTests) + ++ # XXX I believe we don't actually need this: + reactor.run() + - tests_d = runner.runTestCases(test_cases, options, - cmd_line_options, yamloo_filename) - tests_d.addBoth(testsEnded) diff --cc ooni/reporter.py index 29e6049,133b98d..fc41960 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -15,97 -13,23 +14,94 @@@ import o import time import yaml import json + import traceback +from yaml.representer import * +from yaml.emitter import * +from yaml.serializer import * +from yaml.resolver import * from twisted.python.util import untilConcludes from twisted.trial import reporter from twisted.internet import defer, reactor +from twisted.internet.error import ConnectionRefusedError - from ooni.utils import log -from ooni import config -from ooni.templates.httpt import BodyReceiver, StringProducer -from ooni.utils import otime, log, geodata ++from ooni import config, otime ++from ooni.utils import log, geodata + from ooni.utils.hacks import OSafeRepresenter, OSafeDumper ++from ooni.utils.net import BodyReceiver, StringProducer, userAgents try: - from scapy.all import packet -except: - class FooClass: - Packet = object - packet = FooClass + from scapy.packet import Packet +except ImportError: + log.err("Scapy is not installed.") + - - from ooni import otime - from ooni.utils import geodata - from ooni.utils.net import BodyReceiver, StringProducer, userAgents - - from ooni import config - +def createPacketReport(packet_list): + """ + Takes as input a packet a list. + + Returns a dict containing a dict with the packet + summary and the raw packet. + """ + report = [] + for packet in packet_list: + report.append({'raw_packet': str(packet), + 'summary': str(packet.summary())}) + return report + +class OSafeRepresenter(SafeRepresenter): + """ + This is a custom YAML representer that allows us to represent reports + safely. + It extends the SafeRepresenter to be able to also represent complex + numbers and scapy packet. + """ + def represent_data(self, data): + """ + This is very hackish. There is for sure a better way either by using + the add_multi_representer or add_representer, the issue though lies in + the fact that Scapy packets are metaclasses that leads to + yaml.representer.get_classobj_bases to not be able to properly get the + base of class of a Scapy packet. + XXX fully debug this problem + """ + if isinstance(data, Packet): + data = createPacketReport(data) + return SafeRepresenter.represent_data(self, data) + + def represent_complex(self, data): + if data.imag == 0.0: + data = u'%r' % data.real + elif data.real == 0.0: + data = u'%rj' % data.imag + elif data.imag > 0: + data = u'%r+%rj' % (data.real, data.imag) + else: + data = u'%r%rj' % (data.real, data.imag) + return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data) + +OSafeRepresenter.add_representer(complex, + OSafeRepresenter.represent_complex) + +class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): + """ + This is a modification of the YAML Safe Dumper to use our own Safe + Representer that supports complex numbers. + """ + def __init__(self, stream, + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + OSafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style) + Resolver.__init__(self) class NoTestIDSpecified(Exception): pass @@@ -180,10 -111,10 +185,10 @@@ class OReporter(object) pass def testDone(self, test, test_name): - log.msg("Finished running %s" % test_name) + log.debug("Calling reporter to record results") test_report = dict(test.report) - if isinstance(test.input, packet.Packet): + if isinstance(test.input, Packet): test_input = createPacketReport(test.input) else: test_input = test.input @@@ -259,19 -189,11 +264,19 @@@ class OONIBTestDetailsLookupError(OONIB pass class OONIBReporter(OReporter): - def __init__(self, backend_url): - from twisted.web.client import Agent + def __init__(self, cmd_line_options): + self.backend_url = cmd_line_options['collector'] + self.report_id = None + + from ooni.utils.txagentwithsocks import Agent from twisted.internet import reactor - self.agent = Agent(reactor) - self.backend_url = backend_url + try: + self.agent = Agent(reactor, sockshost="127.0.0.1", + socksport=int(config.tor.socks_port)) - except Exception, e: - log.exception(e) ++ except Exception, ex: ++ log.exception(ex) + + OReporter.__init__(self, cmd_line_options) @defer.inlineCallbacks def writeReportEntry(self, entry): @@@ -308,47 -236,35 +313,47 @@@ test_name = options['name'] test_version = options['version'] - log.debug("Creating report with OONIB Reporter") url = self.backend_url + '/report/new' - software_version = '0.0.1' - test_details = yield getTestDetails(options) + try: + test_details = yield getTestDetails(options) - except Exception, e: - log.exception(e) ++ except Exception, ex: ++ log.exception(ex) + + test_details['options'] = self.cmd_line_options + + log.debug("Obtained test_details: %s" % test_details) content = '---\n' content += safe_dump(test_details) content += '...\n' - request = {'software_name': 'ooniprobe', - 'software_version': software_version, + request = {'software_name': test_details['software_name'], + 'software_version': test_details['software_version'], 'test_name': test_name, 'test_version': test_version, - 'progress': 0, 'content': content -- } - log.debug("Creating report via url %s" % url) ++ } + + log.msg("Reporting %s" % url) request_json = json.dumps(request) log.debug("Sending %s" % request_json) bodyProducer = StringProducer(json.dumps(request)) - log.debug("Creating report via url %s" % url) + + log.msg("Creating report with OONIB Reporter. Please be patient.") + log.msg("This may take up to 1-2 minutes...") try: - response = yield self.agent.request("POST", url, + response = yield self.agent.request("POST", url, bodyProducer=bodyProducer) - except: - raise OONIBReportCreationFailed + except ConnectionRefusedError: + log.err("Connection to reporting backend failed (ConnectionRefusedError)") + raise OONIBReportCreationError + - except Exception, e: - log.exception(e) ++ except Exception, ex: ++ log.exception(ex) + raise OONIBReportCreationError # This is a little trix to allow us to unspool the response. We create # a deferred and call yield on it. diff --cc ooni/runner.py index bc9b874,5969dd5..58ec188 --- a/ooni/runner.py +++ b/ooni/runner.py @@@ -13,28 -14,30 +14,36 @@@ import tim import inspect import traceback import itertools +import yaml - from twisted.python import reflect, usage - from twisted.internet import defer + from twisted.python import reflect, usage, failure -from twisted.internet import defer, reactor ++from twisted.internet import defer, reactor, threads from twisted.trial.runner import filenameToModule - from twisted.internet import reactor, threads + from twisted.trial import reporter as txreporter + from twisted.trial import util as txtrutil + from twisted.trial.unittest import utils as txtrutils + from twisted.trial.unittest import SkipTest -from twisted.internet import reactor, threads +from txtorcon import TorProtocolFactory, TorConfig +from txtorcon import TorState, launch_tor + - from ooni import config - ++from ooni import config, nettest, reporter +from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError - from ooni.inputunit import InputUnitFactory - from ooni.nettest import NetTestCase, NoPostProcessor - -from ooni import reporter, nettest -from ooni.utils import log, checkForRoot, PermissionsError +from ooni.utils import log, checkForRoot - from ooni.utils import NotRootError, Storage ++from ooni.utils import PermissionsError, Storage +from ooni.utils.net import randomFreePort - def processTest(obj): + def isTestCase(obj): + """ + Return True if obj is a subclass of NetTestCase, false if otherwise. + """ + try: + return issubclass(obj, nettest.NetTestCase) + except TypeError: + return False + + def processTest(obj, cmd_line_options): """ Process the parameters and :class:`twisted.python.usage.Options` of a :class:`ooni.nettest.Nettest`. @@@ -46,60 -48,61 +55,54 @@@ :param cmd_line_options: A configured and instantiated :class:`twisted.python.usage.Options` class. + """ + if not hasattr(obj.usageOptions, 'optParameters'): + obj.usageOptions.optParameters = [] + + if obj.inputFile: + obj.usageOptions.optParameters.append(obj.inputFile) - input_file = obj.inputFile + if obj.requiresRoot: + try: + checkForRoot() + except PermissionsError: + log.err("%s requires root to run" % obj.name) + sys.exit(1) + - if obj.optParameters or input_file \ - or obj.usageOptions or obj.optFlags: + if obj.baseParameters: + for parameter in obj.baseParameters: + obj.usageOptions.optParameters.append(parameter) - if not obj.optParameters: - obj.optParameters = [] + if obj.baseFlags: + if not hasattr(obj.usageOptions, 'optFlags'): + obj.usageOptions.optFlags = [] + for flag in obj.baseFlags: + obj.usageOptions.optFlags.append(flag) - if input_file: - obj.optParameters.append(input_file) + options = obj.usageOptions() - if obj.usageOptions: - if input_file: - obj.usageOptions.optParameters.append(input_file) - options = obj.usageOptions() - elif obj.optParameters: - log.debug("Got optParameters") - class Options(usage.Options): - optParameters = obj.optParameters - if obj.optFlags: - log.debug("Got optFlags") - optFlags = obj.optFlags - options = Options() + options.parseOptions(config.cmd_line_options['subargs']) + obj.localOptions = options - if options: - options.parseOptions(cmd_line_options['subArgs']) - obj.localOptions = options + if obj.inputFile: + obj.inputFilename = options[obj.inputFile[0]] - if input_file and options: - log.debug("Added input file to options list") - obj.inputFile = options[input_file[0]] - - try: - log.debug("processing options") - tmp_test_case_object = obj() - tmp_test_case_object._processOptions(options) + try: + log.debug("processing options") + tmp_test_case_object = obj() + tmp_test_case_object._checkRequiredOptions() - except usage.UsageError, e: - test_name = tmp_test_case_object.name - print "There was an error in running %s!" % test_name - print "%s" % e - options.opt_help() - raise usage.UsageError("Error in parsing command line args for %s" % test_name) + except usage.UsageError, e: + test_name = tmp_test_case_object.name + log.err("There was an error in running %s!" % test_name) + log.err("%s" % e) + options.opt_help() + raise usage.UsageError("Error in parsing command line args for %s" % test_name) - if obj.requiresRoot: - try: - checkForRoot() - except NotRootError: - log.err("%s requires root to run" % obj.name) - sys.exit(1) - return obj - def isTestCase(obj): - try: - return issubclass(obj, NetTestCase) - except TypeError: - return False - -def findTestClassesFromConfig(cmd_line_options): +def findTestClassesFromFile(filename): """ Takes as input the command line config parameters and returns the test case classes. @@@ -152,408 -153,187 +155,494 @@@ def loadTestsAndOptions(classes, cmd_li return test_cases, options + def getTimeout(test_instance, test_method): + """ + Returns the timeout value set on this test. Check on the instance first, + the the class, then the module, then package. As soon as it finds + something with a timeout attribute, returns that. Returns + twisted.trial.util.DEFAULT_TIMEOUT_DURATION if it cannot find anything. + + See twisted.trial.unittest.TestCase docstring for more details. + """ + try: + testMethod = getattr(test_instance, test_method) + except: + log.debug("_getTimeout couldn't find self.methodName!") + return txtrutil.DEFAULT_TIMEOUT_DURATION + else: + test_instance._parents = [testMethod, test_instance] + test_instance._parents.extend(txtrutil.getPythonContainers(testMethod)) + timeout = txtrutil.acquireAttribute(test_instance._parents, 'timeout', + txtrutil.DEFAULT_TIMEOUT_DURATION) + try: + return float(timeout) + except (ValueError, TypeError): + warnings.warn("'timeout' attribute needs to be a number.", + category=DeprecationWarning) + return txtrutil.DEFAULT_TIMEOUT_DURATION + -def runTestWithInput(test_class, test_method, test_input, oreporter): +def runTestCasesWithInput(test_cases, test_input, yaml_reporter, + oonib_reporter=None): """ - Runs a single testcase from a NetTestCase with one input. + Runs in parallel all the test methods that are inside of the specified test case. + Reporting happens every time a Test Method has concluded running. + Once all the test methods have been called we check to see if the + postProcessing class method returns something. If it does return something + we will write this as another entry inside of the report called post_processing. + + Args: + + test_cases (list): A list of tuples containing the test_class (a + class) and the test_method (a string) + + test_input (instance): Any instance that will be passed as input to + the test. + + yaml_reporter: An instance of :class:ooni.reporter.YAMLReporter + + oonib_reporter: An instance of :class:ooni.reporter.OONIBReporter. If + this is set to none then we will only report to the YAML reporter. + """ - log.debug("Running %s with %s" % (test_method, test_input)) + + # This is used to store a copy of all the test reports + tests_report = {} + def test_timeout(d): + timeout_error = defer.TimeoutError( + "%s test for %s timed out after %s seconds" + % (test_name, test_instance.input, test_instance.timeout)) + timeout_fail = failure.Failure(err) + try: + d.errback(timeout_fail) + except defer.AlreadyCalledError: + # if the deferred has already been called but the *back chain is + # still unfinished, safely crash the reactor and report the timeout + reactor.crash() + test_instance._timedOut = True # see test_instance._wait + test_instance._test_result.addExpectedFailure(test_instance, fail) + test_timeout = txtrutils.suppressWarnings( + test_timeout, txtrutil.suppress(category=DeprecationWarning)) + + def test_skip_class(reason): + try: + d.errback(failure.Failure(SkipTest("%s" % reason))) + except defer.AlreadyCalledError: + pass # XXX not sure what to do here... + def test_done(result, test_instance, test_name): - log.debug("Concluded %s with inputs %s" - % (test_name, test_instance.input)) - return oreporter.testDone(test_instance, test_name) + log.msg("Successfully finished running %s" % test_name) + log.debug("Deferred callback result: %s" % result) + tests_report[test_name] = dict(test_instance.report) + if not oonib_reporter: + return yaml_reporter.testDone(test_instance, test_name) + d1 = oonib_reporter.testDone(test_instance, test_name) + d2 = yaml_reporter.testDone(test_instance, test_name) + return defer.DeferredList([d1, d2]) - def test_error(failure, test_instance, test_name): - log.err("Error in running %s" % test_name) - log.exception(failure) + def test_error(error, test_instance, test_name): + if isinstance(error, SkipTest): + log.warn("%s" % error.message) + else: ++ log.err("Error in running %s" % test_name) + log.exception(error) + return + + def tests_done(result, test_class): + test_instance = test_class() + test_instance.report = {} + test_instance.input = None + test_instance._start_time = time.time() + post = getattr(test_instance, 'postProcessor') + try: + post_processing = post(tests_report) + if not oonib_reporter: + return yaml_reporter.testDone(test_instance, 'summary') + d1 = oonib_reporter.testDone(test_instance, 'summary') + d2 = yaml_reporter.testDone(test_instance, 'summary') + return defer.DeferredList([d1, d2]) + except NoPostProcessor: + log.debug("No post processor configured") + return - test_instance = test_class() - test_instance.input = test_input - test_instance.report = {} - # XXX TODO the twisted.trial.reporter.TestResult is expected by - # test_timeout(), but we should eventually replace it with a stub class - test_instance._test_result = txreporter.TestResult() - # use this to keep track of the test runtime - test_instance._start_time = time.time() - test_instance.timeout = getTimeout(test_instance, test_method) - # call setups on the test - test_instance._setUp() - test_instance.setUp() - - test_skip = txtrutil.acquireAttribute(test_instance._parents, 'skip', None) - if test_skip: - log.warn("%s marked these tests to be skipped: %s" - % (test_instance.name, test_skip)) - skip_list = [test_skip] - - test = getattr(test_instance, test_method) - test_instance._testMethod = test - - d = defer.maybeDeferred(test) - - # register the timer with the reactor - call_timeout = reactor.callLater(test_instance.timeout, test_timeout, d) - d.addBoth(lambda x: call_timeout.active() and call_timeout.cancel() or x) - - # check if the class has been aborted - if hasattr(test_instance.__class__, 'skip'): - reason = getattr(test_instance.__class__, 'skip') - call_skip = reactor.callLater(0, test_skip_class, reason) - d.addBoth(lambda x: call_skip.active() and call_skip.cancel() or x) - - d.addCallback(test_done, test_instance, test_method) - d.addErrback(test_error, test_instance, test_method) + dl = [] + for test_case in test_cases: + log.debug("Processing %s" % test_case[1]) + test_class = test_case[0] + test_method = test_case[1] - - log.msg("Running %s with %s..." % (test_method, test_input)) - + test_instance = test_class() + test_instance.input = test_input + test_instance.report = {} ++ ++ # XXX TODO the twisted.trial.reporter.TestResult is expected by ++ # test_timeout(), but we should eventually replace it with a stub class ++ test_instance._test_result = txreporter.TestResult() ++ + # use this to keep track of the test runtime + test_instance._start_time = time.time() ++ test_instance.timeout = getTimeout(test_instance, test_method) ++ + # call setups on the test + test_instance._setUp() + test_instance.setUp() ++ ++ # check if we're inherited from anything marked to be skipped ++ test_skip = txtrutil.acquireAttribute(test_instance._parents, 'skip', None) ++ if test_skip: ++ log.warn("%s marked these tests to be skipped: %s" ++ % (test_instance.name, test_skip)) ++ skip_list = [test_skip] ++ + test = getattr(test_instance, test_method) ++ test_instance._testMethod = test + + d = defer.maybeDeferred(test) ++ ++ # register the timer with the reactor ++ call_timeout = reactor.callLater(test_instance.timeout, test_timeout, d) ++ d.addBoth(lambda x: call_timeout.active() and call_timeout.cancel() or x) ++ ++ # check if the class has been aborted ++ if hasattr(test_instance.__class__, 'skip'): ++ reason = getattr(test_instance.__class__, 'skip') ++ call_skip = reactor.callLater(0, test_skip_class, reason) ++ d.addBoth(lambda x: call_skip.active() and call_skip.cancel() or x) ++ + d.addCallback(test_done, test_instance, test_method) + d.addErrback(test_error, test_instance, test_method) + dl.append(d) - return d + test_methods_d = defer.DeferredList(dl) + test_methods_d.addCallback(tests_done, test_cases[0][0]) + return test_methods_d -def runTestWithInputUnit(test_class, test_method, input_unit, oreporter): +def runTestCasesWithInputUnit(test_cases, input_unit, yaml_reporter, + oonib_reporter): """ - @param test_class: - The uninstantiated :class:`ooni.nettest.NetTestCase` to be run. - @param test_method: - A string representing the method name to be called. + Runs the Test Cases that are given as input parallely. + A Test Case is a subclass of ooni.nettest.NetTestCase and a list of + methods. + + The deferred list will fire once all the test methods have been + run once per item in the input unit. + - test_cases: A list of tuples containing the test class and the test method as a string. - - input_unit: A generator that yields an input per iteration - ++ @param test_cases: ++ A tuple containing the test_class and test_method as strings. + @param input_unit: + A generator that contains the inputs to be run on the test. - @param oreporter: - A :class:`ooni.reporter.OReporter` instance. - - @return: A DeferredList containing all the tests to be run at this time. ++ @return: ++ A DeferredList containing all the tests to be run at this time. """ - log.debug("Running test cases with input unit") dl = [] for test_input in input_unit: - d = runTestWithInput(test_class, test_method, test_input, oreporter) + log.debug("Running test with this input %s" % test_input) + d = runTestCasesWithInput(test_cases, + test_input, yaml_reporter, oonib_reporter) dl.append(d) return defer.DeferredList(dl) -(a)defer.inlineCallbacks -def runTestCases(test_cases, options, - cmd_line_options, yamloo_filename): +class InvalidResumeFile(Exception): + pass + +class noResumeSession(Exception): + pass + +def loadResumeFile(): """ - XXX we should get rid of the InputUnit class, because we go though the - effort of creating an iterator, only to turn it back into a list, and then - iterate through it. it's also buggy as hell, and it's excess code. + Sets the singleton stateDict object to the content of the resume file. + If the file is empty then it will create an empty one. + + Raises: + + :class:ooni.runner.InvalidResumeFile if the resume file is not valid + """ - try: - assert len(options) != 0, "Length of options is zero!" - except AssertionError, ae: - test_inputs = [] - log.err(ae) - else: + if not config.stateDict: try: - first = options.pop(0) + config.stateDict = yaml.safe_load(open(config.resume_filename)) except: - first = options + log.err("Error loading YAML file") + raise InvalidResumeFile + + if not config.stateDict: + yaml.safe_dump(dict(), open(config.resume_filename, 'w+')) + config.stateDict = dict() - if 'inputs' in first: - test_inputs = options['inputs'] + elif isinstance(config.stateDict, dict): + return else: - log.msg("Could not find inputs!") - log.msg("options[0] = %s" % first) - test_inputs = [None] + log.err("The resume file is of the wrong format") + raise InvalidResumeFile + +def resumeTest(test_filename, input_unit_factory): + """ + Returns the an input_unit_factory that is at the index of the previous run of the test + for the specified test_filename. + + Args: + + test_filename (str): the filename of the test that is being run + including the .py extension. + + input_unit_factory (:class:ooni.inputunit.InputUnitFactory): with the + same input of the past run. + + Returns: + + :class:ooni.inputunit.InputUnitFactory that is at the index of the + previous test run. + + """ + try: + idx = config.stateDict[test_filename] + for x in range(idx): + try: + input_unit_factory.next() + except StopIteration: + log.msg("Previous run was complete") + return input_unit_factory + + return input_unit_factory + + except KeyError: + log.debug("No resume key found for selected test name. It is therefore 0") + config.stateDict[test_filename] = 0 + return input_unit_factory + +(a)defer.inlineCallbacks +def updateResumeFile(test_filename): + """ + update the resume file with the current stateDict state. + """ + log.debug("Acquiring lock for %s" % test_filename) + yield config.resume_lock.acquire() + + current_resume_state = yaml.safe_load(open(config.resume_filename)) + current_resume_state = config.stateDict + yaml.safe_dump(current_resume_state, open(config.resume_filename, 'w+')) + + log.debug("Releasing lock for %s" % test_filename) + config.resume_lock.release() + defer.returnValue(config.stateDict[test_filename]) + +(a)defer.inlineCallbacks +def increaseInputUnitIdx(test_filename): + """ + Args: - reportFile = open(yamloo_filename, 'w+') + test_filename (str): the filename of the test that is being run + including the .py extension. + + input_unit_idx (int): the current input unit index for the test. + + """ + config.stateDict[test_filename] += 1 + yield updateResumeFile(test_filename) + +def updateProgressMeters(test_filename, input_unit_factory, + test_case_number): + """ + Update the progress meters for keeping track of test state. + """ + log.msg("Setting up progress meters") + if not config.state.test_filename: + config.state[test_filename] = Storage() + + config.state[test_filename].per_item_average = 2.0 + + input_unit_idx = float(config.stateDict[test_filename]) + input_unit_items = float(len(input_unit_factory) + 1) + test_case_number = float(test_case_number) + total_iterations = input_unit_items * test_case_number + current_iteration = input_unit_idx * test_case_number + + log.debug("input_unit_items: %s" % input_unit_items) + log.debug("test_case_number: %s" % test_case_number) + + log.debug("Test case number: %s" % test_case_number) + log.debug("Total iterations: %s" % total_iterations) + log.debug("Current iteration: %s" % current_iteration) + + def progress(): + return (current_iteration / total_iterations) * 100.0 + + config.state[test_filename].progress = progress + + def eta(): + return (total_iterations - current_iteration) \ + * config.state[test_filename].per_item_average + config.state[test_filename].eta = eta + + config.state[test_filename].input_unit_idx = input_unit_idx + config.state[test_filename].input_unit_items = input_unit_items + + +(a)defer.inlineCallbacks +def runTestCases(test_cases, options, cmd_line_options): ++ """ ++ Run all test cases found in specified files and modules. ++ ++ @param test_cases: ++ A list of tuples, each tuple in containing the test_class ++ and test_method to run. ++ @param cmd_line_options: ++ The parsed :attr:`twisted.python.usage.Options.optParameters` ++ obtained from the main ooni commandline. ++ """ + log.debug("Running %s" % test_cases) + log.debug("Options %s" % options) + log.debug("cmd_line_options %s" % dict(cmd_line_options)) + + test_inputs = options['inputs'] + + oonib_reporter = OONIBReporter(cmd_line_options) + yaml_reporter = YAMLReporter(cmd_line_options) if cmd_line_options['collector']: - oreporter = reporter.OONIBReporter(cmd_line_options['collector']) + log.msg("Using remote collector, please be patient while we create the report.") + try: + yield oonib_reporter.createReport(options) + except OONIBReportError: + log.err("Error in creating new report") + log.msg("We will only create reports to a file") + oonib_reporter = None else: - oreporter = reporter.YAMLReporter(reportFile) + oonib_reporter = None - input_unit_factory = InputUnitFactory(test_inputs) + yield yaml_reporter.createReport(options) + log.msg("Reporting to file %s" % config.reports.yamloo) + + try: + input_unit_factory = InputUnitFactory(test_inputs) + except Exception, e: + log.exception(e) + + try: + loadResumeFile() + except InvalidResumeFile: + log.err("Error in loading resume file %s" % config.resume_filename) + log.err("Try deleting the resume file") + raise InvalidResumeFile + + test_filename = os.path.basename(cmd_line_options['test']) + + if cmd_line_options['resume']: + log.debug("Resuming %s" % test_filename) + resumeTest(test_filename, input_unit_factory) + else: + log.debug("Not going to resume %s" % test_filename) + config.stateDict[test_filename] = 0 - log.debug("Creating report") - yield oreporter.createReport(options) + updateProgressMeters(test_filename, input_unit_factory, len(test_cases)) - # This deferred list is a deferred list of deferred lists - # it is used to store all the deferreds of the tests that - # are run try: for input_unit in input_unit_factory: - log.debug("Running this input unit %s" % input_unit) - # We do this because generators can't be rewound. - input_list = list(input_unit) - for test_case in test_cases: - log.debug("Processing %s" % test_case[1]) - test_class = test_case[0] - test_method = test_case[1] - yield runTestWithInputUnit(test_class, test_method, - input_list, oreporter) - except Exception, ex: - # XXX we probably want to add a log.warn() at some point - log.warn("Problem in running test") - log.exception(ex) - - oreporter.allDone() - if reactor.running: - reactor.stop() + log.debug("Running %s with input unit %s" % (test_filename, input_unit)) + + yield runTestCasesWithInputUnit(test_cases, input_unit, + yaml_reporter, oonib_reporter) + + yield increaseInputUnitIdx(test_filename) + + updateProgressMeters(test_filename, input_unit_factory, len(test_cases)) + + except Exception: + log.exception("Problem in running test") + +class UnableToStartTor(Exception): + pass + +def startTor(): + @defer.inlineCallbacks + def state_complete(state): + config.tor_state = state + log.msg("Successfully bootstrapped Tor") + log.debug("We now have the following circuits: ") + for circuit in state.circuits.values(): + log.debug(" * %s" % circuit) + + socks_port = yield state.protocol.get_conf("SocksPort") + control_port = yield state.protocol.get_conf("ControlPort") + + config.tor.socks_port = int(socks_port.values()[0]) + config.tor.control_port = int(control_port.values()[0]) + + def setup_failed(failure): + log.exception(failure) + raise UnableToStartTor + + def setup_complete(proto): + """ + Called when we read from stdout that Tor has reached 100%. + """ + log.debug("Building a TorState") + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete) + state.post_bootstrap.addErrback(setup_failed) + return state.post_bootstrap + + def updates(prog, tag, summary): + log.msg("%d%%: %s" % (prog, summary)) + + tor_config = TorConfig() + if config.tor.control_port: + tor_config.ControlPort = config.tor.control_port + else: + control_port = int(randomFreePort()) + tor_config.ControlPort = control_port + config.tor.control_port = control_port + + if config.tor.socks_port: + tor_config.SocksPort = config.tor.socks_port + else: + socks_port = int(randomFreePort()) + tor_config.SocksPort = socks_port + config.tor.socks_port = socks_port + + tor_config.save() + + log.debug("Setting control port as %s" % tor_config.ControlPort) + log.debug("Setting SOCKS port as %s" % tor_config.SocksPort) + + d = launch_tor(tor_config, reactor, + progress_updates=updates) + d.addCallback(setup_complete) + d.addErrback(setup_failed) + return d + +def startSniffing(): + from ooni.utils.txscapy import ScapyFactory, ScapySniffer + try: + checkForRoot() + except NotRootError: + print "[!] Includepcap options requires root priviledges to run" + print " you should run ooniprobe as root or disable the options in ooniprobe.conf" + sys.exit(1) + + print "Starting sniffer" + config.scapyFactory = ScapyFactory(config.advanced.interface) + + sniffer = ScapySniffer(config.reports.pcap) + config.scapyFactory.registerProtocol(sniffer) + +def loadTest(cmd_line_options): + """ + Takes care of parsing test command line arguments and loading their + options. + """ + config.cmd_line_options = cmd_line_options + config.generateReportFilenames() + + if cmd_line_options['reportfile']: + config.reports.yamloo = cmd_line_options['reportfile']+'.yamloo' + config.reports.pcap = config.reports.yamloo+'.pcap' + + if os.path.exists(config.reports.pcap): + print "Report PCAP already exists with filename %s" % config.reports.pcap + print "Renaming it to %s" % config.reports.pcap+".old" + os.rename(config.reports.pcap, config.reports.pcap+".old") + + classes = findTestClassesFromFile(cmd_line_options['test']) + test_cases, options = loadTestsAndOptions(classes, cmd_line_options) + return test_cases, options, cmd_line_options diff --cc ooni/utils/hacks.py index 18e9102,9121982..4cf94d0 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@@ -4,9 -4,14 +4,14 @@@ # ******** # When some software has issues and we need to fix it in a # hackish way, we put it in here. This one day will be empty. - # + # -# :authors: Arturo Filastò ++# :authors: Arturo Filastò, Isis Lovecruft # :licence: see LICENSE + from yaml.representer import * + from yaml.emitter import * + from yaml.serializer import * + from yaml.resolver import * import copy_reg diff --cc ooni/utils/log.py index 3e24804,a13feb9..2721807 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@@ -56,17 -57,35 +62,38 @@@ def warn(msg, *arg, **kw) def err(msg, *arg, **kw): txlog.err("Error: " + str(msg), logLevel=logging.ERROR, *arg, **kw) -def exception(msg): - txlog.err(msg) - exc_type, exc_value, exc_traceback = sys.exc_info() - traceback.print_exception(exc_type, exc_value, exc_traceback) - -def exception(*msg): - logging.exception(msg) +def exception(error): + """ + Error can either be an error message to print to stdout and to the logfile + or it can be a twisted.python.failure.Failure instance. + """ + if isinstance(error, Failure): + error.printTraceback() + else: + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback.print_exception(exc_type, exc_value, exc_traceback) + def fail(*failure): + logging.critical(failure) + + def catch(func): + """ + Quick wrapper to add around test methods for debugging purposes, + catches the given Exception. Use like so: + + @log.catcher + def foo(bar): + if bar == 'baz': + raise Exception("catch me no matter what I am") + foo("baz") + """ + def _catch(*args, **kwargs): + try: + func(*args, **kwargs) + except Exception, exc: + exception(exc) + return _catch + class LoggerFactory(object): """ This is a logger factory to be used by oonib diff --cc ooni/utils/net.py index 12d1939,1814df2..2495197 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@@ -10,9 -10,8 +10,10 @@@ # see attached LICENCE file import sys +import socket +from random import randint + from ipaddr import IPAddress from zope.interface import implements from twisted.internet import protocol, defer from twisted.internet import threads, reactor @@@ -125,34 -141,18 +128,42 @@@ def getIfaces(platform_name=None) else: raise UnsupportedPlatform +def randomFreePort(addr="127.0.0.1"): + """ + Args: + + addr (str): the IP address to attempt to bind to. + + Returns an int representing the free port number at the moment of calling + + Note: there is no guarantee that some other application will attempt to + bind to this port once this function has been called. + """ + free = False + while not free: + port = randint(1024, 65535) + s = socket.socket() + try: + s.bind((addr, port)) + free = True + except: + pass + s.close() + return port + + def checkInterfaces(ifaces=None, timeout=1): """ + Check given network interfaces to see that they can send and receive + packets. This is similar to :func:`getDefaultIface`, except that function + only retrieves the name of the interface which is associated with the LAN, + whereas this function validates tx/rx capabilities. + @param ifaces: - A dictionary in the form of ifaces['if_name'] = 'if_addr'. + (optional) A dictionary in the form of ifaces['if_name'] = 'if_addr'. + @param timeout: + An integer specifying the number of seconds to timeout if + no reply is received for our pings. """ try: from scapy.all import IP, ICMP diff --cc ooni/utils/txscapy.py index 7902133,702d7bb..e91da09 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@@ -17,146 -18,53 +18,146 @@@ from twisted.internet import reactor, t from twisted.internet import defer, abstract from zope.interface import implements - +from scapy.config import conf + from scapy.all import PcapWriter, MTU + from scapy.all import BasePacketList, conf, PcapReader - -from scapy.all import conf, Gen, SetGen ++from scapy.all import Gen, SetGen from ooni.utils import log +from ooni import config + +try: + conf.use_pcap = True + conf.use_dnet = True + + from scapy.all import PcapWriter + from scapy.arch import pcapdnet + + config.pcap_dnet = True + +except ImportError, e: + log.err("pypcap or dnet not installed. " + "Certain tests may not work.") + + config.pcap_dnet = False + conf.use_pcap = False + conf.use_dnet = False + + from scapy.all import PcapWriter - from scapy.all import BasePacketList, conf, PcapReader - from scapy.all import conf, Gen, SetGen, MTU -class TXPcapWriter(PcapWriter): - def __init__(self, *arg, **kw): - PcapWriter.__init__(self, *arg, **kw) - fdesc.setNonBlocking(self.f) ++from scapy.all import BasePacketList, PcapReader ++from scapy.all import Gen, SetGen, MTU -class ScapyProtocol(abstract.FileDescriptor): - def __init__(self, super_socket=None, - reactor=None, timeout=None, receive=True, *a, **kw): +def getNetworksFromRoutes(): + from scapy.all import conf, ltoa, read_routes + from ipaddr import IPNetwork, IPAddress + - ## Hide the 'no routes' warnings - conf.verb = 0 - + networks = [] + for nw, nm, gw, iface, addr in read_routes(): + n = IPNetwork( ltoa(nw) ) + (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]] + n.iface = iface + if not n.compressed in networks: + networks.append(n) + + return networks + +class IfaceError(Exception): + pass + +def getDefaultIface(): + networks = getNetworksFromRoutes() + for net in networks: + if net.is_private: + return net.iface + raise IfaceError + +class ProtocolNotRegistered(Exception): + pass + +class ProtocolAlreadyRegistered(Exception): + pass + +class ScapyFactory(abstract.FileDescriptor): + """ + Inspired by muxTCP scapyLink: + https://github.com/enki/muXTCP/blob/master/scapyLink.py + """ + def __init__(self, interface, super_socket=None, timeout=5): abstract.FileDescriptor.__init__(self, reactor) - # By default we use the conf.L3socket + if interface == 'auto': + interface = getDefaultIface() if not super_socket: - super_socket = conf.L3socket() + super_socket = conf.L3socket(iface=interface, + promisc=True, filter='') + #super_socket = conf.L2socket(iface=interface) + + self.protocols = [] + fdesc._setCloseOnExec(super_socket.ins.fileno()) self.super_socket = super_socket - self.timeout = timeout + def writeSomeData(self, data): + """ + XXX we actually want to use this, but this requires overriding doWrite + or writeSequence. + """ + pass - # This dict is used to store the unique hashes that allow scapy to - # match up request with answer - self.hr_sent_packets = {} + def send(self, packet): + """ + Write a scapy packet to the wire. + """ + return self.super_socket.send(packet) - # These are the packets we have received as answer to the ones we sent - self.answered_packets = [] + def fileno(self): + return self.super_socket.ins.fileno() - # These are the packets we send - self.sent_packets = [] + def doRead(self): + packet = self.super_socket.recv(MTU) + if packet: + for protocol in self.protocols: + protocol.packetReceived(packet) - # This deferred will fire when we have finished sending a receiving packets. - self.d = defer.Deferred() - self.debug = False + def registerProtocol(self, protocol): + if not self.connected: + self.startReading() - self.multi = False - # XXX this needs to be implemented. It would involve keeping track of - # the state of the sending via the super socket file descriptor and - # firing the callback when we have concluded sending. Check out - # twisted.internet.udp to see how this is done. - self.receive = receive + if protocol not in self.protocols: + protocol.factory = self + self.protocols.append(protocol) + else: + raise ProtocolAlreadyRegistered - def fileno(self): - return self.super_socket.ins.fileno() + def unRegisterProtocol(self, protocol): + if protocol in self.protocols: + self.protocols.remove(protocol) + if len(self.protocols) == 0: + self.loseConnection() + else: + raise ProtocolNotRegistered + +class ScapyProtocol(object): + factory = None + + def packetReceived(self, packet): + """ + When you register a protocol, this method will be called with argument + the packet it received. + + Every protocol that is registered will have this method called. + """ + raise NotImplementedError + +class ScapySender(ScapyProtocol): + timeout = 5 + + # This deferred will fire when we have finished sending a receiving packets. + # Should we look for multiple answers for the same sent packet? + multi = False + + # When 0 we stop when all the packets we have sent have received an + # answer + expected_answers = 0 def processPacket(self, packet): """ @@@ -172,18 -79,11 +173,18 @@@ if not self.multi: del(answer_hr[i]) break + if len(self.answered_packets) == len(self.sent_packets): - log.debug("All of our questions have been answered.") + # All of our questions have been answered. self.stopSending() + return - def doRead(self): + if self.expected_answers and \ + self.expected_answers == len(self.answered_packets): + log.debug("Got the number of expected answers") + self.stopSending() + + def packetReceived(self, packet): timeout = time.time() - self._start_time if self.timeout and time.time() - self._start_time > self.timeout: self.stopSending() diff --cc ooniprobe.conf.sample index ed77dfe,0000000..7031ba1 mode 100644,000000..100644 --- a/ooniprobe.conf.sample +++ b/ooniprobe.conf.sample @@@ -1,38 -1,0 +1,41 @@@ +# This is the configuration file for OONIProbe +# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html +# Keep in mind that indentation matters. + +basic: + # Where OONIProbe should be writing it's log file + logfile: ooniprobe.log +privacy: + # Should we include the IP address of the probe in the report? + includeip: false + # Should we include the ASN of the probe in the report? + includeasn: false + # Should we include the ASN of the probe in the report? + includecountry: false + # Should we include the ASN of the probe in the report? + includecity: false + # Should we collect a full packet capture on the client? + includepcap: false +advanced: + # XXX change this to point to the directory where you have stored the GeoIP + # database file. This should be the directory in which OONI is installed + # /path/to/ooni-probe/data/ + geoip_data_dir: /usr/share/GeoIP/ + debug: true ++ # If we're including the client's IP address, how long should we wait for ++ # the connection to complete before timing out? ++ checktimeout: 15 + tor_binary: '/usr/sbin/tor' + # For auto detection + interface: auto + # Of specify a specific interface + #interface: wlan0 + # If you do not specify start_tor, you will have to have Tor running and + # explicitly set the control port and orport. + start_tor: true +tor: + #socks_port: 9050 + #control_port: 9051 + # Specify the absolute path to the Tor bridges to use for testing + bridges: bridges.list +

1 0