- tor-commits - lists.torproject.org

[ooni-probe/master] Merge remote-tracking branch 'tpo-common/master'
by isis＠torproject.org 18 Dec '12

18 Dec '12

1 0

[ooni-probe/master] Merge branch 'dev-missing-test'
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 705587ae5d974bca56d0ac4c407bec4c312c37f4 Merge: d8d286a d38df33 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Dec 18 04:07:47 2012 +0000 Merge branch 'dev-missing-test' ooni/runner.py | 30 +++++++++++++++++++++--------- 1 files changed, 21 insertions(+), 9 deletions(-)

1 0

[ooni-probe/master] Merge branch 'master' into tcp-flags
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit d8d286ac341dfc1a9cb56e8d5d4d1c49e7fee53a Merge: 2e93940 8d3a668 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Dec 18 03:55:51 2012 +0000 Merge branch 'master' into tcp-flags Conflicts: ooni/nettest.py ooni/oonicli.py ooni/reporter.py ooni/runner.py ooni/utils/__init__.py ooni/utils/geodata.py ooni/utils/hacks.py ooni/utils/log.py ooni/utils/net.py ooni/utils/txscapy.py scripts/before_i_commit.sh README.md | 51 +-- before_i_commit.sh | 29 -- before_i_commit.testdeck | 33 -- bin/INSTRUCTIONS | 15 - bin/Makefile | 54 --- bin/canary | 27 -- bin/old_ooniprobe | 80 ---- bin/oonib | 4 - decks/before_i_commit.testdeck | 33 ++ docs/source/api/modules.rst | 7 - docs/source/api/ooni.rst | 8 - docs/source/api/ooni.templates.rst | 22 +- docs/source/api/ooni.utils.rst | 2 - docs/source/architecture.rst | 204 ++++++++++ docs/source/conf.py | 4 +- docs/source/index.rst | 6 +- docs/source/nettests/modules.rst | 8 + docs/source/nettests/nettests.blocking.rst | 26 ++ docs/source/nettests/nettests.examples.rst | 58 +++ docs/source/nettests/nettests.experimental.rst | 58 +++ docs/source/nettests/nettests.manipulation.rst | 58 +++ docs/source/nettests/nettests.scanning.rst | 10 + docs/source/nettests/nettests.third_party.rst | 10 + docs/source/reports.rst | 9 +- docs/source/tests/dnsspoof.rst | 111 ++++++ docs/source/tests/dnstamper.rst | 6 +- .../tests/http_header_field_manipulation.rst | 343 ++++++++++++++++ docs/source/tests/http_host.rst | 10 +- docs/source/tests/http_invalid_request_line.rst | 4 +- docs/source/tests/http_requests.rst | 4 +- docs/source/tests/tcpconnect.rst | 2 +- docs/source/tests/traceroute.rst | 2 +- docs/source/writing_tests.rst | 275 +++++++++++--- nettests/blocking/__init__.py | 1 + nettests/blocking/dnstamper.py | 6 +- nettests/examples/example_http_checksum.py | 27 ++ nettests/third_party/netalyzr.py | 1 - ooni/__init__.py | 3 +- ooni/config.py | 37 +- ooni/kit/daphn3.py | 4 +- ooni/kit/domclass.py | 60 ++-- ooni/lib/__init__.py | 5 - ooni/lib/secdev.org.pem | 20 - ooni/nettest.py | 17 + ooni/nodes.py | 174 -------- ooni/oonicli.py | 23 +- ooni/otime.py | 3 - ooni/reporter.py | 97 +++-- ooni/runner.py | 54 ++- ooni/templates/dnst.py | 57 ++-- ooni/templates/httpt.py | 4 - ooni/templates/scapyt.py | 5 - ooni/utils/__init__.py | 35 ++- ooni/utils/geodata.py | 33 -- ooni/utils/hacks.py | 4 - ooni/utils/log.py | 34 +- ooni/utils/txscapy.py | 13 +- oonib/config.py | 10 + oonib/oonibackend.py | 4 +- oonib/report/__init__.py | 3 - oonib/report/api.py | 6 +- oonib/report/file_collector.py | 96 ++++-- oonib/requirements.txt | 40 +-- oonib/runner.py | 3 + oonib/testhelpers/ssl_helpers.py | 2 +- requirements.txt | 2 - scripts/before_i_commit.sh | 43 ++ scripts/example_parser.py | 22 + scripts/submit-patch | 100 +++++ setup.py | 37 ++ submit-patch | 100 ----- tests/test_runner.py | 20 +- tests/test_utils.py | 20 + to-be-ported/TODO | 418 -------------------- to-be-ported/spec/proxooni-spec.txt | 65 --- to-be-ported/very-old/TODO.plgoons | 79 ---- to-be-ported/very-old/TO_BE_PORTED | 14 - to-be-ported/very-old/ooni-probe.diff | 358 ----------------- to-be-ported/very-old/ooni/#namecheck.py# | 39 -- to-be-ported/very-old/ooni/.DS_Store | Bin 15364 -> 0 bytes to-be-ported/very-old/ooni/__init__.py | 12 - to-be-ported/very-old/ooni/command.py | 250 ------------ to-be-ported/very-old/ooni/dns_poisoning.py | 43 -- to-be-ported/very-old/ooni/dnsooni.py | 356 ----------------- to-be-ported/very-old/ooni/helpers.py | 38 -- to-be-ported/very-old/ooni/http.py | 306 -------------- to-be-ported/very-old/ooni/input.py | 33 -- to-be-ported/very-old/ooni/namecheck.py | 39 -- .../very-old/ooni/plugins/dnstest_plgoo.py | 84 ---- to-be-ported/very-old/ooni/plugins/http_plgoo.py | 70 ---- to-be-ported/very-old/ooni/plugins/marco_plgoo.py | 377 ------------------ to-be-ported/very-old/ooni/plugins/proxy_plgoo.py | 69 ---- .../very-old/ooni/plugins/simple_dns_plgoo.py | 35 -- to-be-ported/very-old/ooni/plugins/tcpcon_plgoo.py | 278 ------------- to-be-ported/very-old/ooni/plugins/tor.py | 80 ---- to-be-ported/very-old/ooni/plugins/torrc | 9 - to-be-ported/very-old/ooni/plugooni.py | 106 ----- to-be-ported/very-old/ooni/transparenthttp.py | 41 -- var/old_notes.txt | 418 ++++++++++++++++++++ var/proxooni-spec.txt | 65 +++ var/secdev.org.pem | 20 + 101 files changed, 2327 insertions(+), 4173 deletions(-) diff --cc ooni/oonicli.py index 0bf4d55,b06bde9..6afc453 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@@ -105,34 -75,21 +105,23 @@@ def updateStatusBar(stop_func) # moment. eta = config.state[test_filename].eta() progress = config.state[test_filename].progress() - progress_bar_frmt = "[%s] %s%%" % (test_filename, progress) - print progress_bar_frmt + while progress is not None: + print "[%s] %s%%" % (test_filename, progress) + else: + print "[%s] All tests in file completed." % test_filename + stop_func() def testsEnded(*arg, **kw): - """ - You can place here all the post shutdown tasks. - """ - log.debug("testsEnded: Finished running all tests") + """You can place here all the post shutdown tasks.""" + log.debug("Finished running all tests") config.start_reactor = False - try: reactor.stop() - except: pass + if not reactor.running: + try: reactor.stop() + except: reactor.runUntilCurrent() - def startSniffing(): - from ooni.utils.txscapy import ScapyFactory, ScapySniffer - try: - checkForRoot() - except PermissionsError: - print "[!] Includepcap options requires root priviledges to run" - print " you should run ooniprobe as root or disable the options in ooniprobe.conf" - sys.exit(1) - - print "Starting sniffer" - config.scapyFactory = ScapyFactory(config.advanced.interface) - - sniffer = ScapySniffer(config.reports.pcap) - config.scapyFactory.registerProtocol(sniffer) + def testFailed(failure): + log.err("Failed in running a test inside a test list") + failure.printTraceback() def runTestList(none, test_list): """ @@@ -149,16 -106,12 +138,17 @@@ deck_dl.append(d1) d2 = defer.DeferredList(deck_dl) - d2.addBoth(testsEnded) + d2.addCallback(testsEnded) + d2.addErrback(testFailed) - # Print every 5 second the list of current tests running - l = task.LoopingCall(updateStatusBar) - l.start(5.0) + try: + # Print every 5 second the list of current tests running + coop = task.Cooperator(started=False) + coop.cooperate(updateStatusBar) #this will need a .next() method + coop.start() + except StopIteration: + return d2 + return d2 def errorRunningTests(failure): @@@ -177,18 -132,13 +168,20 @@@ def run() log.start(cmd_line_options['logfile']) + config.cmd_line_options = cmd_line_options + if config.privacy.includepcap: - log.msg("Starting") - if not config.reports.pcap: - config.generatePcapFilename() - runner.startSniffing() + try: + checkForRoot() + except PermissionsError, pe: + m = ("Capturing packets requires administrator/root privileges. ", + "Run ooniprobe as root or set 'includepcap = false' in ", + "ooniprobe.conf .") + log.warn("%s" % m) + sys.exit(1) + else: + log.msg("Starting packet capture") + runner.startSniffing() resume = cmd_line_options['resume'] diff --cc ooni/reporter.py index 89a62ba,728c3f5..aac163d --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -1,20 -1,12 +1,21 @@@ +#-*- coding: utf-8 -*- +# +# reporter.py +# ----------- +# In here goes the logic for the creation of ooniprobe reports. +# +# :authors: Arturo Filastò, Isis Lovecruft +# :license: see included LICENSE file + + import traceback import itertools import logging +import sys +import os import time import yaml import json - import traceback -import sys -import os + import re from yaml.representer import * from yaml.emitter import * @@@ -25,15 -17,20 +26,16 @@@ from twisted.trial import reporte from twisted.internet import defer, reactor from twisted.internet.error import ConnectionRefusedError -from ooni.utils import log +from ooni import config, otime - from ooni.utils import log, geodata ++from ooni.utils import log, geodata, pushFilenameStack +from ooni.utils.net import BodyReceiver, StringProducer, userAgents try: from scapy.packet import Packet except ImportError: log.err("Scapy is not installed.") + -from ooni import otime -from ooni.utils import geodata, pushFilenameStack -from ooni.utils.net import BodyReceiver, StringProducer, userAgents - -from ooni import config - def createPacketReport(packet_list): """ Takes as input a packet a list. @@@ -159,8 -152,8 +157,8 @@@ def getTestDetails(options) 'test_version': options['version'], 'software_name': 'ooniprobe', 'software_version': software_version - } + } - defer.returnValue(test_details) + return test_details class OReporter(object): def __init__(self, cmd_line_options): @@@ -307,15 -310,12 +315,12 @@@ class OONIBReporter(OReporter) """ Creates a report on the oonib collector. """ - test_name = options['name'] - test_version = options['version'] - - url = self.backend_url + '/report/new' + url = self.backend_url + '/report' try: - test_details = yield getTestDetails(options) + test_details = getTestDetails(options) - except Exception, e: - log.exception(e) + except Exception, ex: + log.exception(ex) test_details['options'] = self.cmd_line_options diff --cc ooni/runner.py index 19dc171,4ebfa0b..7488d76 --- a/ooni/runner.py +++ b/ooni/runner.py @@@ -27,50 -15,18 +27,54 @@@ from twisted.trial.unittest import Skip from txtorcon import TorProtocolFactory, TorConfig from txtorcon import TorState, launch_tor -from ooni import config - +from ooni import config, nettest, reporter +from ooni.inputunit import InputUnitFactory from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError + + from ooni.inputunit import InputUnitFactory + from ooni.nettest import NetTestCase, NoPostProcessor + -from ooni.utils import log, checkForRoot, pushFilenameStack -from ooni.utils import NotRootError, Storage +from ooni.utils import log, checkForRoot +from ooni.utils import PermissionsError, Storage from ooni.utils.net import randomFreePort + +class NoTestCasesFound(Exception): + pass + +class InvalidResumeFile(Exception): + pass + +class noResumeSession(Exception): + pass + +class InvalidConfigFile(Exception): + message = "Invalid setting in ooniprobe.conf: " + +class UnableToStartTor(Exception): + pass + + +def isTestCase(obj): + """Return True if obj is a subclass of NetTestCase, False otherwise.""" + try: + return issubclass(obj, nettest.NetTestCase) + except TypeError: + return False + +def checkRequiredOptions(test_instance): + """ + If test_instance has an attribute 'requiredOptions', then check that + those options were utilised on the commandline. + """ + required = getattr(test_instance, 'requiredOptions', None) + if required: + for required_option in required: + log.debug("Checking if %s is present" % required_option) + if not test_instance.localOptions[required_option]: + raise usage.UsageError("%s not specified!" % required_option) + - def processTest(obj): + def processTest(obj, cmd_line_options): """ Process the parameters and :class:`twisted.python.usage.Options` of a :class:`ooni.nettest.Nettest`. @@@ -101,29 -55,43 +105,29 @@@ obj.usageOptions.optFlags = [] for flag in obj.baseFlags: obj.usageOptions.optFlags.append(flag) + if obj.inputFile: # inputFile is the optParameters list + obj.usageOptions.optParameters.append(obj.inputFile) options = obj.usageOptions() - options.parseOptions(config.cmd_line_options['subargs']) - + options.parseOptions(cmd_line_options['subargs']) obj.localOptions = options - if obj.inputFile: + if obj.inputFile: # inputFilename is the actual filename obj.inputFilename = options[obj.inputFile[0]] try: - log.debug("processing options") - tmp_test_case_object = obj() - tmp_test_case_object._checkRequiredOptions() - - except usage.UsageError, e: - test_name = tmp_test_case_object.name - log.err("There was an error in running %s!" % test_name) - log.err("%s" % e) + log.debug("Parsing commandline options") + tmp_test_instance = obj() + checkRequiredOptions(tmp_test_instance) + except usage.UsageError, ue: + log.err("%s" % ue) options.opt_help() - raise usage.UsageError("Error in parsing command line args for %s" % test_name) - - if obj.requiresRoot: - try: - checkForRoot() - except NotRootError: - log.err("%s requires root to run" % obj.name) - sys.exit(1) - - return obj - -def isTestCase(obj): - try: - return issubclass(obj, NetTestCase) - except TypeError: - return False + raise usage.UsageError("Error parsing command line args for %s" + % tmp_test_case_object.name) + else: + return obj - def findTestClassesFromFile(filename): + def findTestClassesFromFile(cmd_line_options): """ Takes as input the command line config parameters and returns the test case classes. @@@ -456,9 -353,11 +462,8 @@@ def increaseInputUnitIdx(test_filename) config.stateDict[test_filename] += 1 yield updateResumeFile(test_filename) -def updateProgressMeters(test_filename, input_unit_factory, - test_case_number): - """ - Update the progress meters for keeping track of test state. - """ +def updateProgressMeters(test_filename, input_unit_factory, test_case_number): + """Update the progress meters for keeping track of test state.""" - log.msg("Setting up progress meters") if not config.state.test_filename: config.state[test_filename] = Storage() @@@ -560,8 -451,16 +565,13 @@@ def runTestCases(test_cases, options, c except Exception: log.exception("Problem in running test") + yaml_reporter.finish() -class UnableToStartTor(Exception): - pass - def startTor(): + """ Starts Tor + Launches a Tor with :param: socks_port :param: control_port + :param: tor_binary set in ooniprobe.conf + """ @defer.inlineCallbacks def state_complete(state): config.tor_state = state @@@ -631,6 -539,11 +650,12 @@@ def startSniffing() print "Starting sniffer" config.scapyFactory = ScapyFactory(config.advanced.interface) - if os.path.exists(config.reports.pcap): ++ pcapfile = config.reports.pcap ++ if pcapfile and os.path.exists(pcapfile): + print "Report PCAP already exists with filename %s" % config.reports.pcap + print "Renaming files with such name..." + pushFilenameStack(config.reports.pcap) + sniffer = ScapySniffer(config.reports.pcap) config.scapyFactory.registerProtocol(sniffer) diff --cc ooni/utils/__init__.py index be5b38a,8510a3b..efa609a --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@@ -1,13 -1,10 +1,10 @@@ - """ - - """ - +import imp +import os import logging import string import random + import glob import yaml -import imp -import os class Storage(dict): """ @@@ -88,3 -82,34 +85,33 @@@ def randomStr(length, num=True) if num: chars += string.digits return ''.join(random.choice(chars) for x in range(length)) + - + def pushFilenameStack(filename): + """ + Takes as input a target filename and checks to see if a file by such name + already exists. If it does exist then it will attempt to rename it to .1, + if .1 exists it will rename .1 to .2 if .2 exists then it will rename it to + .3, etc. + This is similar to pushing into a LIFO stack. + + XXX: This will not work with stacks bigger than 10 elements because + glob.glob(".*") will return them in the wrong order (a.1, a.11, a.2, a.3, + etc.) + This is probably not an issue since the only thing it causes is that files + will be renamed in the wrong order and you shouldn't have the same report + filename for more than 10 reports anyways, because you should be making + ooniprobe generate the filename for you. + + Args: + filename (str): the path to filename that you wish to create. + """ + stack = glob.glob(filename+".*") + for f in reversed(stack): + c_idx = f.split(".")[-1] + c_filename = '.'.join(f.split(".")[:-1]) + new_idx = int(c_idx) + 1 + new_filename = "%s.%s" % (c_filename, new_idx) + os.rename(f, new_filename) + os.rename(filename, filename+".1") + + diff --cc ooni/utils/hacks.py index 4cf94d0,64b5a53..4bbdf48 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@@ -1,17 -1,5 +1,13 @@@ +# -*- encoding: utf-8 -*- +# +# hacks.py +# ******** # When some software has issues and we need to fix it in a # hackish way, we put it in here. This one day will be empty. +# +# :authors: Arturo Filastò, Isis Lovecruft +# :licence: see LICENSE + - from yaml.representer import * - from yaml.emitter import * - from yaml.serializer import * - from yaml.resolver import * import copy_reg diff --cc ooni/utils/log.py index 2721807,0740c10..70543ce --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@@ -1,24 -1,28 +1,31 @@@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from functools import wraps import sys import os -import logging import traceback +import logging from twisted.python import log as txlog + from twisted.python import util from twisted.python.failure import Failure from twisted.python.logfile import DailyLogFile from ooni import otime from ooni import config --## Get rid of the annoying "No route found for --## IPv6 destination warnings": --logging.getLogger("scapy.runtime").setLevel(logging.ERROR) + + class LogWithNoPrefix(txlog.FileLogObserver): + def emit(self, eventDict): + text = txlog.textFromEventDict(eventDict) + if text is None: + return + + util.untilConcludes(self.write, "%s\n" % text) + util.untilConcludes(self.flush) # Hoorj! def start(logfile=None, application_name="ooniprobe"): daily_logfile = None @@@ -46,21 -42,17 +45,22 @@@ txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit) def stop(): - txlog.msg("Stopping OONI") + print "Stopping OONI" def msg(msg, *arg, **kw): - txlog.msg(msg, logLevel=logging.INFO, *arg, **kw) + print "%s" % msg def debug(msg, *arg, **kw): - txlog.msg(msg, logLevel=logging.DEBUG, *arg, **kw) + if config.advanced.debug: + print "[D] %s" % msg +def warn(msg, *arg, **kw): + txlog.logging.captureWarnings('true') + txlog.logging.warn(msg) + #txlog.showwarning() + def err(msg, *arg, **kw): - txlog.err("Error: " + str(msg), logLevel=logging.ERROR, *arg, **kw) + print "[!] %s" % msg def exception(error): """ diff --cc ooni/utils/txscapy.py index 647f20e,62bde94..7fa31fa --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@@ -36,17 -31,20 +36,24 @@@ except ImportError config.pcap_dnet = False conf.use_pcap = False conf.use_dnet = False - from scapy.all import PcapWriter + + class DummyPcapWriter: + def __init__(self, pcap_filename, *arg, **kw): + log.err("Initializing DummyPcapWriter. We will not actually write to a pcapfile") - + def write(self): + pass - + PcapWriter = DummyPcapWriter +class ProtocolNotRegistered(Exception): + pass + +class ProtocolAlreadyRegistered(Exception): + pass + def getNetworksFromRoutes(): + """ Return a list of networks from the routing table """ from scapy.all import conf, ltoa, read_routes from ipaddr import IPNetwork, IPAddress diff --cc scripts/before_i_commit.sh index 0000000,918b137..a504ad8 mode 000000,100755..100755 --- a/scripts/before_i_commit.sh +++ b/scripts/before_i_commit.sh @@@ -1,0 -1,42 +1,43 @@@ -#!/bin/bash ++#!/bin/sh + # This script should be run before you commit to verify that the basic tests + # are working as they should + # Once you have run it you can inspect the log file via + # + # $ less before_i_commit.log + # To clean up everything that is left by the running of this tool, do as + # following: + # + # rm *.yamloo; rm before_i_commit.log + # + + if [ -f before_i_commit.log ]; + then + # this is technically the date it was moved, not the date it was created + mv before_i_commit.log before_i_commit-`date +%s`.log; + touch before_i_commit.log; + else + touch before_i_commit.log; + fi + + find . -type f -name "*.py[co]" -delete + + if [ -f env/bin/activate ]; + then + source env/bin/activate; + else + echo "Assuming that your virtual environment is pre-configured..."; + fi + + ./bin/ooniprobe -i decks/before_i_commit.testdeck + + echo "Below you should not see anything" + echo "---------------------------------" -grep "Error: " before_i_commit.log ++[ -f before_i_commit.log ] && grep "Error: " before_i_commit.log + echo "---------------------------------" + echo "If you do, it means something is wrong." + echo "Read through the log file and fix it." + echo "If you are having some problems fixing some things that have to do with" + echo "the core of OONI, let's first discuss it on IRC, or open a ticket" + read -#cat *.yamloo | less ++cat *yamloo | less ++rm -f *yamloo

1 0

[ooni-probe/master] Cleaned up handling of missing file in /nettest or /test.
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit d38df3373f9c56f191d17422e90f0703947d0093 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Dec 18 01:13:12 2012 +0000 Cleaned up handling of missing file in /nettest or /test. --- ooni/runner.py | 30 +++++++++++++++++++++--------- 1 files changed, 21 insertions(+), 9 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 4ebfa0b..94b2986 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -103,13 +103,19 @@ def findTestClassesFromFile(cmd_line_options): A list of class objects found in a file or module given on the commandline. """ - filename = cmd_line_options['test'] classes = [] - module = filenameToModule(filename) - for name, val in inspect.getmembers(module): - if isTestCase(val): - classes.append(processTest(val, cmd_line_options)) - return classes + filename = cmd_line_options['test'] + relative = filename.rsplit('/', 1)[1] + try: + module = filenameToModule(filename) + except ValueError, ve: + log.fail("%r doesn't exist." % relative) + else: + for name, val in inspect.getmembers(module): + if isTestCase(val): + classes.append(processTest(val, cmd_line_options)) + finally: + return classes def makeTestCases(klass, tests, method_prefix): """ @@ -556,6 +562,12 @@ def loadTest(cmd_line_options): # Ideally this would get all wrapped in a nice little class that get's # instanced with it's cmd_line_options as an instance attribute classes = findTestClassesFromFile(cmd_line_options) - test_cases, options = loadTestsAndOptions(classes, cmd_line_options) - - return test_cases, options, cmd_line_options + try: + test_cases, options = loadTestsAndOptions(classes, cmd_line_options) + return test_cases, options, cmd_line_options + except NoTestCasesFound, ntcf: + log.warn(ntcf) + if not 'testdeck' in cmd_line_options: # exit if this was this only test + sys.exit(1) # file and there aren't any tests + else: + pass # there are more tests, so continue

1 0

[ooni-probe/master] Fixed an error with the test_timeout callback taking the wrong parameters, and
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 623eeca5b719be980c8ddb0444c89eecaf5cadd6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Dec 18 05:18:44 2012 +0000 Fixed an error with the test_timeout callback taking the wrong parameters, and cleaned up a couple things due to the ooni logger formatting changes. --- ooni/oonicli.py | 2 +- ooni/runner.py | 11 +++++++---- ooni/utils/log.py | 15 +++++++-------- ooniprobe.conf.sample | 8 +++++--- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 6afc453..582a6ae 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -215,5 +215,5 @@ def run(): d.addErrback(errorRunningTests) # XXX I believe we don't actually need this: - reactor.run() + #reactor.run() diff --git a/ooni/runner.py b/ooni/runner.py index 4bb3e9f..a0710cd 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -248,10 +248,10 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, # This is used to store a copy of all the test reports tests_report = {} - def test_timeout(d): + def test_timeout(d, test_instance): timeout_error = defer.TimeoutError( "%s test for %s timed out after %s seconds" - % (test_name, test_instance.input, test_instance.timeout)) + % (test_instance.name, test_instance.input, test_instance.timeout)) timeout_fail = failure.Failure(err) try: d.errback(timeout_fail) @@ -323,8 +323,10 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, # call setups on the test test_instance._setUp() test_instance.setUp() + # get the timeout and _parents, in case it was set in setUp() test_instance.timeout = getTestTimeout(test_instance, test_method) + test_instance.timedOut = False test = getattr(test_instance, test_method) test_instance._testMethod = test @@ -332,7 +334,8 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, d = defer.maybeDeferred(test) # register the timer with the reactor - call_timeout = reactor.callLater(test_instance.timeout, test_timeout, d) + call_timeout = reactor.callLater(test_instance.timeout, test_timeout, d, + test_instance) d.addBoth(lambda x: call_timeout.active() and call_timeout.cancel() or x) # check if anything has been aborted or marked as 'skip' @@ -678,7 +681,7 @@ def loadTest(cmd_line_options): test_cases, options = loadTestsAndOptions(classes, cmd_line_options) return test_cases, options, cmd_line_options except NoTestCasesFound, ntcf: - log.warn(ntcf) + log.err(ntcf) if not 'testdeck' in cmd_line_options: # exit if this was this only test sys.exit(1) # file and there aren't any tests else: diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 70543ce..95163f3 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -50,17 +50,16 @@ def stop(): def msg(msg, *arg, **kw): print "%s" % msg -def debug(msg, *arg, **kw): +def debug(message, *arg, **kw): if config.advanced.debug: - print "[D] %s" % msg + print "[D] %s" % message -def warn(msg, *arg, **kw): - txlog.logging.captureWarnings('true') - txlog.logging.warn(msg) - #txlog.showwarning() +def warn(message, *arg, **kw): + if config.advanced.show_warnings: + print "[W] %s" % message -def err(msg, *arg, **kw): - print "[!] %s" % msg +def err(message, *arg, **kw): + print "[!] %s" % message def exception(error): """ diff --git a/ooniprobe.conf.sample b/ooniprobe.conf.sample index 2d6ef47..99faa4b 100644 --- a/ooniprobe.conf.sample +++ b/ooniprobe.conf.sample @@ -21,15 +21,17 @@ advanced: # GeoIP database file. This should be the directory in which OONI is # installed /path/to/ooni-probe/data/ geoip_data_dir: /usr/share/GeoIP/ - # Should we display debug messages: + # Should we display debug messages?: debug: true + # Should we display warning messages?: + show_warnings: true # How many seconds should we wait for connections before timing out?: default_timeout: 30 # Location where Tor is installed: tor_binary: '/usr/sbin/tor' - # For auto detection + # To automatically detect the system default networking interface, use: interface: auto - # Of specify a specific interface + # Or specify a specific interface: #interface: wlan0 # If you do not specify start_tor, you will have to have Tor running and # explicitly set the control port and orport.

1 0

[ooni-probe/master] Reactor seems to not be cleared, or my actual network here is severely messed
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 9a2cb04b2f5938e8d11ecd13ba0e5d4b01f9cdef Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 16 11:22:14 2012 +0000 Reactor seems to not be cleared, or my actual network here is severely messed up. --- nettests/bridge_reachability/echo.py | 75 ++++++++++++++++++---------------- 1 files changed, 40 insertions(+), 35 deletions(-) diff --git a/nettests/bridge_reachability/echo.py b/nettests/bridge_reachability/echo.py index 40436e7..df121b4 100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@ -17,15 +17,15 @@ import sys from twisted.python import usage from twisted.internet import reactor, defer -from ooni import nettest +from ooni.nettest import NetTestCase from ooni.utils import log, net, Storage try: - from scapy.all import IP, ICMP - from scapy.all import sr1 - from ooni.lib import txscapy - from ooni.lib.txscapy import txsr, txsend - from ooni.templates.scapyt import BaseScapyTest + from scapy.all import IP, ICMP + from scapy.all import sr1 + from ooni.lib import txscapy + from ooni.lib.txscapy import txsr, txsend + from ooni.templates.scapyt import BaseScapyTest except: log.msg("This test requires scapy, see www.secdev.org/projects/scapy") @@ -41,9 +41,9 @@ class UsageOptions(usage.Options): ['pcap', 'p', None, 'Save pcap to this file'], ['receive', 'r', True, 'Receive response packets']] -class EchoTest(nettest.NetTestCase): +class EchoTest(BaseScapyTest): """ - xxx fill me in + Basic ping test. This takes an input file containing one IP or hostname """ name = 'echo' author = 'Isis Lovecruft <isis(a)torproject.org>' @@ -69,7 +69,7 @@ class EchoTest(nettest.NetTestCase): iface = net.getDefaultIface() except Exception, e: log.msg("No network interface specified!") - log.err(e) + log.exception(e) else: log.msg("Using system default interface: %s" % iface) self.interface = iface @@ -85,10 +85,10 @@ class EchoTest(nettest.NetTestCase): if not self.dst: if self.file: self.dstProcessor(self.file) - for key, value in self.destinations.items(): - for label, data in value.items(): - if not 'ans' in data: - self.dst = label + for address, details in self.destinations.items(): + for labels, data in details.items(): + if not 'ans' in labels: + self.dst = details['dst_ip'] else: self.addDest(self.dst) log.debug("self.dst is now: %s" % self.dst) @@ -100,8 +100,6 @@ class EchoTest(nettest.NetTestCase): self.destinations[d] = {'dst_ip': d} def dstProcessor(self, inputfile): - from ipaddr import IPAddress - if os.path.isfile(inputfile): with open(inputfile) as f: for line in f.readlines(): @@ -110,23 +108,30 @@ class EchoTest(nettest.NetTestCase): self.addDest(line) def test_icmp(self): - def process_response(echo_reply, dest): - ans, unans = echo_reply - if ans: - log.msg("Recieved echo reply from %s: %s" % (dest, ans)) - else: - log.msg("No reply was received from %s. Possible censorship event." % dest) - log.debug("Unanswered packets: %s" % unans) - self.report[dest] = echo_reply - - for label, data in self.destinations.items(): - reply = sr1(IP(dst=lebal)/ICMP()) - process = process_reponse(reply, label) - - #(ans, unans) = ping - #self.destinations[self.dst].update({'ans': ans, - # 'unans': unans, - # 'response_packet': ping}) - #return ping - - #return reply + + def process_response(pkt, dest): + try: + ans, unans = pkt + if ans: + log.msg("Recieved echo-reply: %s" % pkt.summary()) + self.destinations[dest]['ans'] = a.show2() + self.report['response'] = [a.show2() for a in ans] + self.report['censored'] = False + else: + log.msg("No reply from %s. Possible censorship event." % dest) + log.debug("Unanswered packets: %s" % unans.summary()) + self.report['response'] = [u.show2() for u in unans] + self.report['censored'] = True + except Exception, e: + log.exception(e) + + try: + for dest, data in self.destinations.items(): + reply = txsr(IP(dst=dest)/ICMP(), + iface=self.interface, + retry=self.count, + multi=True, + timeout=self.timeout) + process = process_response(reply, dest) + except Exception, e: + log.exception(e)

1 0

[ooni-probe/master] Re-ported echo. Fixed parameters not being passed through txscapy.sr to parent methods.
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 8af6d133c5f48ae516842674602a5198c9dcece8 Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Nov 16 15:20:49 2012 +0000 Re-ported echo. Fixed parameters not being passed through txscapy.sr to parent methods. --- nettests/bridge_reachability/echo.py | 141 ++++++++++++++++++++++++--------- ooni/utils/txscapy.py | 12 ++- 2 files changed, 110 insertions(+), 43 deletions(-) diff --git a/nettests/bridge_reachability/echo.py b/nettests/bridge_reachability/echo.py index df121b4..06c3aa4 100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@ -16,34 +16,39 @@ import os import sys from twisted.python import usage -from twisted.internet import reactor, defer -from ooni.nettest import NetTestCase +from twisted.internet import reactor, defer, address +from ooni import nettest from ooni.utils import log, net, Storage try: - from scapy.all import IP, ICMP - from scapy.all import sr1 - from ooni.lib import txscapy - from ooni.lib.txscapy import txsr, txsend - from ooni.templates.scapyt import BaseScapyTest -except: + from scapy.all import IP, ICMP + from scapy.all import sr1 + from ooni.utils import txscapy +except Exception, e: log.msg("This test requires scapy, see www.secdev.org/projects/scapy") + log.exception(e) class UsageOptions(usage.Options): + """ + Options for EchoTest. + + Note: 'count', 'size', and 'ttl' have yet to be implemented. + """ optParameters = [ ['dst', 'd', None, 'Host IP to ping'], ['file', 'f', None, 'File of list of IPs to ping'], + ['pcap', 'p', None, 'Save pcap to this file'], ['interface', 'i', None, 'Network interface to use'], + ['receive', 'r', True, 'Receive response packets'], + ['timeout', 't', 2, 'Seconds to wait if no response', int], ['count', 'c', 1, 'Number of packets to send', int], - ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], - ['ttl', 'l', 25, 'Set the IP Time to Live', int], - ['timeout', 't', 2, 'Seconds until timeout if no response', int], - ['pcap', 'p', None, 'Save pcap to this file'], - ['receive', 'r', True, 'Receive response packets']] + ['size', 's', 56, 'Bytes to send in ICMP data field', int], + ['ttl', 'l', 25, 'Set the IP Time to Live', int]] -class EchoTest(BaseScapyTest): +class EchoTest(nettest.NetTestCase): """ Basic ping test. This takes an input file containing one IP or hostname + per line. """ name = 'echo' author = 'Isis Lovecruft <isis(a)torproject.org>' @@ -55,6 +60,22 @@ class EchoTest(BaseScapyTest): #requiredOptions = ['dst'] def setUp(self, *a, **kw): + """ + Send an ICMP-8 packet to a host IP, and process the response. + + @param timeout: + Seconds after sending the last packet to timeout. + @param interface: + The interface to restrict listening to. + @param dst: + A single host to ping. + @param file: + A file of hosts to ping, one per line. + @param receive: + Whether or not to receive replies. Defaults to True. + @param pcap: + The file to save packet captures to. + """ self.destinations = {} if self.localOptions: @@ -87,11 +108,10 @@ class EchoTest(BaseScapyTest): self.dstProcessor(self.file) for address, details in self.destinations.items(): for labels, data in details.items(): - if not 'ans' in labels: + if not 'response' in labels: self.dst = details['dst_ip'] else: self.addDest(self.dst) - log.debug("self.dst is now: %s" % self.dst) log.debug("Initialization of %s test completed." % self.name) @@ -107,31 +127,74 @@ class EchoTest(BaseScapyTest): continue self.addDest(line) - def test_icmp(self): + def build_packets(self): + """ + Construct a list of packets to send out. + """ + packets = [] + for dest, data in self.destinations.items(): + pkt = IP(dst=dest)/ICMP() + packets.append(pkt) + ## XXX if a domain was specified, we need a way to check that + ## its IP matches the one we're seeing in pkt.src + #try: + # address.IPAddress(dest) + #except: + # data['dst_ip'] = pkt.dst + return packets - def process_response(pkt, dest): - try: - ans, unans = pkt - if ans: - log.msg("Recieved echo-reply: %s" % pkt.summary()) - self.destinations[dest]['ans'] = a.show2() - self.report['response'] = [a.show2() for a in ans] - self.report['censored'] = False - else: - log.msg("No reply from %s. Possible censorship event." % dest) - log.debug("Unanswered packets: %s" % unans.summary()) - self.report['response'] = [u.show2() for u in unans] - self.report['censored'] = True - except Exception, e: - log.exception(e) + def test_icmp(self): + """ + Send the list of ICMP packets. + TODO: add end summary progress report for % answered, etc. + """ try: - for dest, data in self.destinations.items(): - reply = txsr(IP(dst=dest)/ICMP(), - iface=self.interface, - retry=self.count, - multi=True, - timeout=self.timeout) - process = process_response(reply, dest) + def nicely(packets): + """Print scapy summary nicely.""" + return list([x.summary() for x in packets]) + + def process_answered((answered, sent)): + """Callback function for txscapy.sr().""" + self.report['sent'] = nicely(sent) + self.report['answered'] = [nicely(ans) for ans in answered] + + for req, resp in answered: + log.msg("Received echo-reply:\n%s" % resp.summary()) + for dest, data in self.destinations.items(): + if data['dst_ip'] == resp.src: + data['response'] = resp.summary() + data['censored'] = False + for snd in sent: + if snd.dst == resp.src: + answered.remove((req, resp)) + return (answered, sent) + + def process_unanswered((unanswered, sent)): + """ + Callback function for remaining packets and destinations which + do not have an associated response. + """ + if len(unanswered) > 0: + nicer = [nicely(unans) for unans in unanswered] + log.msg("Unanswered/remaining packets:\n%s" + % nicer) + self.report['unanswered'] = nicer + for dest, data in self.destinations.items(): + if not 'response' in data: + log.msg("No reply from %s. Possible censorship event." + % dest) + data['response'] = None + data['censored'] = True + return (unanswered, sent) + + packets = self.build_packets() + d = txscapy.sr(packets, iface=self.interface, multi=True) + d.addCallback(process_answered) + d.addErrback(log.exception) + d.addCallback(process_unanswered) + d.addErrback(log.exception) + self.report['destinations'] = self.destinations + return d except Exception, e: log.exception(e) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 2b108ca..9c33d18 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -1,4 +1,5 @@ -# -*- coding:utf8 -*- +# -*- coding: utf-8 -*- +# # txscapy # ******* # Here shall go functions related to using scapy with twisted. @@ -32,7 +33,7 @@ class TXPcapWriter(PcapWriter): class ScapyProtocol(abstract.FileDescriptor): def __init__(self, super_socket=None, - reactor=None, timeout=None, receive=True): + reactor=None, timeout=None, receive=True, *a, **kw): abstract.FileDescriptor.__init__(self, reactor) # By default we use the conf.L3socket if not super_socket: @@ -54,7 +55,10 @@ class ScapyProtocol(abstract.FileDescriptor): # This deferred will fire when we have finished sending a receiving packets. self.d = defer.Deferred() self.debug = False + self.multi = False + if kw['multi']: + self.multi = kw['multi'] # XXX this needs to be implemented. It would involve keeping track of # the state of the sending via the super socket file descriptor and # firing the callback when we have concluded sending. Check out @@ -127,9 +131,9 @@ class ScapyProtocol(abstract.FileDescriptor): self.sendPackets(packets) return self.d -def sr(x, filter=None, iface=None, nofilter=0, timeout=None): +def sr(x, filter=None, iface=None, nofilter=0, timeout=None, *a, **kw): super_socket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter) - sp = ScapyProtocol(super_socket=super_socket, timeout=timeout) + sp = ScapyProtocol(super_socket=super_socket, timeout=timeout, *a, **kw) return sp.startSending(x) def send(x, filter=None, iface=None, nofilter=0, timeout=None):

1 0

[ooni-probe/master] Updated ooni.inputunit.InputUnit documentation, and fixed __repr__ to show
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 8ab5a3ff1b131cb7794ec79aabde2e5cc3f780fa Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Nov 18 21:39:40 2012 +0000 Updated ooni.inputunit.InputUnit documentation, and fixed __repr__ to show original input list. --- ooni/inputunit.py | 19 ++++++++++++++++--- 1 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ooni/inputunit.py b/ooni/inputunit.py index 65d578c..4a5c792 100644 --- a/ooni/inputunit.py +++ b/ooni/inputunit.py @@ -50,24 +50,37 @@ class InputUnitFactory(object): class InputUnit(object): """ This is a python iterable object that contains the input elements to be - passed onto a TestCase. + passed onto a :class:`ooni.nettest.NetTestCase`. """ def __init__(self, inputs=[]): + """ + Create an iterable from a list of inputs, which can be given to a NetTestCase. + + @param inputs: A list of inputs for a NetTestCase. + """ self._inputs = iter(inputs) + # _inputs_copy is to avoid stealing things from + # the iterator when __repr__ is called: + _inputs_copy = inputs + self.__contains__ = _inputs_copy def __repr__(self): - return "<%s inputs=%s>" % (self.__class__, self._inputs) + """Prints the original input list.""" + return "<%s inputs=%s>" % (self.__class__, self.__contains__) def __add__(self, inputs): + """Add a list of inputs to the iterator.""" for input in inputs: self._inputs.append(input) def __iter__(self): + """Self explanatory.""" return self def next(self): + """Return the next item from the InputUnit iterator.""" return self._inputs.next() def append(self, input): + """Add an item to the end of the InputUnit iterator.""" self._inputs.append(input) -

1 0

[ooni-probe/master] Consolidated a custom Exception class which was the same in ooni.utils.net and
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit ea1d801f242c3a2ef4204867e6b92504c3be5456 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Nov 18 21:45:02 2012 +0000 Consolidated a custom Exception class which was the same in ooni.utils.net and ooni.oonicli. * Cleaned up documentation and indentation in oonicli.py, runner.py, utils/net.py, and utils/__init__.py. * Removed all occurences of the duplicate Exception class. * Added IPv4/v6 parser checkIPandPort() to utils/net.py. --- ooni/oonicli.py | 33 +++++++++++----------- ooni/runner.py | 60 +++++++++++++++++++++------------------- ooni/utils/__init__.py | 17 +++++++----- ooni/utils/net.py | 71 +++++++++++++++++++++++++++++++++-------------- 4 files changed, 107 insertions(+), 74 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 9473ad8..263494b 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -24,7 +24,7 @@ from ooni import nettest, runner, reporter, config from ooni.inputunit import InputUnitFactory from ooni.utils import net -from ooni.utils import checkForRoot, NotRootError +from ooni.utils import checkForRoot, PermissionsError from ooni.utils import log class Options(usage.Options, app.ReactorSelectionMixin): @@ -37,13 +37,14 @@ class Options(usage.Options, app.ReactorSelectionMixin): optFlags = [["help", "h"], ['debug-stacktraces', 'B', - 'Report deferred creation and callback stack traces'],] + 'Report deferred creation and callback stack traces'],] - optParameters = [["reportfile", "o", None, "report file name"], - ["collector", "c", None, - "Address of the collector of test results. (example: http://127.0.0.1:8888)"], - ["logfile", "l", None, "log file name"], - ["pcapfile", "p", None, "pcap file name"]] + optParameters = [ + ["reportfile", "o", None, "report file name"], + ["collector", "c", None, + "Address of the collector of test results. (example: http://127.0.0.1:8888)"], + ["logfile", "l", None, "log file name"], + ["pcapfile", "p", None, "pcap file name"]] compData = usage.Completions( extraActions=[usage.CompleteFiles( @@ -75,15 +76,12 @@ class Options(usage.Options, app.ReactorSelectionMixin): raise usage.UsageError("No test filename specified!") def testsEnded(*arg, **kw): - """ - You can place here all the post shutdown tasks. - """ + """You can place here all the post shutdown tasks.""" log.debug("testsEnded: Finished running all tests") def run(): - """ - Call me to begin testing from a file. - """ + """Call me to begin testing from a file.""" + cmd_line_options = Options() if len(sys.argv) == 1: cmd_line_options.getUsage() @@ -120,15 +118,16 @@ def run(): if config.privacy.includepcap: try: checkForRoot() - except NotRootError: - log.err("includepcap options requires root priviledges to run") - log.err("you should run ooniprobe as root or disable the options in ooniprobe.conf") + except PermissionsError, pe: + log.err(str("'includepcap' option requires administrator or root ", + "privileges to run. Run ooniprobe as root or disable ", + "the includepcap option in ooniprobe.conf ")) sys.exit(1) log.debug("Starting sniffer") sniffer_d = net.capturePackets(pcap_filename) tests_d = runner.runTestCases(test_cases, options, - cmd_line_options, yamloo_filename) + cmd_line_options, yamloo_filename) tests_d.addBoth(testsEnded) reactor.run() diff --git a/ooni/runner.py b/ooni/runner.py index f7fa5dc..f1321cd 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -25,7 +25,7 @@ from ooni.nettest import NetTestCase from ooni import reporter -from ooni.utils import log, checkForRoot, NotRootError +from ooni.utils import log, checkForRoot, PermissionsError def processTest(obj, cmd_line_options): """ @@ -44,11 +44,10 @@ def processTest(obj, cmd_line_options): if obj.requiresRoot: try: checkForRoot() - except NotRootError: + except PermissionsError: log.err("%s requires root to run" % obj.name) sys.exit(1) - if obj.optParameters or input_file \ or obj.usageOptions or obj.optFlags: @@ -152,6 +151,9 @@ def loadTestsAndOptions(classes, cmd_line_options): return test_cases, options def runTestWithInput(test_class, test_method, test_input, oreporter): + """ + Runs a single testcase from a NetTestCase with one input. + """ log.debug("Running %s with %s" % (test_method, test_input)) def test_done(result, test_instance, test_name): @@ -178,33 +180,33 @@ def runTestWithInput(test_class, test_method, test_input, oreporter): log.debug("returning %s input" % test_method) return d -def runTestWithInputUnit(test_class, - test_method, input_unit, - oreporter): +def runTestWithInputUnit(test_class, test_method, input_unit, oreporter): """ - test_class: the uninstantiated class of the test to be run - - test_method: a string representing the method name to be called - - input_unit: a generator that contains the inputs to be run on the test - - oreporter: ooni.reporter.OReporter instance - - returns a deferred list containing all the tests to be run at this time + @param test_class: + The uninstantiated :class:`ooni.nettest.NetTestCase` to be run. + @param test_method: + A string representing the method name to be called. + @param input_unit: + A generator that contains the inputs to be run on the test. + @param oreporter: + A :class:`ooni.reporter.OReporter` instance. + + @return: A DeferredList containing all the tests to be run at this time. """ - dl = [] - log.debug("input unit %s" % input_unit) for test_input in input_unit: - log.debug("running with input: %s" % test_input) - d = runTestWithInput(test_class, - test_method, test_input, oreporter) + d = runTestWithInput(test_class, test_method, test_input, oreporter) dl.append(d) return defer.DeferredList(dl) @defer.inlineCallbacks def runTestCases(test_cases, options, - cmd_line_options, yamloo_filename): + cmd_line_options, yamloo_filename): + """ + XXX we should get rid of the InputUnit class, because we go though the + effort of creating an iterator, only to turn it back into a list, and then + iterate through it. it's also buggy as hell, and it's excess code. + """ try: assert len(options) != 0, "Length of options is zero!" except AssertionError, ae: @@ -225,7 +227,6 @@ def runTestCases(test_cases, options, reportFile = open(yamloo_filename, 'w+') - if cmd_line_options['collector']: oreporter = reporter.OONIBReporter(cmd_line_options['collector']) else: @@ -234,7 +235,6 @@ def runTestCases(test_cases, options, input_unit_factory = InputUnitFactory(test_inputs) log.debug("Creating report") - yield oreporter.createReport(options) # This deferred list is a deferred list of deferred lists @@ -243,17 +243,19 @@ def runTestCases(test_cases, options, try: for input_unit in input_unit_factory: log.debug("Running this input unit %s" % input_unit) - # We do this because generators can't we rewound. + # We do this because generators can't be rewound. input_list = list(input_unit) for test_case in test_cases: log.debug("Processing %s" % test_case[1]) test_class = test_case[0] test_method = test_case[1] - yield runTestWithInputUnit(test_class, - test_method, input_list, - oreporter) - except Exception: - log.exception("Problem in running test") + yield runTestWithInputUnit(test_class, test_method, + input_list, oreporter) + except Exception, ex: + # XXX we probably want to add a log.warn() at some point + log.msg("Problem in running test") + log.exception(ex) reactor.stop() + oreporter.allDone() diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 5947519..74e1fc2 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -51,16 +51,18 @@ class Storage(dict): for (k, v) in value.items(): self[k] = v -class NotRootError(Exception): - pass +class PermissionsError(Exception): + """This test requires administrator or root permissions.""" def checkForRoot(): + """Check permissions.""" if os.getuid() != 0: - raise NotRootError("This test requires root") + raise PermissionsError def randomSTR(length, num=True): """ - Returns a random all uppercase alfa-numerical (if num True) string long length + Returns a random, all-uppercase, alpha-numeric (if num=True), string of + specified character length. """ chars = string.ascii_uppercase if num: @@ -69,7 +71,8 @@ def randomSTR(length, num=True): def randomstr(length, num=True): """ - Returns a random all lowercase alfa-numerical (if num True) string long length + Returns a random, all-lowercase, alpha-numeric (if num=True), string + specified character length. """ chars = string.ascii_lowercase if num: @@ -78,8 +81,8 @@ def randomstr(length, num=True): def randomStr(length, num=True): """ - Returns a random a mixed lowercase, uppercase, alfanumerical (if num True) - string long length + Returns a random a mixed lowercase, uppercase, alpha-numeric (if num=True) + string of specified character length. """ chars = string.ascii_lowercase + string.ascii_uppercase if num: diff --git a/ooni/utils/net.py b/ooni/utils/net.py index c5b01a3..7c1c2a1 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -11,6 +11,7 @@ import sys +from ipaddr import IPAddress from zope.interface import implements from twisted.internet import protocol, defer from twisted.internet import threads, reactor @@ -18,6 +19,7 @@ from twisted.web.iweb import IBodyProducer from scapy.all import utils from ooni.utils import log, txscapy +from ooni.utils import PermissionsError #if sys.platform.system() == 'Windows': # import _winreg as winreg @@ -51,27 +53,6 @@ class UnsupportedPlatform(Exception): class IfaceError(Exception): """Could not find default network interface.""" -class PermissionsError(SystemExit): - """This test requires admin or root privileges to run. Exiting...""" - - -PLATFORMS = {'LINUX': sys.platform.startswith("linux"), - 'OPENBSD': sys.platform.startswith("openbsd"), - 'FREEBSD': sys.platform.startswith("freebsd"), - 'NETBSD': sys.platform.startswith("netbsd"), - 'DARWIN': sys.platform.startswith("darwin"), - 'SOLARIS': sys.platform.startswith("sunos"), - 'WINDOWS': sys.platform.startswith("win32")} - -class UnsupportedPlatform(Exception): - """Support for this platform is not currently available.""" - -class IfaceError(Exception): - """Could not find default network interface.""" - -class PermissionsError(SystemExit): - """This test requires admin or root privileges to run. Exiting...""" - class StringProducer(object): implements(IBodyProducer) @@ -220,6 +201,26 @@ def getNonLoopbackIfaces(platform_name=None): return interfaces def getNetworksFromRoutes(): + """ + + Get the networks this client is current on from the kernel routing table. + Each network is returned as a :class:`ipaddr.IPNetwork`, with the + network range as the name of the network, i.e.: + + network.compressed = '127.0.0.1/32' + network.netmask = IPv4Address('255.0.0.0') + network.ipaddr = IPv4Address('127.0.0.1') + network.gateway = IPv4Address('0.0.0.0') + network.iface = 'lo' + + This is mostly useful for retrieving the default network interface in a + portable manner, though it could be used to conduct local network checks + for things like rogue DHCP servers, or perhaps test that the clients NAT + router is not the mistakenly the source of a perceived censorship event. + + @return: A list of :class:`ipaddr.IPNetwork` objects with routing table + information. + """ from scapy.all import conf, ltoa, read_routes from ipaddr import IPNetwork, IPAddress @@ -237,6 +238,12 @@ def getNetworksFromRoutes(): return networks def getDefaultIface(): + """ + Get the client's default network interface. + + @return: A string containing the name of the default working interface. + @raise IfaceError: If no working interface is found. + """ networks = getNetworksFromRoutes() for net in networks: if net.is_private: @@ -244,5 +251,27 @@ def getDefaultIface(): raise IfaceError def getLocalAddress(): + """ + Get the rfc1918 IP address of the default working network interface. + + @return: The properly-formatted, validated, local IPv4/6 address of the + client's default working network interface. + """ default_iface = getDefaultIface() return default_iface.ipaddr + +def checkIPandPort(raw_ip, raw_port): + """ + Check that IP and Port are a legitimate address and portnumber. + + @return: The validated ip and port, else None. + """ + try: + port = int(raw_port) + assert port in xrange(1, 65535), "Port out of range." + ip = IPAddress(raw_ip) ## either IPv4 or IPv6 + except Exception, e: + log.err(e) + return + else: + return ip.compressed, port

1 0

[ooni-probe/master] Removed 'multi' kwarg from txscapy.ScapyProtocol for now, because Arturo is
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit f7f03cd80b7ebfb8bdfa9763afb8707f78117c4a Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Nov 18 22:00:35 2012 +0000 Removed 'multi' kwarg from txscapy.ScapyProtocol for now, because Arturo is correct that the parent classes don't correctly handle it. --- ooni/utils/txscapy.py | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 9c33d18..702d7bb 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -57,8 +57,6 @@ class ScapyProtocol(abstract.FileDescriptor): self.debug = False self.multi = False - if kw['multi']: - self.multi = kw['multi'] # XXX this needs to be implemented. It would involve keeping track of # the state of the sending via the super socket file descriptor and # firing the callback when we have concluded sending. Check out

1 0