commit ea1d801f242c3a2ef4204867e6b92504c3be5456 Author: Isis Lovecruft isis@torproject.org Date: Sun Nov 18 21:45:02 2012 +0000

Consolidated a custom Exception class which was the same in ooni.utils.net and ooni.oonicli.

* Cleaned up documentation and indentation in oonicli.py, runner.py, utils/net.py, and utils/__init__.py. * Removed all occurences of the duplicate Exception class. * Added IPv4/v6 parser checkIPandPort() to utils/net.py. --- ooni/oonicli.py | 33 +++++++++++----------- ooni/runner.py | 60 +++++++++++++++++++++------------------- ooni/utils/__init__.py | 17 +++++++----- ooni/utils/net.py | 71 +++++++++++++++++++++++++++++++++-------------- 4 files changed, 107 insertions(+), 74 deletions(-)

diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 9473ad8..263494b 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -24,7 +24,7 @@ from ooni import nettest, runner, reporter, config from ooni.inputunit import InputUnitFactory

from ooni.utils import net -from ooni.utils import checkForRoot, NotRootError +from ooni.utils import checkForRoot, PermissionsError from ooni.utils import log

class Options(usage.Options, app.ReactorSelectionMixin): @@ -37,13 +37,14 @@ class Options(usage.Options, app.ReactorSelectionMixin):

optFlags = [["help", "h"], ['debug-stacktraces', 'B', - 'Report deferred creation and callback stack traces'],] + 'Report deferred creation and callback stack traces'],]

- optParameters = [["reportfile", "o", None, "report file name"], - ["collector", "c", None, - "Address of the collector of test results. (example: http://127.0.0.1:8888)"], - ["logfile", "l", None, "log file name"], - ["pcapfile", "p", None, "pcap file name"]] + optParameters = [ + ["reportfile", "o", None, "report file name"], + ["collector", "c", None, + "Address of the collector of test results. (example: http://127.0.0.1:8888)"], + ["logfile", "l", None, "log file name"], + ["pcapfile", "p", None, "pcap file name"]]

compData = usage.Completions( extraActions=[usage.CompleteFiles( @@ -75,15 +76,12 @@ class Options(usage.Options, app.ReactorSelectionMixin): raise usage.UsageError("No test filename specified!")

def testsEnded(*arg, **kw): - """ - You can place here all the post shutdown tasks. - """ + """You can place here all the post shutdown tasks.""" log.debug("testsEnded: Finished running all tests")

def run(): - """ - Call me to begin testing from a file. - """ + """Call me to begin testing from a file.""" + cmd_line_options = Options() if len(sys.argv) == 1: cmd_line_options.getUsage() @@ -120,15 +118,16 @@ def run(): if config.privacy.includepcap: try: checkForRoot() - except NotRootError: - log.err("includepcap options requires root priviledges to run") - log.err("you should run ooniprobe as root or disable the options in ooniprobe.conf") + except PermissionsError, pe: + log.err(str("'includepcap' option requires administrator or root ", + "privileges to run. Run ooniprobe as root or disable ", + "the includepcap option in ooniprobe.conf ")) sys.exit(1) log.debug("Starting sniffer") sniffer_d = net.capturePackets(pcap_filename)

tests_d = runner.runTestCases(test_cases, options, - cmd_line_options, yamloo_filename) + cmd_line_options, yamloo_filename) tests_d.addBoth(testsEnded)

reactor.run() diff --git a/ooni/runner.py b/ooni/runner.py index f7fa5dc..f1321cd 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -25,7 +25,7 @@ from ooni.nettest import NetTestCase

from ooni import reporter

-from ooni.utils import log, checkForRoot, NotRootError +from ooni.utils import log, checkForRoot, PermissionsError

def processTest(obj, cmd_line_options): """ @@ -44,11 +44,10 @@ def processTest(obj, cmd_line_options): if obj.requiresRoot: try: checkForRoot() - except NotRootError: + except PermissionsError: log.err("%s requires root to run" % obj.name) sys.exit(1)

- if obj.optParameters or input_file \ or obj.usageOptions or obj.optFlags:

@@ -152,6 +151,9 @@ def loadTestsAndOptions(classes, cmd_line_options): return test_cases, options

def runTestWithInput(test_class, test_method, test_input, oreporter): + """ + Runs a single testcase from a NetTestCase with one input. + """ log.debug("Running %s with %s" % (test_method, test_input))

def test_done(result, test_instance, test_name): @@ -178,33 +180,33 @@ def runTestWithInput(test_class, test_method, test_input, oreporter): log.debug("returning %s input" % test_method) return d

-def runTestWithInputUnit(test_class, - test_method, input_unit, - oreporter): +def runTestWithInputUnit(test_class, test_method, input_unit, oreporter): """ - test_class: the uninstantiated class of the test to be run - - test_method: a string representing the method name to be called - - input_unit: a generator that contains the inputs to be run on the test - - oreporter: ooni.reporter.OReporter instance - - returns a deferred list containing all the tests to be run at this time + @param test_class: + The uninstantiated :class:`ooni.nettest.NetTestCase` to be run. + @param test_method: + A string representing the method name to be called. + @param input_unit: + A generator that contains the inputs to be run on the test. + @param oreporter: + A :class:`ooni.reporter.OReporter` instance. + + @return: A DeferredList containing all the tests to be run at this time. """ - dl = [] - log.debug("input unit %s" % input_unit) for test_input in input_unit: - log.debug("running with input: %s" % test_input) - d = runTestWithInput(test_class, - test_method, test_input, oreporter) + d = runTestWithInput(test_class, test_method, test_input, oreporter) dl.append(d) return defer.DeferredList(dl)

@defer.inlineCallbacks def runTestCases(test_cases, options, - cmd_line_options, yamloo_filename): + cmd_line_options, yamloo_filename): + """ + XXX we should get rid of the InputUnit class, because we go though the + effort of creating an iterator, only to turn it back into a list, and then + iterate through it. it's also buggy as hell, and it's excess code. + """ try: assert len(options) != 0, "Length of options is zero!" except AssertionError, ae: @@ -225,7 +227,6 @@ def runTestCases(test_cases, options,

reportFile = open(yamloo_filename, 'w+')

- if cmd_line_options['collector']: oreporter = reporter.OONIBReporter(cmd_line_options['collector']) else: @@ -234,7 +235,6 @@ def runTestCases(test_cases, options, input_unit_factory = InputUnitFactory(test_inputs)

log.debug("Creating report") - yield oreporter.createReport(options)

# This deferred list is a deferred list of deferred lists @@ -243,17 +243,19 @@ def runTestCases(test_cases, options, try: for input_unit in input_unit_factory: log.debug("Running this input unit %s" % input_unit) - # We do this because generators can't we rewound. + # We do this because generators can't be rewound. input_list = list(input_unit) for test_case in test_cases: log.debug("Processing %s" % test_case[1]) test_class = test_case[0] test_method = test_case[1] - yield runTestWithInputUnit(test_class, - test_method, input_list, - oreporter) - except Exception: - log.exception("Problem in running test") + yield runTestWithInputUnit(test_class, test_method, + input_list, oreporter) + except Exception, ex: + # XXX we probably want to add a log.warn() at some point + log.msg("Problem in running test") + log.exception(ex) reactor.stop() + oreporter.allDone()

diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 5947519..74e1fc2 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -51,16 +51,18 @@ class Storage(dict): for (k, v) in value.items(): self[k] = v

-class NotRootError(Exception): - pass +class PermissionsError(Exception): + """This test requires administrator or root permissions."""

def checkForRoot(): + """Check permissions.""" if os.getuid() != 0: - raise NotRootError("This test requires root") + raise PermissionsError

def randomSTR(length, num=True): """ - Returns a random all uppercase alfa-numerical (if num True) string long length + Returns a random, all-uppercase, alpha-numeric (if num=True), string of + specified character length. """ chars = string.ascii_uppercase if num: @@ -69,7 +71,8 @@ def randomSTR(length, num=True):

def randomstr(length, num=True): """ - Returns a random all lowercase alfa-numerical (if num True) string long length + Returns a random, all-lowercase, alpha-numeric (if num=True), string + specified character length. """ chars = string.ascii_lowercase if num: @@ -78,8 +81,8 @@ def randomstr(length, num=True):

def randomStr(length, num=True): """ - Returns a random a mixed lowercase, uppercase, alfanumerical (if num True) - string long length + Returns a random a mixed lowercase, uppercase, alpha-numeric (if num=True) + string of specified character length. """ chars = string.ascii_lowercase + string.ascii_uppercase if num: diff --git a/ooni/utils/net.py b/ooni/utils/net.py index c5b01a3..7c1c2a1 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -11,6 +11,7 @@

import sys

+from ipaddr import IPAddress from zope.interface import implements from twisted.internet import protocol, defer from twisted.internet import threads, reactor @@ -18,6 +19,7 @@ from twisted.web.iweb import IBodyProducer from scapy.all import utils

from ooni.utils import log, txscapy +from ooni.utils import PermissionsError

#if sys.platform.system() == 'Windows': # import _winreg as winreg @@ -51,27 +53,6 @@ class UnsupportedPlatform(Exception): class IfaceError(Exception): """Could not find default network interface."""

-class PermissionsError(SystemExit): - """This test requires admin or root privileges to run. Exiting...""" - - -PLATFORMS = {'LINUX': sys.platform.startswith("linux"), - 'OPENBSD': sys.platform.startswith("openbsd"), - 'FREEBSD': sys.platform.startswith("freebsd"), - 'NETBSD': sys.platform.startswith("netbsd"), - 'DARWIN': sys.platform.startswith("darwin"), - 'SOLARIS': sys.platform.startswith("sunos"), - 'WINDOWS': sys.platform.startswith("win32")} - -class UnsupportedPlatform(Exception): - """Support for this platform is not currently available.""" - -class IfaceError(Exception): - """Could not find default network interface.""" - -class PermissionsError(SystemExit): - """This test requires admin or root privileges to run. Exiting...""" - class StringProducer(object): implements(IBodyProducer)

@@ -220,6 +201,26 @@ def getNonLoopbackIfaces(platform_name=None): return interfaces

def getNetworksFromRoutes(): + """ + + Get the networks this client is current on from the kernel routing table. + Each network is returned as a :class:`ipaddr.IPNetwork`, with the + network range as the name of the network, i.e.: + + network.compressed = '127.0.0.1/32' + network.netmask = IPv4Address('255.0.0.0') + network.ipaddr = IPv4Address('127.0.0.1') + network.gateway = IPv4Address('0.0.0.0') + network.iface = 'lo' + + This is mostly useful for retrieving the default network interface in a + portable manner, though it could be used to conduct local network checks + for things like rogue DHCP servers, or perhaps test that the clients NAT + router is not the mistakenly the source of a perceived censorship event. + + @return: A list of :class:`ipaddr.IPNetwork` objects with routing table + information. + """ from scapy.all import conf, ltoa, read_routes from ipaddr import IPNetwork, IPAddress

@@ -237,6 +238,12 @@ def getNetworksFromRoutes(): return networks

def getDefaultIface(): + """ + Get the client's default network interface. + + @return: A string containing the name of the default working interface. + @raise IfaceError: If no working interface is found. + """ networks = getNetworksFromRoutes() for net in networks: if net.is_private: @@ -244,5 +251,27 @@ def getDefaultIface(): raise IfaceError

def getLocalAddress(): + """ + Get the rfc1918 IP address of the default working network interface. + + @return: The properly-formatted, validated, local IPv4/6 address of the + client's default working network interface. + """ default_iface = getDefaultIface() return default_iface.ipaddr + +def checkIPandPort(raw_ip, raw_port): + """ + Check that IP and Port are a legitimate address and portnumber. + + @return: The validated ip and port, else None. + """ + try: + port = int(raw_port) + assert port in xrange(1, 65535), "Port out of range." + ip = IPAddress(raw_ip) ## either IPv4 or IPv6 + except Exception, e: + log.err(e) + return + else: + return ip.compressed, port