commit 5c29daa8c9d33f272f618d37d1ba76b5f7998640 Merge: 85f0da8 e10033c Author: Isis Lovecruft isis@torproject.org Date: Thu Nov 8 06:40:55 2012 +0000
Merge branch 'master' of git-rw.torproject.org:ooni-probe
Conflicts: ooni/nettest.py ooni/oonicli.py ooni/runner.py
.gitignore | 2 + before_i_commit.sh | 37 ++ data/Makefile | 10 + docs/source/api/modules.rst | 7 + docs/source/api/ooni.kit.rst | 19 + docs/source/api/ooni.lib.rst | 36 ++ docs/source/api/ooni.rst | 15 +- docs/source/api/ooni.templates.rst | 15 +- docs/source/api/ooni.utils.rst | 114 ++++- docs/source/index.rst | 41 +- docs/source/writing_tests.rst | 3 +- inputs/Makefile | 3 + inputs/captive_portal_tests.txt.good | 4 - inputs/cctld.txt | 511 ------------------ inputs/dns_servers.txt.bak | 6 - inputs/dns_servers.txt.bak2 | 1 - inputs/example_exp_list.txt | 3 - inputs/major_isp_dns_servers.txt | 796 ----------------------------- inputs/short_hostname_list.txt | 7 - inputs/tld-list-cc.txt | 511 ------------------ inputs/tld-list-mozilla.txt | 5 - inputs/top-1m.txt.bak2 | 11 - nettests/bridge_reachability/bridget.py | 462 +++++++++++++++++ nettests/bridge_reachability/echo.py | 195 +++++++ nettests/bridget.py | 499 ------------------ nettests/core/captiveportal.py | 5 +- nettests/core/dnstamper.py | 97 +++-- nettests/core/echo.py | 1 - nettests/core/keyword_filtering.py | 39 ++ nettests/core/url_list.py | 23 + nettests/echo.py | 196 ------- nettests/examples/example_httpt.py | 9 +- nettests/examples/example_scapyt.py | 1 + ooni/__init__.py | 15 +- ooni/config.py | 38 ++- ooni/custodiet.py | 421 --------------- ooni/inputunit.py | 133 +----- ooni/lib/Makefile | 36 -- ooni/lib/txscapy.py | 55 ++- ooni/nettest.py | 391 +++------------ ooni/nodes.py | 176 +++++++ ooni/oonicli.py | 40 +- ooni/plugoo/__init__.py | 47 -- ooni/plugoo/assets.py | 62 --- ooni/plugoo/interface.py | 56 -- ooni/plugoo/nodes.py | 176 ------- ooni/plugoo/reports.py | 145 ------ ooni/plugoo/tests.py | 142 ----- ooni/plugoo/work.py | 148 ------ ooni/protocols/http.py | 141 ----- ooni/protocols/scapyproto.py | 55 -- ooni/reporter.py | 95 ++-- ooni/runner.py | 261 ++++------- ooni/scaffolding.py | 78 --- ooni/templates/httpt.py | 51 ++- ooni/templates/scapyt.py | 108 ++++- ooni/utils/geodata.py | 31 ++ ooni/utils/legacy.py | 459 ----------------- ooni/utils/log.py | 20 +- ooni/utils/onion.py | 4 +- oonib/oonibackend.py | 12 +- oonib/report/api.py | 63 +-- oonib/report/db/models.py | 83 --- ooniprobe.conf | 25 + requirements.txt | 27 +- test_inputs/README | 48 ++ test_inputs/dns_tamper_file.txt | 3 + test_inputs/dns_tamper_test_resolvers.txt | 2 + test_inputs/http_host_file.txt | 2 + test_inputs/keyword_filtering_file.txt | 2 + test_inputs/url_lists_file.txt | 2 + tests/test_dns.py | 24 + 72 files changed, 1868 insertions(+), 5493 deletions(-)
diff --cc nettests/bridge_reachability/echo.py index 0000000,0c20a3f..e59eee5 mode 000000,100644..100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@@ -1,0 -1,195 +1,195 @@@ + #!/usr/bin/env python + # -*- coding: utf-8 -*- + # + # +---------+ + # | echo.py | + # +---------+ + # A simple ICMP-8 ping test. + # + # :author: Isis Lovecruft + # :version: 0.0.1-pre-alpha + # :license: (c) 2012 Isis Lovecruft + # see attached LICENCE file + # + + import os + import sys + + from pprint import pprint + + from twisted.internet import reactor + from twisted.plugin import IPlugin + from twisted.python import usage + from ooni.nettest import NetTestCase + from ooni.utils import log, Storage + from ooni.utils.net import PermissionsError, IfaceError + + try: + from scapy.all import sr1, IP, ICMP ## XXX v4/v6? + from ooni.lib import txscapy + from ooni.lib.txscapy import txsr, txsend + from ooni.templates.scapyt import ScapyTest + except: + log.msg("This test requires scapy, see www.secdev.org/projects/scapy") + + ## xxx TODO: move these to a utility function for determining OSes + LINUX=sys.platform.startswith("linux") + OPENBSD=sys.platform.startswith("openbsd") + FREEBSD=sys.platform.startswith("freebsd") + NETBSD=sys.platform.startswith("netbsd") + DARWIN=sys.platform.startswith("darwin") + SOLARIS=sys.platform.startswith("sunos") + WINDOWS=sys.platform.startswith("win32") + + class EchoTest(ScapyTest): + """ + xxx fill me in + """ + name = 'echo' + author = 'Isis Lovecruft isis@torproject.org' + description = 'A simple ICMP-8 test to see if a host is reachable.' + version = '0.0.1' + inputFile = ['file', 'f', None, 'File of list of IPs to ping'] + requiresRoot = True + + optParameters = [ + ['interface', 'i', None, 'Network interface to use'], + ['count', 'c', 5, 'Number of packets to send', int], + ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], + ['ttl', 'l', 25, 'Set the IP Time to Live', int], + ['timeout', 't', 2, 'Seconds until timeout if no response', int], + ['pcap', 'p', None, 'Save pcap to this file'], + ['receive', 'r', True, 'Receive response packets'] + ] + + def setUp(self, *a, **kw): + ''' + :ivar ifaces: + Struct returned from getifaddrs(3) and turned into a tuple in the + form (*ifa_name, AF_FAMILY, *ifa_addr) + ''' + + if self.localOptions: + log.debug("%s: local_options found" % self.name) + for key, value in self.localOptions.items(): + log.debug("setting self.%s = %s" % (key, value)) + setattr(self, key, value) + + ## xxx is this now .subOptions? + #self.inputFile = self.localOptions['file'] + self.timeout *= 1000 ## convert to milliseconds + + if not self.interface: + log.msg("No network interface specified!") + log.debug("OS detected: %s" % sys.platform) + if LINUX or OPENBSD or NETBSD or FREEBSD or DARWIN or SOLARIS: + from twisted.internet.test import _posixifaces + log.msg("Attempting to discover network interfaces...") + ifaces = _posixifaces._interfaces() + elif WINDOWS: + from twisted.internet.test import _win32ifaces + log.msg("Attempting to discover network interfaces...") + ifaces = _win32ifaces._interfaces() + else: + log.debug("Client OS %s not accounted for!" % sys.platform) + log.debug("Unable to discover network interfaces...") + ifaces = [('lo', '')] + + ## found = {'eth0': '1.1.1.1'} + found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] + log.info("Found interfaces:\n%s" % pprint(found)) + self.interfaces = self.tryInterfaces(found) + else: + ## xxx need a way to check that iface exists, is up, and + ## we have permissions on it + log.debug("Our interface has been set to %s" % self.interface) + + if self.pcap: + try: + self.pcapfile = open(self.pcap, 'a+') + except: + log.msg("Unable to write to pcap file %s" % self.pcap) + self.pcapfile = None + + try: + assert os.path.isfile(self.file) + fp = open(self.file, 'r') + except Exception, e: + hosts = ['8.8.8.8', '38.229.72.14'] + log.err(e) + else: + self.inputs = self.inputProcessor(fp) + self.removePorts(hosts) + + log.debug("Initialization of %s test completed with:\n%s" + % (self.name, ''.join(self.__dict__))) + + @staticmethod - def inputParser(inputs): ++ def inputParser(self, one_input): + log.debug("Removing possible ports from host addresses...") + log.debug("Initial inputs:\n%s" % pprint(inputs)) + - assert isinstance(inputs, list) - hosts = [h.rsplit(':', 1)[0] for h in inputs] ++ #host = [h.rsplit(':', 1)[0] for h in inputs] ++ host = h.rsplit(':', 1)[0] + log.debug("Inputs converted to:\n%s" % hosts) + - return hosts ++ return host + + def tryInterfaces(self, ifaces): + try: + from scapy.all import sr1 ## we want this check to be blocking + except: + log.msg("This test requires scapy: www.secdev.org/projects/scapy") + raise SystemExit + + ifup = {} + while ifaces: + for ifname, ifaddr in ifaces: + log.debug("Currently testing network capabilities of interface" + + "%s by sending a packet to our address %s" + % (ifname, ifaddr)) + try: + pkt = IP(dst=ifaddr)/ICMP() + ans, unans = sr(pkt, iface=ifname, timeout=self.timeout) + except Exception, e: + raise PermissionsError if e.find("Errno 1") else log.err(e) + else: + ## xxx i think this logic might be wrong + log.debug("Interface test packet\n%s\n\n%s" + % (pkt.summary(), pkt.show2())) + if ans.summary(): + log.info("Received answer for test packet on interface" + +"%s :\n%s" % (ifname, ans.summary())) + ifup.update(ifname, ifaddr) + else: + log.info("Our interface test packet was unanswered:\n%s" + % unans.summary()) + + if len(ifup) > 0: + log.msg("Discovered the following working network interfaces: %s" + % ifup) + return ifup + else: + raise IfaceError("Could not find a working network interface.") + + def buildPackets(self): + log.debug("self.input is %s" % self.input) + log.debug("self.hosts is %s" % self.hosts) + for addr in self.input: + packet = IP(dst=self.input)/ICMP() + self.request.append(packet) + return packet + + def test_icmp(self): + if self.recieve: + self.buildPackets() + all = [] + for packet in self.request: + d = self.sendReceivePackets(packets=packet) + all.append(d) + self.response.update({packet: d}) + d_list = defer.DeferredList(all) + return d_list + else: + d = self.sendPackets() + return d diff --cc ooni/nettest.py index 84c3289,3534a32..27e733c --- a/ooni/nettest.py +++ b/ooni/nettest.py @@@ -1,17 -1,8 +1,16 @@@ # -*- encoding: utf-8 -*- # -# :authors: Arturo "hellais" Filastò art@fuffa.org +# nettest.py +# -------------------> +# +# :authors: Arturo "hellais" Filastò art@fuffa.org, +# Isis Lovecruft isis@torproject.org # :licence: see LICENSE +# :copyright: 2012 Arturo Filasto, Isis Lovecruft +# :version: 0.1.0-alpha +# +# <-------------------
- from functools import partial import sys import os import itertools @@@ -392,10 -118,10 +127,10 @@@ class NetTestCase(unittest.TestCase) """ name = "I Did Not Change The Name" author = "Jane Doe foo@example.com" - version = "0" + version = "0.0.0"
+ inputs = [None] inputFile = None - inputs = [None]
report = {} report['errors'] = [] diff --cc ooni/oonicli.py index a61b3f9,5ee6deb..4f5eac5 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@@ -41,21 -41,19 +41,15 @@@ class Options(usage.Options, app.Reacto
optFlags = [["help", "h"], ['debug-stacktraces', 'B', -- 'Report deferred creation and callback stack traces'], -- ] ++ 'Report deferred creation and callback stack traces'],]
-- optParameters = [ -- ["reportfile", "o", None, "report file name"], - ["logfile", "l", "test.log", "log file name"], - ['temp-directory', None, '_ooni_temp', - 'Path to use as working directory for tests.'] - ] - ["logfile", "l", None, "log file name"], - ] ++ optParameters = [["reportfile", "o", None, "report file name"], ++ ["logfile", "l", None, "log file name"],]
compData = usage.Completions( extraActions=[usage.CompleteFiles( "*.py", descr="file | module | package | TestCase | testMethod", -- repeat=True)], -- ) ++ repeat=True)],)
tracer = None
@@@ -86,29 -85,22 +80,27 @@@
def run(): - if len(sys.argv) == 1: - sys.argv.append("--help") + """ + Call me to begin testing a file or module. + """ - - config = Options() - + cmd_line_options = Options() + if len(sys.argv) == 1: - config.getUsage() - ++ cmd_line_options.getUsage() try: - config.parseOptions() + cmd_line_options.parseOptions() - except usage.error, ue: + except usage.UsageError, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue)
+ log.start() - log.debug("oonicli.run: config set to %s" % config) + - if config['debug-stacktraces']: + if cmd_line_options['debug-stacktraces']: defer.setDebugging(True)
- classes = runner.findTestClassesFromConfig(config) - casesList, options = runner.loadTestsAndOptions(classes, config) + classes = runner.findTestClassesFromConfig(cmd_line_options) + casesList, options = runner.loadTestsAndOptions(classes, cmd_line_options)
for idx, cases in enumerate(casesList): - orunner = runner.ORunner(cases, options[idx], config) + orunner = runner.ORunner(cases, options[idx], cmd_line_options) + log.start(cmd_line_options['logfile']) orunner.run() + diff --cc ooni/runner.py index 52c6596,797a9d4..c8c6812 --- a/ooni/runner.py +++ b/ooni/runner.py @@@ -239,27 -149,31 +149,34 @@@ class ORunner(object) self.cases = cases self.options = options
+ log.debug("ORunner: cases=%s" % type(cases)) + log.debug("ORunner: options=%s" % options) + try: - first = options.pop(0) - except: - first = options - - if 'inputs' in first: - self.inputs = self.options['inputs'] + assert len(options) != 0, "Length of options is zero!" + except AssertionError, ae: + self.inputs = [] + log.err(ae) else: - log.msg("Could not find inputs!") - self.inputs = [None] + try: + first = options.pop(0) + except: + first = options + + if 'inputs' in first: + self.inputs = options['inputs'] + else: + log.msg("Could not find inputs!") + log.msg("options[0] = %s" % first) + self.inputs = [None]
try: - reportFile = open(config['reportfile'], 'a+') - except: + reportFile = open(cmd_line_options['reportfile'], 'a+') + except TypeError: filename = 'report_'+date.timestamp()+'.yaml' reportFile = open(filename, 'a+') - self.reporterFactory = ReporterFactory( - reportFile, testSuite=self.baseSuite(self.cases)) + self.reporterFactory = ReporterFactory(reportFile, + testSuite=self.baseSuite(self.cases))
def runWithInputUnit(self, input_unit): idx = 0