November 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Refactoring on runner, reporter and oonicli
by art＠torproject.org 25 Nov '12

25 Nov '12

commit eb46b078184500efd2024d403170047e64b272de Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 01:48:51 2012 +0100 Refactoring on runner, reporter and oonicli * Include the command line arguments in the test report --- ooni/config.py | 32 ++++++++++++++++++-------------- ooni/oonicli.py | 35 +++++++++++++++-------------------- ooni/reporter.py | 24 +++++++++++++++++++----- ooni/runner.py | 40 +++++++++++++++++----------------------- ooniprobe.conf | 2 +- 5 files changed, 70 insertions(+), 63 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index ef44ee4..5168c34 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -11,6 +11,9 @@ from twisted.internet import reactor, threads from ooni.utils import otime from ooni.utils import Storage +reports = Storage() +basic = None +cmd_line_options = None def get_root_path(): this_directory = os.path.dirname(__file__) @@ -18,19 +21,6 @@ def get_root_path(): root = os.path.abspath(root) return root -def oreport_filenames(file_name): - """ - returns the filenames for the pcap file and the yamloo report - - returns - yamloo_filename, pcap_filename - """ - test_name = '.'.join(file_name.split(".")[:-1]) - base_filename = "%s_%s_"+otime.timestamp()+".%s" - yamloo_filename = base_filename % (test_name, "report", "yamloo") - pcap_filename = base_filename % (test_name, "packets", "pcap") - return yamloo_filename, pcap_filename - def loadConfigFile(): """ This is a helper function that makes sure that the configuration attributes @@ -62,7 +52,21 @@ def loadConfigFile(): advanced[k] = v return basic, privacy, advanced -basic = None +class TestFilenameNotSet(Exception): + pass + +def generateReportFilenames(): + try: + test_file_name = os.path.basename(cmd_line_options['test']) + except IndexError: + raise TestFilenameNotSet + + test_name = '.'.join(test_file_name.split(".")[:-1]) + base_filename = "%s_%s_"+otime.timestamp()+".%s" + print "Setting yamloo to %s" % base_filename + reports.yamloo = base_filename % (test_name, "report", "yamloo") + reports.pcap = base_filename % (test_name, "packets", "pcap") + if not basic: # Here we make sure that we instance the config file attributes only once basic, privacy, advanced = loadConfigFile() diff --git a/ooni/oonicli.py b/ooni/oonicli.py index c8385ed..41a31f6 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -70,7 +70,7 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): try: self['test'] = args[0] - self['subArgs'] = args[1:] + self['subargs'] = args[1:] except: raise usage.UsageError("No test filename specified!") @@ -95,24 +95,19 @@ def run(): if cmd_line_options['debug-stacktraces']: defer.setDebugging(True) - test_file_name = os.path.basename(cmd_line_options['test']) - - yamloo_filename, pcap_filename = config.oreport_filenames(test_file_name) + config.cmd_line_options = cmd_line_options + config.generateReportFilenames() if cmd_line_options['reportfile']: - yamloo_filename = cmd_line_options['reportfile'] - pcap_filename = yamloo_filename+".pcap" - - if os.path.exists(yamloo_filename): - print "Report already exists with filename %s" % yamloo_filename - print "Renaming it to %s" % yamloo_filename+'.old' - os.rename(yamloo_filename, yamloo_filename+'.old') - if os.path.exists(pcap_filename): - print "Report PCAP already exists with filename %s" % pcap_filename - print "Renaming it to %s" % pcap_filename+'.old' - os.rename(pcap_filename, pcap_filename+'.old') - - classes = runner.findTestClassesFromConfig(cmd_line_options) + config.reports.yamloo = cmd_line_options['reportfile'] + config.reports.pcap = config.reports.yamloo+".pcap" + + if os.path.exists(config.reports.pcap): + print "Report PCAP already exists with filename %s" % config.reports.pcap + print "Renaming it to %s" % config.reports.pcap+'.old' + os.rename(config.reports.pcap, config.reports.pcap+'.old') + + classes = runner.findTestClassesFromFile(cmd_line_options['test']) test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options) if config.privacy.includepcap: try: @@ -121,13 +116,13 @@ def run(): print "[!] Includepcap options requires root priviledges to run" print " you should run ooniprobe as root or disable the options in ooniprobe.conf" sys.exit(1) + print "Starting sniffer" - net.capturePackets(pcap_filename) + net.capturePackets(config.reports.pcap) log.start(cmd_line_options['logfile']) - tests_d = runner.runTestCases(test_cases, options, - cmd_line_options, yamloo_filename) + tests_d = runner.runTestCases(test_cases, options, cmd_line_options) tests_d.addBoth(testsEnded) reactor.run() diff --git a/ooni/reporter.py b/ooni/reporter.py index 1e2924d..99b7f7a 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -10,6 +10,7 @@ import itertools import logging import sys +import os import time import yaml import json @@ -155,6 +156,9 @@ def getTestDetails(options): defer.returnValue(test_details) class OReporter(object): + def __init__(self, cmd_line_options): + self.cmd_line_options = dict(cmd_line_options) + def createReport(self, options): """ Override this with your own logic to implement tests. @@ -172,7 +176,6 @@ class OReporter(object): def testDone(self, test, test_name): log.msg("Finished running %s" % test_name) - log.debug("Writing report") test_report = dict(test.report) if isinstance(test.input, Packet): @@ -204,8 +207,15 @@ class YAMLReporter(OReporter): """ These are useful functions for reporting to YAML format. """ - def __init__(self, stream): - self._stream = stream + def __init__(self, cmd_line_options): + if os.path.exists(config.reports.yamloo): + log.msg("Report already exists with filename %s" % config.reports.yamloo) + log.msg("Renaming it to %s" % config.reports.yamloo+'.old') + os.rename(config.reports.yamloo, config.reports.yamloo+'.old') + + log.debug("Creating %s" % config.reports.yamloo) + self._stream = open(config.reports.yamloo, 'w+') + OReporter.__init__(self, cmd_line_options) def _writeln(self, line): self._write("%s\n" % line) @@ -233,6 +243,7 @@ class YAMLReporter(OReporter): self._writeln("###########################################") test_details = yield getTestDetails(options) + test_details['options'] = self.cmd_line_options self.writeReportEntry(test_details) @@ -250,7 +261,9 @@ class OONIBTestDetailsLookupFailed(Exception): pass class OONIBReporter(OReporter): - def __init__(self, backend_url): + def __init__(self, cmd_line_options): + self.backend_url = cmd_line_options['collector'] + from ooni.utils.txagentwithsocks import Agent from twisted.internet import reactor try: @@ -259,7 +272,7 @@ class OONIBReporter(OReporter): except Exception, e: log.exception(e) - self.backend_url = backend_url + OReporter.__init__(self, cmd_line_options) @defer.inlineCallbacks def writeReportEntry(self, entry): @@ -306,6 +319,7 @@ class OONIBReporter(OReporter): software_version = '0.0.1' test_details = yield getTestDetails(options) + test_details['options'] = self.cmd_line_options content = '---\n' content += safe_dump(test_details) diff --git a/ooni/runner.py b/ooni/runner.py index 92aa406..5e8536c 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -23,11 +23,11 @@ from twisted.internet import reactor, threads from ooni.inputunit import InputUnitFactory from ooni.nettest import NetTestCase, NoPostProcessor -from ooni import reporter +from ooni import reporter, config from ooni.utils import log, checkForRoot, NotRootError -def processTest(obj, cmd_line_options): +def processTest(obj): """ Process the parameters and :class:`twisted.python.usage.Options` of a :class:`ooni.nettest.Nettest`. @@ -57,7 +57,7 @@ def processTest(obj, cmd_line_options): options = obj.usageOptions() - options.parseOptions(cmd_line_options['subArgs']) + options.parseOptions(config.cmd_line_options['subargs']) obj.localOptions = options if obj.inputFile: @@ -73,7 +73,7 @@ def processTest(obj, cmd_line_options): log.err("There was an error in running %s!" % test_name) log.err("%s" % e) options.opt_help() - raise usage.UsageError("Error in parsing command line args for %s" % test_name) + raise usage.UsageError("Error in parsing command line args for %s" % test_name) if obj.requiresRoot: try: @@ -90,28 +90,23 @@ def isTestCase(obj): except TypeError: return False -def findTestClassesFromConfig(cmd_line_options): +def findTestClassesFromFile(filename): """ Takes as input the command line config parameters and returns the test case classes. - If it detects that a certain test class is using the old OONIProbe format, - then it will adapt it to the new testing system. - :param cmd_line_options: - A configured and instantiated :class:`twisted.python.usage.Options` - class. + :param filename: + the absolute path to the file containing the ooniprobe test classes + :return: A list of class objects found in a file or module given on the commandline. """ - - filename = cmd_line_options['test'] classes = [] - module = filenameToModule(filename) for name, val in inspect.getmembers(module): if isTestCase(val): - classes.append(processTest(val, cmd_line_options)) + classes.append(processTest(val)) return classes def makeTestCases(klass, tests, method_prefix): @@ -227,8 +222,10 @@ def runTestCasesWithInputUnit(test_cases, input_unit, oreporter): return defer.DeferredList(dl) @defer.inlineCallbacks -def runTestCases(test_cases, options, - cmd_line_options, yamloo_filename): +def runTestCases(test_cases, options, cmd_line_options): + log.debug("Running %s" % test_cases) + log.debug("Options %s" % options) + log.debug("cmd_line_options %s" % dict(cmd_line_options)) try: assert len(options) != 0, "Length of options is zero!" except AssertionError, ae: @@ -247,15 +244,12 @@ def runTestCases(test_cases, options, log.msg("options[0] = %s" % first) test_inputs = [None] - log.debug("Creating %s" % yamloo_filename) - if cmd_line_options['collector']: - log.debug("Using remote collector %s" % cmd_line_options['collector']) - oreporter = reporter.OONIBReporter(cmd_line_options['collector']) + log.debug("Using remote collector") + oreporter = reporter.OONIBReporter(cmd_line_options) else: - reportFile = open(yamloo_filename, 'w+') - log.debug("Reporting to file %s" % reportFile) - oreporter = reporter.YAMLReporter(reportFile) + log.debug("Reporting to file %s" % config.reports.yamloo) + oreporter = reporter.YAMLReporter(cmd_line_options) try: input_unit_factory = InputUnitFactory(test_inputs) diff --git a/ooniprobe.conf b/ooniprobe.conf index ad761e1..e9f208f 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -22,7 +22,7 @@ advanced: # /path/to/ooni-probe/data/ #geoip_data_dir: /usr/share/GeoIP/ geoip_data_dir: /home/x/code/networking/ooni-probe/data/ - debug: false + debug: true threadpool_size: 10 tor_socksport: 9050 # For auto detection

1 0

[ooni-probe/master] Implement test deck system for automating the specification of command line arguments for tests
by art＠torproject.org 25 Nov '12

25 Nov '12

commit 263568f28bcac776da7f9936a0e80958c4027f46 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 25 02:43:10 2012 +0100 Implement test deck system for automating the specification of command line arguments for tests * Change before_i_commit script to use test deck system --- before_i_commit.sh | 13 ++------- before_i_commit.testdeck | 40 ++++++++++++++++++++++++++++ ooni/oonicli.py | 65 +++++++++++++++++++++++++++++----------------- ooni/reporter.py | 2 - ooni/runner.py | 2 - 5 files changed, 84 insertions(+), 38 deletions(-) diff --git a/before_i_commit.sh b/before_i_commit.sh index 5147149..8b8180f 100755 --- a/before_i_commit.sh +++ b/before_i_commit.sh @@ -14,15 +14,7 @@ rm before_i_commit.log find . -type f -name "*.py[co]" -delete -./bin/ooniprobe -l before_i_commit.log -o http_url_lists.yamloo nettests/core/http_url_list.py -f test_inputs/url_lists_file.txt - -./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -T test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt - -./bin/ooniprobe -l before_i_commit.log -o captive_portal_test.yamloo nettests/core/captiveportal.py - -./bin/ooniprobe -l before_i_commit.log -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt - -./bin/ooniprobe -l before_i_commit.log -o http_keyword_filtering.yamloo nettests/core/http_keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt +./bin/ooniprobe -i before_i_commit.testdeck echo "Below you should not see anything" echo "---------------------------------" @@ -32,5 +24,6 @@ echo "If you do, it means something is wrong." echo "Read through the log file and fix it." echo "If you are having some problems fixing some things that have to do with" echo "the core of OONI, let's first discuss it on IRC, or open a ticket" -less *yamloo +read +cat *yamloo | less rm -f *yamloo diff --git a/before_i_commit.testdeck b/before_i_commit.testdeck new file mode 100644 index 0000000..e0b30ea --- /dev/null +++ b/before_i_commit.testdeck @@ -0,0 +1,40 @@ +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: captive_portal_test.yamloo + subargs: [] + test: nettests/core/captiveportal.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: dns_tamper_test.yamloo + subargs: [-T, test_inputs/dns_tamper_test_resolvers.txt, -f, test_inputs/dns_tamper_file.txt] + test: nettests/core/dnstamper.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: http_host.yamloo + subargs: [-b, 'http://ooni.nu/test', -f, test_inputs/http_host_file.txt] + test: nettests/core/http_host.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: http_keyword_filtering.yamloo + subargs: [-b, 'http://ooni.nu/test/', -f, test_inputs/keyword_filtering_file.txt] + test: nettests/core/http_keyword_filtering.py +- options: + collector: null + help: 0 + logfile: before_i_commit.log + pcapfile: null + reportfile: http_url_lists.yamloo + subargs: [-f, test_inputs/url_lists_file.txt] + test: nettests/core/http_url_list.py diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 41a31f6..5c582b2 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -13,6 +13,7 @@ import sys import os import random import time +import yaml from twisted.internet import defer, reactor from twisted.application import app @@ -27,7 +28,7 @@ from ooni.utils import net from ooni.utils import checkForRoot, NotRootError from ooni.utils import log -class Options(usage.Options, app.ReactorSelectionMixin): +class Options(usage.Options): synopsis = """%s [options] [path to test].py """ % (os.path.basename(sys.argv[0]),) @@ -35,12 +36,12 @@ class Options(usage.Options, app.ReactorSelectionMixin): " network tests. These are loaded from modules, packages and" " files listed on the command line") - optFlags = [["help", "h"], - ['debug-stacktraces', 'B', - 'Report deferred creation and callback stack traces'],] + optFlags = [["help", "h"]] optParameters = [["reportfile", "o", None, "report file name"], - ["collector", "c", None, + ["testdeck", "i", None, + "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], + ["collector", "c", None, "Address of the collector of test results. (example: http://127.0.0.1:8888)"], ["logfile", "l", None, "log file name"], ["pcapfile", "p", None, "pcap file name"]] @@ -68,6 +69,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): sys.settrace(spewer) def parseArgs(self, *args): + if self['testdeck']: + return try: self['test'] = args[0] self['subargs'] = args[1:] @@ -79,22 +82,9 @@ def testsEnded(*arg, **kw): You can place here all the post shutdown tasks. """ log.debug("testsEnded: Finished running all tests") + reactor.stop() -def run(): - """ - Call me to begin testing from a file. - """ - cmd_line_options = Options() - if len(sys.argv) == 1: - cmd_line_options.getUsage() - try: - cmd_line_options.parseOptions() - except usage.UsageError, ue: - raise SystemExit, "%s: %s" % (sys.argv[0], ue) - - if cmd_line_options['debug-stacktraces']: - defer.setDebugging(True) - +def runTest(cmd_line_options): config.cmd_line_options = cmd_line_options config.generateReportFilenames() @@ -120,10 +110,37 @@ def run(): print "Starting sniffer" net.capturePackets(config.reports.pcap) - log.start(cmd_line_options['logfile']) + return runner.runTestCases(test_cases, options, cmd_line_options) - tests_d = runner.runTestCases(test_cases, options, cmd_line_options) - tests_d.addBoth(testsEnded) +def run(): + """ + Call me to begin testing from a file. + """ + cmd_line_options = Options() + if len(sys.argv) == 1: + cmd_line_options.getUsage() + try: + cmd_line_options.parseOptions() + except usage.UsageError, ue: + raise SystemExit, "%s: %s" % (sys.argv[0], ue) - reactor.run() + deck_dl = [] + log.start(cmd_line_options['logfile']) + if cmd_line_options['testdeck']: + test_deck = yaml.safe_load(open(cmd_line_options['testdeck'])) + for test in test_deck: + del cmd_line_options + cmd_line_options = test['options'] + d1 = runTest(cmd_line_options) + deck_dl.append(d1) + else: + log.msg("No test deck detected") + del cmd_line_options['testdeck'] + d1 = runTest(cmd_line_options) + deck_dl.append(d1) + + d2 = defer.DeferredList(deck_dl) + d2.addCallback(testsEnded) + + reactor.run() diff --git a/ooni/reporter.py b/ooni/reporter.py index 99b7f7a..26c6e43 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -197,10 +197,8 @@ class OReporter(object): log.debug("allDone: Finished running all tests") try: log.debug("Stopping the reactor") - reactor.stop() except: log.debug("Unable to stop the reactor") - pass return None class YAMLReporter(OReporter): diff --git a/ooni/runner.py b/ooni/runner.py index 5e8536c..bc271a6 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -262,7 +262,6 @@ def runTestCases(test_cases, options, cmd_line_options): yield oreporter.createReport(options) except reporter.OONIBReportCreationFailed: log.err("Error in creating new report") - reactor.stop() raise except Exception, e: log.exception(e) @@ -280,6 +279,5 @@ def runTestCases(test_cases, options, cmd_line_options): except Exception: log.exception("Problem in running test") - reactor.stop() oreporter.allDone()

1 0

[ooni-probe/master] Update before_i_commit script with command line option of DNS tamper
by art＠torproject.org 24 Nov '12

24 Nov '12

commit ad22e3182df140e05d86f15b43b14dd4f26e1873 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 23:08:20 2012 +0100 Update before_i_commit script with command line option of DNS tamper --- before_i_commit.sh | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/before_i_commit.sh b/before_i_commit.sh index 3e01366..5147149 100755 --- a/before_i_commit.sh +++ b/before_i_commit.sh @@ -16,7 +16,7 @@ find . -type f -name "*.py[co]" -delete ./bin/ooniprobe -l before_i_commit.log -o http_url_lists.yamloo nettests/core/http_url_list.py -f test_inputs/url_lists_file.txt -./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -t test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt +./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -T test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt ./bin/ooniprobe -l before_i_commit.log -o captive_portal_test.yamloo nettests/core/captiveportal.py

1 0

[ooni-probe/master] If libpcap and libdnet is present we will use them. If not use raw sockets.
by art＠torproject.org 24 Nov '12

24 Nov '12

commit 3d100e9495bf2459e13bba73a2eaa8f1cd58d521 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 23:04:40 2012 +0100 If libpcap and libdnet is present we will use them. If not use raw sockets. --- ooni/utils/net.py | 1 - ooni/utils/txscapy.py | 32 ++++++++++++++++++++++++-------- ooniprobe.conf | 2 +- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index e828977..649dc64 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -15,7 +15,6 @@ from zope.interface import implements from twisted.internet import protocol, defer from twisted.internet import threads, reactor from twisted.web.iweb import IBodyProducer -from scapy.all import utils from ooni.utils import log, txscapy diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 432a987..679eb42 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -17,13 +17,30 @@ from twisted.internet import reactor, threads, error from twisted.internet import defer, abstract from zope.interface import implements -from scapy.all import PcapWriter, MTU -from scapy.all import BasePacketList, conf, PcapReader - -from scapy.all import conf, Gen, SetGen -from scapy.arch import pcapdnet +from scapy.config import conf from ooni.utils import log +from ooni import config + +try: + conf.use_pcap = True + conf.use_dnet = True + + from scapy.all import PcapWriter + from scapy.arch import pcapdnet + + config.pcap_dnet = True + +except ImportError, e: + log.err("pypcap or dnet not installed. Certain tests may not work.") + config.pcap_dnet = False + conf.use_pcap = False + conf.use_dnet = False + + from scapy.all import PcapWriter + +from scapy.all import BasePacketList, conf, PcapReader +from scapy.all import conf, Gen, SetGen, MTU def getNetworksFromRoutes(): from scapy.all import conf, ltoa, read_routes @@ -58,9 +75,8 @@ class ScapyProtocol(abstract.FileDescriptor): def __init__(self, interface, super_socket=None, timeout=5): abstract.FileDescriptor.__init__(self, reactor) if not super_socket: - # XXX this is what we would want - #super_socket = pcapdnet.L3dnetSocket(iface=interface) - super_socket = conf.L3Socket(iface=interface) + super_socket = conf.L3socket(iface=interface, promisc=True, filter='') + #super_socket = conf.L2socket(iface=interface) fdesc._setCloseOnExec(super_socket.ins.fileno()) self.super_socket = super_socket diff --git a/ooniprobe.conf b/ooniprobe.conf index e9f208f..ad761e1 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -22,7 +22,7 @@ advanced: # /path/to/ooni-probe/data/ #geoip_data_dir: /usr/share/GeoIP/ geoip_data_dir: /home/x/code/networking/ooni-probe/data/ - debug: true + debug: false threadpool_size: 10 tor_socksport: 9050 # For auto detection

1 0

[ooni-probe/master] Comment out what we would want to have
by art＠torproject.org 24 Nov '12

24 Nov '12

commit 17e73e2253cf27e6c2d21d11bf33bcedeb82d7a1 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 21:39:35 2012 +0100 Comment out what we would want to have * Uncomment set close-on-exec on the socket file descriptor. Would not be useful in our case since we are not forking, but it's does not hurt to be safe. * This functionality does not work on windows based machines, but twisteds wrapper for it takes care of handling that. --- ooni/utils/txscapy.py | 12 ++++-------- 1 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index e41d649..432a987 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -25,9 +25,6 @@ from scapy.arch import pcapdnet from ooni.utils import log -conf.use_pcap = True -conf.use_dnet = True - def getNetworksFromRoutes(): from scapy.all import conf, ltoa, read_routes from ipaddr import IPNetwork, IPAddress @@ -60,13 +57,12 @@ class TXPcapWriter(PcapWriter): class ScapyProtocol(abstract.FileDescriptor): def __init__(self, interface, super_socket=None, timeout=5): abstract.FileDescriptor.__init__(self, reactor) - # By default we use the conf.L3socket if not super_socket: - super_socket = pcapdnet.L3dnetSocket(iface=interface) - print super_socket - log.msg("Creating layer 3 socket with interface %s" % interface) + # XXX this is what we would want + #super_socket = pcapdnet.L3dnetSocket(iface=interface) + super_socket = conf.L3Socket(iface=interface) - #fdesc._setCloseOnExec(super_socket.ins.fileno()) + fdesc._setCloseOnExec(super_socket.ins.fileno()) self.super_socket = super_socket self.interface = interface

1 0

[ooni-probe/master] Fix bug in tests that do not have usageOptions
by art＠torproject.org 24 Nov '12

24 Nov '12

commit 18a6cde20774311de552fcae966d847b5eb9f187 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 20:43:23 2012 +0100 Fix bug in tests that do not have usageOptions --- nettests/core/traceroute.py | 1 + ooni/runner.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nettests/core/traceroute.py b/nettests/core/traceroute.py index 6a443d9..45bbb67 100644 --- a/nettests/core/traceroute.py +++ b/nettests/core/traceroute.py @@ -19,6 +19,7 @@ class UsageOptions(usage.Options): ['maxttl', 'm', 30, 'The maximum value of ttl to set on packets'], ['srcport', 'p', None, 'Set the source port to a specific value (only applies to TCP and UDP)'] ] + optFlags = [['randomize','r', 'Randomize the source port']] class TracerouteTest(scapyt.BaseScapyTest): diff --git a/ooni/runner.py b/ooni/runner.py index f5ab651..92aa406 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -39,12 +39,13 @@ def processTest(obj, cmd_line_options): A configured and instantiated :class:`twisted.python.usage.Options` class. """ + if not hasattr(obj.usageOptions, 'optParameters'): + obj.usageOptions.optParameters = [] + if obj.inputFile: obj.usageOptions.optParameters.append(obj.inputFile) if obj.baseParameters: - if not hasattr(obj.usageOptions, 'optParameters'): - obj.usageOptions.optParameters = [] for parameter in obj.baseParameters: obj.usageOptions.optParameters.append(parameter)

1 0

[ooni-probe/master] Do some important refactoring of scapy related functionality
by art＠torproject.org 24 Nov '12

24 Nov '12

commit c44cc83cbefa9fcd6ba81bb18a6e52a1762f935f Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 21:32:44 2012 +0100 Do some important refactoring of scapy related functionality * Move functions for detecting the default network interface into txscapy * Make sr1 follow the syntax of scapy * Add notes as to why the parasitic traceroute test does not work --- nettests/bridge_reachability/echo.py | 4 +- nettests/core/dnsspoof.py | 4 +- nettests/core/dnstamper.py | 12 ++-- nettests/core/parasitictraceroute.py | 117 ++++++++++++++++++---------------- nettests/core/tcpconnect.py | 2 +- ooni/templates/scapyt.py | 65 +++++++++++++------ ooni/utils/net.py | 42 +------------ ooni/utils/txscapy.py | 75 +++++++++++++--------- ooniprobe.conf | 4 + 9 files changed, 166 insertions(+), 159 deletions(-) diff --git a/nettests/bridge_reachability/echo.py b/nettests/bridge_reachability/echo.py index 40436e7..d4033dd 100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@ -18,7 +18,7 @@ import sys from twisted.python import usage from twisted.internet import reactor, defer from ooni import nettest -from ooni.utils import log, net, Storage +from ooni.utils import log, net, Storage, txscapy try: from scapy.all import IP, ICMP @@ -66,7 +66,7 @@ class EchoTest(nettest.NetTestCase): if not self.interface: try: - iface = net.getDefaultIface() + iface = txscapy.getDefaultIface() except Exception, e: log.msg("No network interface specified!") log.err(e) diff --git a/nettests/core/dnsspoof.py b/nettests/core/dnsspoof.py index 8f3e6a2..48991ea 100644 --- a/nettests/core/dnsspoof.py +++ b/nettests/core/dnsspoof.py @@ -51,7 +51,7 @@ class DNSSpoof(scapyt.ScapyTest): question = IP(dst=self.resolverAddr)/UDP()/DNS(rd=1, qd=DNSQR(qtype="A", qclass="IN", qname=self.hostname)) log.msg("Performing query to %s with %s:%s" % (self.hostname, self.resolverAddr, self.resolverPort)) - answered, unanswered = yield self.sr1(question) + yield self.sr1(question) @defer.inlineCallbacks def test_control_a_lookup(self): @@ -59,6 +59,6 @@ class DNSSpoof(scapyt.ScapyTest): qd=DNSQR(qtype="A", qclass="IN", qname=self.hostname)) log.msg("Performing query to %s with %s:%s" % (self.hostname, self.controlResolverAddr, self.controlResolverPort)) - answered, unanswered = yield self.sr1(question) + yield self.sr1(question) diff --git a/nettests/core/dnstamper.py b/nettests/core/dnstamper.py index d007d77..967ae26 100644 --- a/nettests/core/dnstamper.py +++ b/nettests/core/dnstamper.py @@ -28,8 +28,10 @@ from ooni.utils import log class UsageOptions(usage.Options): optParameters = [['backend', 'b', '8.8.8.8:53', 'The OONI backend that runs the DNS resolver'], - ['testresolvers', 't', None, - 'file containing list of DNS resolvers to test against'] + ['testresolvers', 'T', None, + 'File containing list of DNS resolvers to test against'], + ['testresolver', 't', None, + 'Specify a single test resolver to use for testing'] ] class DNSTamperTest(dnst.DNSTest): @@ -44,12 +46,10 @@ class DNSTamperTest(dnst.DNSTest): 'Input file of list of hostnames to attempt to resolve'] usageOptions = UsageOptions - requiredOptions = ['backend', 'file', 'testresolvers'] + requiredOptions = ['backend', 'file'] def setUp(self): - if not self.localOptions['testresolvers']: - self.test_resolvers = ['8.8.8.8'] raise usage.UsageError("You did not specify a file of DNS servers to test!" "See the '--testresolvers' option.") @@ -68,7 +68,7 @@ class DNSTamperTest(dnst.DNSTest): self.report['control_resolver'] = self.control_dns_server @defer.inlineCallbacks - def test_a_queries(self): + def test_a_lookup(self): """ We perform an A lookup on the DNS test servers for the domains to be tested and an A lookup on the known good DNS server. diff --git a/nettests/core/parasitictraceroute.py b/nettests/core/parasitictraceroute.py index 8ea27bc..631c24b 100644 --- a/nettests/core/parasitictraceroute.py +++ b/nettests/core/parasitictraceroute.py @@ -13,14 +13,12 @@ from scapy.all import * from ooni.utils import log class UsageOptions(usage.Options): - optParameters = [['backend', 'b', '8.8.8.8', 'Test backend to use'], + optParameters = [['backend', 'b', 'google.com', 'Test backend to use'], ['timeout', 't', 5, 'The timeout for the traceroute test'], - ['maxttl', 'm', 30, 'The maximum value of ttl to set on packets'], + ['maxttl', 'm', 64, 'The maximum value of ttl to set on packets'], ['dstport', 'd', 80, 'Set the destination port of the traceroute test'], ['srcport', 'p', None, 'Set the source port to a specific value']] - optFlags = [['randomize','r', 'Randomize the source port']] - class ParasiticalTracerouteTest(scapyt.BaseScapyTest): name = "Parasitic TCP Traceroute Test" author = "Arturo Filastò" @@ -32,58 +30,57 @@ class ParasiticalTracerouteTest(scapyt.BaseScapyTest): def get_sport(): if self.localOptions['srcport']: return int(self.localOptions['srcport']) - elif self.localOptions['randomize']: - return random.randint(1024, 65535) else: - return 80 - + return random.randint(1024, 65535) self.get_sport = get_sport - self.dport = int(self.localOptions['dstport']) - def max_ttl_and_timeout(self): - max_ttl = int(self.localOptions['maxttl']) - timeout = int(self.localOptions['timeout']) - self.report['max_ttl'] = max_ttl - self.report['timeout'] = timeout - return max_ttl, timeout + self.dst_ip = socket.gethostbyaddr(self.localOptions['backend'])[2][0] + + self.dport = int(self.localOptions['dstport']) + self.max_ttl = int(self.localOptions['maxttl']) @defer.inlineCallbacks def test_parasitic_tcp_traceroute(self): """ - Establishes a TCP stream and send the packets inside of such stream. - Requires the backend to respond with an ACK to our SYN packet. - """ - max_ttl, timeout = self.max_ttl_and_timeout() + Establishes a TCP stream, then sequentially sends TCP packets with + increasing TTL until we reach the ttl of the destination. + + Requires the backend to respond with an ACK to our SYN packet (i.e. + the port must be open) + XXX this currently does not work properly. The problem lies in the fact + that we are currently using the scapy layer 3 socket. This socket makes + packets received be trapped by the kernel TCP stack, therefore when we + send out a SYN and get back a SYN-ACK the kernel stack will reply with + a RST because it did not send a SYN. + + The quick fix to this would be to establish a TCP stream using socket + calls and then "cannibalizing" the TCP session with scapy. + + The real fix is to make scapy use libpcap instead of raw sockets + obviously as we previously did... arg. + """ sport = self.get_sport() dport = self.dport ipid = int(RandShort()) - packet = IP(dst=self.localOptions['backend'], ttl=max_ttl, - id=ipid)/TCP(sport=sport, dport=dport, - flags="S", seq=0) + ip_layer = IP(dst=self.dst_ip, + id=ipid, ttl=self.max_ttl) - log.msg("Sending SYN towards %s" % dport) + syn = ip_layer/TCP(sport=sport, dport=dport, flags="S", seq=0) - try: - answered, unanswered = yield self.sr(packet, timeout=timeout) - except Exception, e: - log.exception(e) - except: - log.exception() + log.msg("Sending...") + syn.show2() - try: - snd, rcv = answered[0] - synack = rcv[0] + synack = yield self.sr1(syn) - except IndexError: - print answered, unanswered + log.msg("Got response...") + synack.show2() + + if not synack: log.err("Got no response. Try increasing max_ttl") return - except Exception, e: - log.exception(e) - if synack[TCP].flags == 11: log.msg("Got back a FIN ACK. The destination port is closed") return @@ -92,33 +89,41 @@ class ParasiticalTracerouteTest(scapyt.BaseScapyTest): log.msg("Got a SYN ACK. All is well.") else: log.err("Got an unexpected result") + return - self.report['hops'] = [] - for ttl in range(1, max_ttl): - log.msg("Sending ACK with ttl %s" % ttl) - # We generate an ack for the syn-ack we got with increasing ttl - packet = IP(dst=self.localOptions['backend'], - ttl=ttl, id=ipid)/TCP(sport=synack.dport, + ack = ip_layer/TCP(sport=synack.dport, dport=dport, flags="A", seq=synack.ack, ack=synack.seq + 1) - answered, unanswered = yield self.sr(packet, timeout=timeout) - try: - snd, rcv = answered[0] - except IndexError: - log.err("Got no response.") + yield self.send(ack) - try: - icmp = rcv[ICMP] + self.report['hops'] = [] + # For the time being we make the assumption that we are NATted and + # that the NAT will forward the packet to the destination even if the TTL has + for ttl in range(1, self.max_ttl): + log.msg("Sending packet with ttl of %s" % ttl) + ip_layer.ttl = ttl + empty_tcp_packet = ip_layer/TCP(sport=synack.dport, + dport=dport, flags="A", + seq=synack.ack, ack=synack.seq + 1) + + answer = yield self.sr1(empty_tcp_packet) + if not answer: + log.err("Got no response for ttl %s" % ttl) + continue - except IndexError: - report = {'ttl': snd.ttl, - 'address': rcv.src, - 'rtt': rcv.time - snd.time + try: + icmp = answer[ICMP] + report = {'ttl': empty_tcp_packet.ttl, + 'address': answer.src, + 'rtt': answer.time - empty_tcp_packet.time } - log.debug("%s: %s" % (dport, report)) + log.msg("%s: %s" % (dport, report)) self.report['hops'].append(report) - if rcv.src == self.localOptions['backend']: + + except IndexError: + if answer.src == self.dst_ip: + answer.show() log.msg("Reached the destination. We have finished the traceroute") return diff --git a/nettests/core/tcpconnect.py b/nettests/core/tcpconnect.py index b763bf8..d0a53f8 100644 --- a/nettests/core/tcpconnect.py +++ b/nettests/core/tcpconnect.py @@ -16,10 +16,10 @@ class TCPConnectTest(nettest.NetTestCase): name = "TCP Connect" author = "Arturo Filastò" version = "0.1" - inputFile = ['file', 'f', None, 'File containing the IP:PORT combinations to be tested, one per line'] + requiredOptions = ['file'] def test_connect(self): """ This test performs a TCP connection to the remote host on the specified port. diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index d1ffb36..11b4381 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -12,13 +12,11 @@ from twisted.internet import protocol, defer, threads from scapy.all import send, sr, IP, TCP, config from ooni.reporter import createPacketReport - from ooni.nettest import NetTestCase from ooni.utils import log - from ooni import config -from ooni.utils.txscapy import ScapyProtocol +from ooni.utils.txscapy import ScapyProtocol, getDefaultIface class BaseScapyTest(NetTestCase): """ @@ -35,9 +33,12 @@ class BaseScapyTest(NetTestCase): requiresRoot = True baseFlags = [ - ['ipsrc', 's', 'Does *not* check if IP src and ICMP IP citation matches when processing answers'], - ['seqack', 'k', 'Check if TCP sequence number and ACK match in the ICMP citation when processing answers'], - ['ipid', 'i', 'Check if the IPID matches when processing answers'] + ['ipsrc', 's', + 'Does *not* check if IP src and ICMP IP citation matches when processing answers'], + ['seqack', 'k', + 'Check if TCP sequence number and ACK match in the ICMP citation when processing answers'], + ['ipid', 'i', + 'Check if the IPID matches when processing answers'] ] def _setUp(self): @@ -65,15 +66,26 @@ class BaseScapyTest(NetTestCase): else: config.check_TCPerror_seqack = 0 + if config.advanced.interface == 'auto': + self.interface = getDefaultIface() + else: + self.interface = config.advanced.interface + + def reportSentPacket(self, packet): + if 'sent_packets' not in self.report: + self.report['sent_packets'] = [] + self.report['sent_packets'].append(packet) + + def reportReceivedPacket(self, packet): + if 'answered_packets' not in self.report: + self.report['answered_packets'] = [] + self.report['answered_packets'].append(packet) + def finishedSendReceive(self, packets): """ This gets called when all packets have been sent and received. """ answered, unanswered = packets - if 'answered_packets' not in self.report: - self.report['answered_packets'] = [] - if 'sent_packets' not in self.report: - self.report['sent_packets'] = [] for snd, rcv in answered: log.debug("Writing report for scapy test") @@ -86,11 +98,8 @@ class BaseScapyTest(NetTestCase): sent_packet.src = '127.0.0.1' received_packet.dst = '127.0.0.1' - #pkt_report_r = createPacketReport(received_packet) - #pkt_report_s = createPacketReport(sent_packet) - self.report['answered_packets'].append(received_packet) - self.report['sent_packets'].append(sent_packet) - log.debug("Done") + self.reportSentPacket(sent_packet) + self.reportReceivedPacket(received_packet) return packets def sr(self, packets, *arg, **kw): @@ -98,25 +107,41 @@ class BaseScapyTest(NetTestCase): Wrapper around scapy.sendrecv.sr for sending and receiving of packets at layer 3. """ - scapyProtocol = ScapyProtocol(*arg, **kw) + scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) d = scapyProtocol.startSending(packets) d.addCallback(self.finishedSendReceive) return d def sr1(self, packets, *arg, **kw): - scapyProtocol = ScapyProtocol(*arg, **kw) + def done(packets): + """ + We do this so that the returned value is only the one packet that + we expected a response for, identical to the scapy implementation + of sr1. + """ + try: + return packets[0][0][1] + except IndexError: + log.err("Got no response...") + return None + + scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) scapyProtocol.expected_answers = 1 log.debug("Running sr1") d = scapyProtocol.startSending(packets) log.debug("Started to send") d.addCallback(self.finishedSendReceive) + d.addCallback(done) return d - def send(self, pkts, *arg, **kw): + def send(self, packets, *arg, **kw): """ Wrapper around scapy.sendrecv.send for sending of packets at layer 3 """ - raise Exception("Not implemented") + scapyProtocol = ScapyProtocol(interface=self.interface, *arg, **kw) + scapyProtocol.sendPackets(packets) + scapyProtocol.stopSending() + for packet in packets: + self.reportSentPacket(packet) ScapyTest = BaseScapyTest - diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 155abd2..e828977 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -22,13 +22,6 @@ from ooni.utils import log, txscapy #if sys.platform.system() == 'Windows': # import _winreg as winreg -PLATFORMS = {'LINUX': sys.platform.startswith("linux"), - 'OPENBSD': sys.platform.startswith("openbsd"), - 'FREEBSD': sys.platform.startswith("freebsd"), - 'NETBSD': sys.platform.startswith("netbsd"), - 'DARWIN': sys.platform.startswith("darwin"), - 'SOLARIS': sys.platform.startswith("sunos"), - 'WINDOWS': sys.platform.startswith("win32")} userAgents = [ ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), @@ -44,7 +37,6 @@ userAgents = [ ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP") ] - class UnsupportedPlatform(Exception): """Support for this platform is not currently available.""" @@ -54,7 +46,6 @@ class IfaceError(Exception): class PermissionsError(SystemExit): """This test requires admin or root privileges to run. Exiting...""" - PLATFORMS = {'LINUX': sys.platform.startswith("linux"), 'OPENBSD': sys.platform.startswith("openbsd"), 'FREEBSD': sys.platform.startswith("freebsd"), @@ -63,14 +54,6 @@ PLATFORMS = {'LINUX': sys.platform.startswith("linux"), 'SOLARIS': sys.platform.startswith("sunos"), 'WINDOWS': sys.platform.startswith("win32")} -class UnsupportedPlatform(Exception): - """Support for this platform is not currently available.""" - -class IfaceError(Exception): - """Could not find default network interface.""" - -class PermissionsError(SystemExit): - """This test requires admin or root privileges to run. Exiting...""" class StringProducer(object): implements(IBodyProducer) @@ -128,7 +111,6 @@ def getSystemResolver(): XXX implement a function that returns the resolver that is currently default on the system. """ - pass def getClientPlatform(platform_name=None): for name, test in PLATFORMS.items(): @@ -225,30 +207,8 @@ def getNonLoopbackIfaces(platform_name=None): else: return interfaces -def getNetworksFromRoutes(): - from scapy.all import conf, ltoa, read_routes - from ipaddr import IPNetwork, IPAddress - - ## Hide the 'no routes' warnings - conf.verb = 0 - - networks = [] - for nw, nm, gw, iface, addr in read_routes(): - n = IPNetwork( ltoa(nw) ) - (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]] - n.iface = iface - if not n.compressed in networks: - networks.append(n) - - return networks - -def getDefaultIface(): - networks = getNetworksFromRoutes() - for net in networks: - if net.is_private: - return net.iface - raise IfaceError def getLocalAddress(): default_iface = getDefaultIface() return default_iface.ipaddr + diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index 4341f68..e41d649 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -17,28 +17,59 @@ from twisted.internet import reactor, threads, error from twisted.internet import defer, abstract from zope.interface import implements - from scapy.all import PcapWriter, MTU from scapy.all import BasePacketList, conf, PcapReader from scapy.all import conf, Gen, SetGen +from scapy.arch import pcapdnet from ooni.utils import log +conf.use_pcap = True +conf.use_dnet = True + +def getNetworksFromRoutes(): + from scapy.all import conf, ltoa, read_routes + from ipaddr import IPNetwork, IPAddress + + ## Hide the 'no routes' warnings + conf.verb = 0 + + networks = [] + for nw, nm, gw, iface, addr in read_routes(): + n = IPNetwork( ltoa(nw) ) + (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]] + n.iface = iface + if not n.compressed in networks: + networks.append(n) + + return networks + +def getDefaultIface(): + networks = getNetworksFromRoutes() + for net in networks: + if net.is_private: + return net.iface + raise IfaceError + class TXPcapWriter(PcapWriter): def __init__(self, *arg, **kw): PcapWriter.__init__(self, *arg, **kw) fdesc.setNonBlocking(self.f) class ScapyProtocol(abstract.FileDescriptor): - def __init__(self, super_socket=None, - reactor=None, timeout=4, receive=True): + def __init__(self, interface, super_socket=None, timeout=5): abstract.FileDescriptor.__init__(self, reactor) # By default we use the conf.L3socket if not super_socket: - super_socket = conf.L3socket() + super_socket = pcapdnet.L3dnetSocket(iface=interface) + print super_socket + log.msg("Creating layer 3 socket with interface %s" % interface) + + #fdesc._setCloseOnExec(super_socket.ins.fileno()) self.super_socket = super_socket + self.interface = interface self.timeout = timeout # This dict is used to store the unique hashes that allow scapy to @@ -53,13 +84,8 @@ class ScapyProtocol(abstract.FileDescriptor): # This deferred will fire when we have finished sending a receiving packets. self.d = defer.Deferred() - self.debug = False + # Should we look for multiple answers for the same sent packet? self.multi = False - # XXX this needs to be implemented. It would involve keeping track of - # the state of the sending via the super socket file descriptor and - # firing the callback when we have concluded sending. Check out - # twisted.internet.udp to see how this is done. - self.receive = receive # When 0 we stop when all the packets we have sent have received an # answer @@ -85,7 +111,6 @@ class ScapyProtocol(abstract.FileDescriptor): if len(self.answered_packets) == len(self.sent_packets): log.debug("All of our questions have been answered.") - log.debug("%s" % self.__hash__) self.stopSending() return @@ -94,12 +119,11 @@ class ScapyProtocol(abstract.FileDescriptor): log.debug("Got the number of expected answers") self.stopSending() - def doRead(self): timeout = time.time() - self._start_time if self.timeout and time.time() - self._start_time > self.timeout: self.stopSending() - packet = self.super_socket.recv() + packet = self.super_socket.recv(MTU) if packet: self.processPacket(packet) # A string that has the same value for the request than for the @@ -110,7 +134,6 @@ class ScapyProtocol(abstract.FileDescriptor): self.processAnswer(packet, answer_hr) def stopSending(self): - log.debug("Stopping sending") self.stopReading() self.super_socket.close() if hasattr(self, "d"): @@ -120,20 +143,20 @@ class ScapyProtocol(abstract.FileDescriptor): def write(self, packet): """ - Write a scapy packet to the wire + Write a scapy packet to the wire. """ - hashret = packet.hashret() - if hashret in self.hr_sent_packets: - self.hr_sent_packets[hashret].append(packet) - else: - self.hr_sent_packets[hashret] = [packet] - self.sent_packets.append(packet) return self.super_socket.send(packet) def sendPackets(self, packets): if not isinstance(packets, Gen): packets = SetGen(packets) for packet in packets: + hashret = packet.hashret() + if hashret in self.hr_sent_packets: + self.hr_sent_packets[hashret].append(packet) + else: + self.hr_sent_packets[hashret] = [packet] + self.sent_packets.append(packet) self.write(packet) def startSending(self, packets): @@ -142,14 +165,4 @@ class ScapyProtocol(abstract.FileDescriptor): self.sendPackets(packets) return self.d -def sr(x, filter=None, iface=None, nofilter=0, timeout=None): - super_socket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter) - sp = ScapyProtocol(super_socket=super_socket, timeout=timeout) - return sp.startSending(x) - -def send(x, filter=None, iface=None, nofilter=0, timeout=None): - super_socket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter) - sp = ScapyProtocol(super_socket=super_socket, timeout=timeout) - return sp.startSending(x) - diff --git a/ooniprobe.conf b/ooniprobe.conf index 1998c93..e9f208f 100644 --- a/ooniprobe.conf +++ b/ooniprobe.conf @@ -25,4 +25,8 @@ advanced: debug: true threadpool_size: 10 tor_socksport: 9050 + # For auto detection + interface: auto + # Of specify a specific interface + #interface: wlan0

1 0

[ooni-probe/master] Put a sane timeout default on txscapy
by art＠torproject.org 24 Nov '12

24 Nov '12

commit 144263051c84c45e29c6a4cd29e6c04e5a30ff47 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 14:49:17 2012 +0100 Put a sane timeout default on txscapy --- ooni/utils/txscapy.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index d87a7a0..4341f68 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -32,7 +32,7 @@ class TXPcapWriter(PcapWriter): class ScapyProtocol(abstract.FileDescriptor): def __init__(self, super_socket=None, - reactor=None, timeout=None, receive=True): + reactor=None, timeout=4, receive=True): abstract.FileDescriptor.__init__(self, reactor) # By default we use the conf.L3socket if not super_socket:

1 0

[ooni-probe/master] Refactor the logic for dumping the YAML reports of packets.
by art＠torproject.org 24 Nov '12

24 Nov '12

commit 620a591c290d4b47803dc2f4cf07e343d56d3618 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 14:24:48 2012 +0100 Refactor the logic for dumping the YAML reports of packets. * Override the yaml.representer.represent_data as a hackish quick fix to the problem. * Outline a possible long term solution to it. --- nettests/core/traceroute.py | 8 ++++- ooni/reporter.py | 78 +++++++++++++++++++++++++++++++++++++++++-- ooni/utils/hacks.py | 45 ------------------------- 3 files changed, 82 insertions(+), 49 deletions(-) diff --git a/nettests/core/traceroute.py b/nettests/core/traceroute.py index a6fc9f1..6a443d9 100644 --- a/nettests/core/traceroute.py +++ b/nettests/core/traceroute.py @@ -50,6 +50,12 @@ class TracerouteTest(scapyt.BaseScapyTest): return max_ttl, timeout + def postProcessor(self, report): + tcp_hops = report['test_tcp_traceroute'] + udp_hops = report['test_udp_traceroute'] + icmp_hops = report['test_icmp_traceroute'] + + def test_tcp_traceroute(self): """ Does a traceroute to the destination by sending TCP SYN packets @@ -126,7 +132,7 @@ class TracerouteTest(scapyt.BaseScapyTest): dl = [] max_ttl, timeout = self.max_ttl_and_timeout() packets = IP(dst=self.localOptions['backend'], - ttl=(1,max_ttl),id=RandShort())/ICMP() + ttl=(1,max_ttl), id=RandShort())/ICMP() d = self.sr(packets, timeout=timeout) d.addCallback(finished) diff --git a/ooni/reporter.py b/ooni/reporter.py index 7fc49e2..1e2924d 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -15,6 +15,13 @@ import yaml import json import traceback +from yaml.representer import * +from yaml.emitter import * +from yaml.serializer import * +from yaml.resolver import * + +from scapy.packet import Packet + from twisted.python.util import untilConcludes from twisted.trial import reporter from twisted.internet import defer, reactor @@ -23,10 +30,75 @@ from twisted.internet.error import ConnectionRefusedError from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni.utils import otime, log, geodata -from ooni.utils.hacks import OSafeRepresenter, OSafeDumper from ooni import config -from scapy.all import packet +def createPacketReport(packet_list): + """ + Takes as input a packet a list. + + Returns a dict containing a dict with the packet + summary and the raw packet. + """ + report = [] + for packet in packet_list: + report.append({'raw_packet': str(packet), + 'summary': str(packet.summary())}) + return report + +class OSafeRepresenter(SafeRepresenter): + """ + This is a custom YAML representer that allows us to represent reports + safely. + It extends the SafeRepresenter to be able to also represent complex + numbers and scapy packet. + """ + def represent_data(self, data): + """ + This is very hackish. There is for sure a better way either by using + the add_multi_representer or add_representer, the issue though lies in + the fact that Scapy packets are metaclasses that leads to + yaml.representer.get_classobj_bases to not be able to properly get the + base of class of a Scapy packet. + XXX fully debug this problem + """ + if isinstance(data, Packet): + data = createPacketReport(data) + return SafeRepresenter.represent_data(self, data) + + def represent_complex(self, data): + if data.imag == 0.0: + data = u'%r' % data.real + elif data.real == 0.0: + data = u'%rj' % data.imag + elif data.imag > 0: + data = u'%r+%rj' % (data.real, data.imag) + else: + data = u'%r%rj' % (data.real, data.imag) + return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data) + +OSafeRepresenter.add_representer(complex, + OSafeRepresenter.represent_complex) + +class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): + """ + This is a modification of the YAML Safe Dumper to use our own Safe + Representer that supports complex numbers. + """ + def __init__(self, stream, + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + OSafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style) + Resolver.__init__(self) class NoTestIDSpecified(Exception): pass @@ -103,7 +175,7 @@ class OReporter(object): log.debug("Writing report") test_report = dict(test.report) - if isinstance(test.input, packet.Packet): + if isinstance(test.input, Packet): test_input = createPacketReport(test.input) else: test_input = test.input diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py index 8912bbb..d4e832c 100644 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@ -8,10 +8,6 @@ # :authors: Arturo Filastò # :licence: see LICENSE -from yaml.representer import * -from yaml.emitter import * -from yaml.serializer import * -from yaml.resolver import * import copy_reg @@ -67,44 +63,3 @@ def patched_reduce_ex(self, proto): else: return copy_reg._reconstructor, args -class OSafeRepresenter(SafeRepresenter): - """ - This is a custom YAML representer that allows us to represent reports - safely. - It extends the SafeRepresenter to be able to also represent complex numbers - """ - def represent_complex(self, data): - if data.imag == 0.0: - data = u'%r' % data.real - elif data.real == 0.0: - data = u'%rj' % data.imag - elif data.imag > 0: - data = u'%r+%rj' % (data.real, data.imag) - else: - data = u'%r%rj' % (data.real, data.imag) - return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data) - -OSafeRepresenter.add_representer(complex, - OSafeRepresenter.represent_complex) - -class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): - """ - This is a modification of the YAML Safe Dumper to use our own Safe - Representer that supports complex numbers. - """ - def __init__(self, stream, - default_style=None, default_flow_style=None, - canonical=None, indent=None, width=None, - allow_unicode=None, line_break=None, - encoding=None, explicit_start=None, explicit_end=None, - version=None, tags=None): - Emitter.__init__(self, stream, canonical=canonical, - indent=indent, width=width, - allow_unicode=allow_unicode, line_break=line_break) - Serializer.__init__(self, encoding=encoding, - explicit_start=explicit_start, explicit_end=explicit_end, - version=version, tags=tags) - OSafeRepresenter.__init__(self, default_style=default_style, - default_flow_style=default_flow_style) - Resolver.__init__(self) -

1 0

[ooni-probe/master] Update scapy test to support sr1
by art＠torproject.org 24 Nov '12

24 Nov '12

commit 36e8839465032b20ded258de4460a38edf28a40c Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 24 14:25:49 2012 +0100 Update scapy test to support sr1 * Refactor scapy test to avoid too much code duplication --- ooni/templates/scapyt.py | 76 ++++++++++++++++++++++++---------------------- 1 files changed, 40 insertions(+), 36 deletions(-) diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index af90260..d1ffb36 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -11,6 +11,8 @@ from twisted.internet import protocol, defer, threads from scapy.all import send, sr, IP, TCP, config +from ooni.reporter import createPacketReport + from ooni.nettest import NetTestCase from ooni.utils import log @@ -18,17 +20,6 @@ from ooni import config from ooni.utils.txscapy import ScapyProtocol -def createPacketReport(packet_list): - """ - Takes as input a packet a list containing a dict with the packet - summary and the raw packet. - """ - report = [] - for packet in packet_list: - report.append({'raw_packet': str(packet), - 'summary': str(packet.summary())}) - return report - class BaseScapyTest(NetTestCase): """ The report of a test run with scapy looks like this: @@ -74,39 +65,51 @@ class BaseScapyTest(NetTestCase): else: config.check_TCPerror_seqack = 0 + def finishedSendReceive(self, packets): + """ + This gets called when all packets have been sent and received. + """ + answered, unanswered = packets + if 'answered_packets' not in self.report: + self.report['answered_packets'] = [] + if 'sent_packets' not in self.report: + self.report['sent_packets'] = [] + + for snd, rcv in answered: + log.debug("Writing report for scapy test") + sent_packet = snd + received_packet = rcv + + if not config.privacy.includeip: + log.msg("Detected you would not like to include your ip in the report") + log.msg("Stripping source and destination IPs from the reports") + sent_packet.src = '127.0.0.1' + received_packet.dst = '127.0.0.1' + + #pkt_report_r = createPacketReport(received_packet) + #pkt_report_s = createPacketReport(sent_packet) + self.report['answered_packets'].append(received_packet) + self.report['sent_packets'].append(sent_packet) + log.debug("Done") + return packets + def sr(self, packets, *arg, **kw): """ Wrapper around scapy.sendrecv.sr for sending and receiving of packets at layer 3. """ - def finished(packets): - answered, unanswered = packets - if 'answered_packets' not in self.report: - self.report['answered_packets'] = [] - if 'sent_packets' not in self.report: - self.report['sent_packets'] = [] - - for snd, rcv in answered: - log.debug("Writing report for scapy test") - sent_packet = snd - received_packet = rcv - - if not config.privacy.includeip: - log.msg("Detected you would not like to include your ip in the report") - log.msg("Stripping source and destination IPs from the reports") - sent_packet.src = '127.0.0.1' - received_packet.dst = '127.0.0.1' - - pkt_report_r = createPacketReport(received_packet) - pkt_report_s = createPacketReport(sent_packet) - self.report['answered_packets'].append(pkt_report_r) - self.report['sent_packets'].append(pkt_report_s) - log.debug("Done") - return packets + scapyProtocol = ScapyProtocol(*arg, **kw) + d = scapyProtocol.startSending(packets) + d.addCallback(self.finishedSendReceive) + return d + def sr1(self, packets, *arg, **kw): scapyProtocol = ScapyProtocol(*arg, **kw) + scapyProtocol.expected_answers = 1 + log.debug("Running sr1") d = scapyProtocol.startSending(packets) - d.addCallback(finished) + log.debug("Started to send") + d.addCallback(self.finishedSendReceive) return d def send(self, pkts, *arg, **kw): @@ -115,4 +118,5 @@ class BaseScapyTest(NetTestCase): """ raise Exception("Not implemented") +ScapyTest = BaseScapyTest

1 0