commit 4f4cf2e0e2adfb8ae3d71fb29ac413d9e4c81ae7 Author: kudrom kudrom@riseup.net Date: Sat Jul 19 18:09:12 2014 +0200

Fixed a couple things: * sniffer subsystem to allow parallel recording of pcaps * keyword_filtering nettest * improved some tests in test_oonibclient --- .gitignore | 3 +- data/ooniprobe.1 | 2 +- data/ooniprobe.conf.sample | 2 +- ooni/director.py | 56 +++++++++++++---------- ooni/nettests/experimental/keyword_filtering.py | 31 +++++++------ ooni/oonicli.py | 4 ++ ooni/settings.py | 5 -- ooni/templates/scapyt.py | 8 +--- ooni/tests/test_oonibclient.py | 18 ++------ ooni/utils/txscapy.py | 21 +++------ 10 files changed, 68 insertions(+), 82 deletions(-)

diff --git a/.gitignore b/.gitignore index 9632014..1e2dba3 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ private/* ooni/report*.yaml report*.yaml *.yamloo +*.pcap pcaps/

ooniprobe.conf @@ -41,4 +42,4 @@ docs/build/* .vagrant/*

cover/* -ooni_home/* \ No newline at end of file +ooni_home/* diff --git a/data/ooniprobe.1 b/data/ooniprobe.1 index cb5309b..7da9ae6 100644 --- a/data/ooniprobe.1 +++ b/data/ooniprobe.1 @@ -95,7 +95,7 @@ the addresses of test helpers. default: httpo://nkvphnp3p6agi5qq.onion Path to the log file to write .TP .BR -^O " or " --pcapfile -Path to the pcap file name. +Prefix to the pcap file name. .TP .BR -^f " or " --configfile Specify a path to the ooniprobe configuration file. diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index d32248f..b08edf6 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -17,7 +17,7 @@ privacy: # Should we collect a full packet capture on the client? includepcap: false reports: - # This is a packet capture file (.pcap) to load as a test: + # This is a prefix for each packet capture file (.pcap) per test: pcap: null collector: null advanced: diff --git a/ooni/director.py b/ooni/director.py index 901088f..3634115 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -1,4 +1,6 @@ import os +import otime +from psutil import Process

from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report @@ -15,7 +17,6 @@ from twisted.internet.endpoints import TCP4ClientEndpoint

class Director(object): - """ Singleton object responsible for coordinating the Measurements Manager and the Reporting Manager. @@ -90,14 +91,13 @@ class Director(object): # This deferred is fired once all the measurements and their reporting # tasks are completed. self.allTestsDone = defer.Deferred() - self.sniffer = None + self.sniffers = {}

def getNetTests(self): nettests = {}

def is_nettest(filename): - return not filename == '__init__.py' \ - and filename.endswith('.py') + return not filename == '__init__.py' and filename.endswith('.py')

for category in self.categories: dirname = os.path.join(config.nettest_directory, category) @@ -191,6 +191,11 @@ class Director(object): self.totalMeasurementRuntime += measurement.runtime self.successfulMeasurements += 1 measurement.result = result + test_name = measurement.testInstance.__class__.__name__ + sniffer = self.sniffers[test_name] + config.scapyFactory.unRegisterProtocol(sniffer) + sniffer.close() + del self.sniffers[test_name] return measurement

def measurementFailed(self, failure, measurement): @@ -229,16 +234,11 @@ class Director(object): net_test_loader: an instance of :class:ooni.nettest.NetTestLoader """ - if self.allTestsDone.called: self.allTestsDone = defer.Deferred()

if config.privacy.includepcap: - if not config.reports.pcap: - config.reports.pcap = config.generate_pcap_filename( - net_test_loader.testDetails - ) - self.startSniffing() + self.startSniffing(net_test_loader.testDetails)

report = Report(net_test_loader.testDetails, report_filename, self.reportEntryManager, collector_address) @@ -258,24 +258,30 @@ class Director(object): finally: self.netTestDone(net_test)

- def startSniffing(self): + def startSniffing(self, testDetails): """ Start sniffing with Scapy. Exits if required privileges (root) are not available. """ - from ooni.utils.txscapy import ScapyFactory, ScapySniffer - config.scapyFactory = ScapyFactory(config.advanced.interface) - - if os.path.exists(config.reports.pcap): - log.msg("Report PCAP already exists with filename %s" % - config.reports.pcap) - log.msg("Renaming files with such name...") - pushFilenameStack(config.reports.pcap) - - if self.sniffer: - config.scapyFactory.unRegisterProtocol(self.sniffer) - self.sniffer = ScapySniffer(config.reports.pcap) - config.scapyFactory.registerProtocol(self.sniffer) - log.msg("Starting packet capture to: %s" % config.reports.pcap) + from ooni.utils.txscapy import ScapySniffer + + test_name, start_time = testDetails['test_name'], testDetails['start_time'] + start_time = otime.epochToTimestamp(start_time) + suffix = "%s-%s.%s" % (test_name, start_time, "pcap") + if not config.reports.pcap: + filename_pcap= "%s-%s" % ("report", suffix) + else: + filename_pcap = "%s-%s" % (config.reports.pcap, suffix) + + if len(self.sniffers) > 1: + pcap_filenames = set(sniffer.pcapwriter.filename for sniffer in self.sniffers) + pcap_filenames.add(filename_pcap) + log.msg("pcap files %s can be messed up because several netTests are being executed in parallel." % + ','.join(pcap_filenames)) + + sniffer = ScapySniffer(filename_pcap) + self.sniffers[testDetails['test_name']] = sniffer + config.scapyFactory.registerProtocol(sniffer) + log.msg("Starting packet capture to: %s" % filename_pcap)

@defer.inlineCallbacks def getTorState(self): diff --git a/ooni/nettests/experimental/keyword_filtering.py b/ooni/nettests/experimental/keyword_filtering.py index 0937efd..b54ba0d 100644 --- a/ooni/nettests/experimental/keyword_filtering.py +++ b/ooni/nettests/experimental/keyword_filtering.py @@ -9,23 +9,26 @@ from twisted.internet import defer from ooni.utils import log from ooni.templates import scapyt

-from scapy.all import * +from scapy.layers.inet import TCP, IP +from scapy.volatile import RandShort +

class UsageOptions(usage.Options): optParameters = [ - ['backend', 'b', '127.0.0.1:57002', 'Test backend running TCP echo'], - ['timeout', 't', 5, 'Timeout after which to give up waiting for RST packets'] - ] + ['backend', 'b', '127.0.0.1:57002', 'Test backend running TCP echo'], + ['timeout', 't', 5, 'Timeout after which to give up waiting for RST packets'] + ] +

class KeywordFiltering(scapyt.BaseScapyTest): name = "Keyword Filtering detection based on RST packets" author = "Arturo Filastò" - version = "0.1" + version = "0.2"

usageOptions = UsageOptions

- inputFile = ['file', 'f', None, - 'List of keywords to use for censorship testing'] + inputFile = ['file', 'f', None, + 'List of keywords to use for censorship testing'] requiresRoot = True requiresTor = False

@@ -36,19 +39,19 @@ class KeywordFiltering(scapyt.BaseScapyTest): though this should not be an issue since we are testing all the keywords in parallel. """ + backend_ip, backend_port = self.localOptions['backend'].split(':') + timeout = int(self.localOptions['timeout']) + keyword_to_test = str(self.input) + packets = IP(dst=backend_ip, id=RandShort()) / TCP(sport=4000, dport=int(backend_port)) / keyword_to_test + d = self.sr(packets, timeout=timeout) + + @d.addCallback def finished(packets): - log.debug("Finished running TCP traceroute test on port %s" % port) answered, unanswered = packets self.report['rst_packets'] = [] for snd, rcv in answered: # The received packet has the RST flag if rcv[TCP].flags == 4: self.report['rst_packets'].append(rcv) - - backend_ip, backend_port = self.localOptions['backend'] - keyword_to_test = str(self.input) - packets = IP(dst=backend_ip,id=RandShort())/TCP(dport=backend_port)/keyword_to_test - d = self.sr(packets, timeout=timeout) - d.addCallback(finished) return d

diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 8313822..7c3e9a2 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -12,6 +12,7 @@ from ooni.settings import config from ooni.director import Director from ooni.deck import Deck, nettest_to_path from ooni.nettest import NetTestLoader +from ooni.utils.txscapy import ScapyFactory

from ooni.utils import log, checkForRoot

@@ -111,14 +112,17 @@ def runWithDirector(logging=True, start_tor=True): config.set_paths() config.initialize_ooni_home() config.read_config_file() + if global_options['verbose']: config.advanced.debug = True + if not start_tor: config.advanced.start_tor = False

if logging: log.start(global_options['logfile'])

+ config.scapyFactory = ScapyFactory(config.advanced.interface) if config.privacy.includepcap: try: checkForRoot() diff --git a/ooni/settings.py b/ooni/settings.py index 0d1275e..5bebc05 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -113,9 +113,4 @@ class OConfig(object): pass self.set_paths()

- def generate_pcap_filename(self, testDetails): - test_name, start_time = testDetails['test_name'], testDetails['start_time'] - start_time = otime.epochToTimestamp(start_time) - return "report-%s-%s.%s" % (test_name, start_time, "pcap") - config = OConfig() diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index ee21508..d99ea3f 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -41,10 +41,6 @@ class BaseScapyTest(NetTestCase): def _setUp(self): super(BaseScapyTest, self)._setUp()

- if not config.scapyFactory: - log.debug("Scapy factory not set, registering it.") - config.scapyFactory = ScapyFactory(config.advanced.interface) - self.report['answer_flags'] = [] if self.localOptions['ipsrc']: config.checkIPsrc = 0 @@ -93,12 +89,12 @@ class BaseScapyTest(NetTestCase): self.report['answered_packets'].append(received_packet) return packets

- def sr(self, packets, *arg, **kw): + def sr(self, packets, timeout=None, *arg, **kw): """ Wrapper around scapy.sendrecv.sr for sending and receiving of packets at layer 3. """ - scapySender = ScapySender() + scapySender = ScapySender(timeout=timeout)

config.scapyFactory.registerProtocol(scapySender) log.debug("Using sending with hash %s" % scapySender.__hash__) diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index d80b606..87ea8da 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -28,10 +28,8 @@ class TestOONIBClient(ConfigTestCase): data_dir = '/tmp/testooni' config.advanced.data_dir = data_dir

- try: + if os.path.exists(data_dir): shutil.rmtree(data_dir) - except: - pass os.mkdir(data_dir) os.mkdir(os.path.join(data_dir, 'inputs')) os.mkdir(os.path.join(data_dir, 'decks')) @@ -101,21 +99,11 @@ class TestOONIBClient(ConfigTestCase):

@defer.inlineCallbacks def test_input_descriptor_not_found(self): - try: - yield self.oonibclient.queryBackend('GET', '/input/' + 'a'*64) - except e.OONIBInputDescriptorNotFound: - pass - else: - assert False + yield self.assertFailure(self.oonibclient.queryBackend('GET', '/input/' + 'a'*64), e.OONIBInputDescriptorNotFound)

@defer.inlineCallbacks def test_http_errors(self): - try: - yield self.oonibclient.queryBackend('PUT', '/policy/input') - except error.Error: - pass - else: - assert False + yield self.assertFailure(self.oonibclient.queryBackend('PUT', '/policy/input'), error.Error)

@defer.inlineCallbacks def test_create_report(self): diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index bfd5d7f..89a0236 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -1,5 +1,3 @@ -import ipaddr -import struct import socket import os import sys @@ -9,12 +7,10 @@ import random from twisted.internet import protocol, base, fdesc from twisted.internet import reactor, threads, error from twisted.internet import defer, abstract -from zope.interface import implements

from scapy.config import conf from scapy.supersocket import L3RawSocket -from scapy.all import RandShort, IP, IPerror, ICMP, ICMPerror -from scapy.all import TCP, TCPerror, UDP, UDPerror +from scapy.all import RandShort, IP, IPerror, ICMP, ICMPerror, TCP, TCPerror, UDP, UDPerror, read_routes

from ooni.utils import log from ooni.settings import config @@ -53,10 +49,8 @@ def pcapdnet_installed():

config.pcap_dnet = True

- except ImportError: - log.err("pypcap or dnet not installed. " - "Certain tests may not work.") - + except ImportError as e: + log.err(e.message + ". Pypcap or dnet are not properly installed. Certain tests may not work.") config.pcap_dnet = False conf.use_pcap = False conf.use_dnet = False @@ -320,6 +314,9 @@ class ScapySniffer(ScapyProtocol): def packetReceived(self, packet): self.pcapwriter.write(packet)

+ def close(self): + self.pcapwriter.close() +

class ParasiticTraceroute(ScapyProtocol): def __init__(self): @@ -363,11 +360,7 @@ class ParasiticTraceroute(ScapyProtocol):

def maxttl(packet=None): if packet: - return min(self.ttl_max, - min( - abs(64 - packet.ttl), - abs(128 - packet.ttl), - abs(256 - packet.ttl))) - 1 + return min(self.ttl_max, *map(lambda x: x - packet.ttl, [64, 128, 256])) - 1 else: return self.ttl_max