commit 4f4cf2e0e2adfb8ae3d71fb29ac413d9e4c81ae7
Author: kudrom <kudrom(a)riseup.net>
Date: Sat Jul 19 18:09:12 2014 +0200
Fixed a couple things:
* sniffer subsystem to allow parallel recording of pcaps
* keyword_filtering nettest
* improved some tests in test_oonibclient
---
.gitignore | 3 +-
data/ooniprobe.1 | 2 +-
data/ooniprobe.conf.sample | 2 +-
ooni/director.py | 56 +++++++++++++----------
ooni/nettests/experimental/keyword_filtering.py | 31 +++++++------
ooni/oonicli.py | 4 ++
ooni/settings.py | 5 --
ooni/templates/scapyt.py | 8 +---
ooni/tests/test_oonibclient.py | 18 ++------
ooni/utils/txscapy.py | 21 +++------
10 files changed, 68 insertions(+), 82 deletions(-)
diff --git a/.gitignore b/.gitignore
index 9632014..1e2dba3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ private/*
ooni/report*.yaml
report*.yaml
*.yamloo
+*.pcap
pcaps/
ooniprobe.conf
@@ -41,4 +42,4 @@ docs/build/*
.vagrant/*
cover/*
-ooni_home/*
\ No newline at end of file
+ooni_home/*
diff --git a/data/ooniprobe.1 b/data/ooniprobe.1
index cb5309b..7da9ae6 100644
--- a/data/ooniprobe.1
+++ b/data/ooniprobe.1
@@ -95,7 +95,7 @@ the addresses of test helpers. default: httpo://nkvphnp3p6agi5qq.onion
Path to the log file to write
.TP
.BR \-\^O " or " \-\-pcapfile
-Path to the pcap file name.
+Prefix to the pcap file name.
.TP
.BR \-\^f " or " \-\-configfile
Specify a path to the ooniprobe configuration file.
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index d32248f..b08edf6 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -17,7 +17,7 @@ privacy:
# Should we collect a full packet capture on the client?
includepcap: false
reports:
- # This is a packet capture file (.pcap) to load as a test:
+ # This is a prefix for each packet capture file (.pcap) per test:
pcap: null
collector: null
advanced:
diff --git a/ooni/director.py b/ooni/director.py
index 901088f..3634115 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -1,4 +1,6 @@
import os
+import otime
+from psutil import Process
from ooni.managers import ReportEntryManager, MeasurementManager
from ooni.reporter import Report
@@ -15,7 +17,6 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
class Director(object):
-
"""
Singleton object responsible for coordinating the Measurements Manager
and the Reporting Manager.
@@ -90,14 +91,13 @@ class Director(object):
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
- self.sniffer = None
+ self.sniffers = {}
def getNetTests(self):
nettests = {}
def is_nettest(filename):
- return not filename == '__init__.py' \
- and filename.endswith('.py')
+ return not filename == '__init__.py' and filename.endswith('.py')
for category in self.categories:
dirname = os.path.join(config.nettest_directory, category)
@@ -191,6 +191,11 @@ class Director(object):
self.totalMeasurementRuntime += measurement.runtime
self.successfulMeasurements += 1
measurement.result = result
+ test_name = measurement.testInstance.__class__.__name__
+ sniffer = self.sniffers[test_name]
+ config.scapyFactory.unRegisterProtocol(sniffer)
+ sniffer.close()
+ del self.sniffers[test_name]
return measurement
def measurementFailed(self, failure, measurement):
@@ -229,16 +234,11 @@ class Director(object):
net_test_loader:
an instance of :class:ooni.nettest.NetTestLoader
"""
-
if self.allTestsDone.called:
self.allTestsDone = defer.Deferred()
if config.privacy.includepcap:
- if not config.reports.pcap:
- config.reports.pcap = config.generate_pcap_filename(
- net_test_loader.testDetails
- )
- self.startSniffing()
+ self.startSniffing(net_test_loader.testDetails)
report = Report(net_test_loader.testDetails, report_filename,
self.reportEntryManager, collector_address)
@@ -258,24 +258,30 @@ class Director(object):
finally:
self.netTestDone(net_test)
- def startSniffing(self):
+ def startSniffing(self, testDetails):
""" Start sniffing with Scapy. Exits if required privileges (root) are not
available.
"""
- from ooni.utils.txscapy import ScapyFactory, ScapySniffer
- config.scapyFactory = ScapyFactory(config.advanced.interface)
-
- if os.path.exists(config.reports.pcap):
- log.msg("Report PCAP already exists with filename %s" %
- config.reports.pcap)
- log.msg("Renaming files with such name...")
- pushFilenameStack(config.reports.pcap)
-
- if self.sniffer:
- config.scapyFactory.unRegisterProtocol(self.sniffer)
- self.sniffer = ScapySniffer(config.reports.pcap)
- config.scapyFactory.registerProtocol(self.sniffer)
- log.msg("Starting packet capture to: %s" % config.reports.pcap)
+ from ooni.utils.txscapy import ScapySniffer
+
+ test_name, start_time = testDetails['test_name'], testDetails['start_time']
+ start_time = otime.epochToTimestamp(start_time)
+ suffix = "%s-%s.%s" % (test_name, start_time, "pcap")
+ if not config.reports.pcap:
+ filename_pcap= "%s-%s" % ("report", suffix)
+ else:
+ filename_pcap = "%s-%s" % (config.reports.pcap, suffix)
+
+ if len(self.sniffers) > 1:
+ pcap_filenames = set(sniffer.pcapwriter.filename for sniffer in self.sniffers)
+ pcap_filenames.add(filename_pcap)
+ log.msg("pcap files %s can be messed up because several netTests are being executed in parallel." %
+ ','.join(pcap_filenames))
+
+ sniffer = ScapySniffer(filename_pcap)
+ self.sniffers[testDetails['test_name']] = sniffer
+ config.scapyFactory.registerProtocol(sniffer)
+ log.msg("Starting packet capture to: %s" % filename_pcap)
@defer.inlineCallbacks
def getTorState(self):
diff --git a/ooni/nettests/experimental/keyword_filtering.py b/ooni/nettests/experimental/keyword_filtering.py
index 0937efd..b54ba0d 100644
--- a/ooni/nettests/experimental/keyword_filtering.py
+++ b/ooni/nettests/experimental/keyword_filtering.py
@@ -9,23 +9,26 @@ from twisted.internet import defer
from ooni.utils import log
from ooni.templates import scapyt
-from scapy.all import *
+from scapy.layers.inet import TCP, IP
+from scapy.volatile import RandShort
+
class UsageOptions(usage.Options):
optParameters = [
- ['backend', 'b', '127.0.0.1:57002', 'Test backend running TCP echo'],
- ['timeout', 't', 5, 'Timeout after which to give up waiting for RST packets']
- ]
+ ['backend', 'b', '127.0.0.1:57002', 'Test backend running TCP echo'],
+ ['timeout', 't', 5, 'Timeout after which to give up waiting for RST packets']
+ ]
+
class KeywordFiltering(scapyt.BaseScapyTest):
name = "Keyword Filtering detection based on RST packets"
author = "Arturo Filastò"
- version = "0.1"
+ version = "0.2"
usageOptions = UsageOptions
- inputFile = ['file', 'f', None,
- 'List of keywords to use for censorship testing']
+ inputFile = ['file', 'f', None,
+ 'List of keywords to use for censorship testing']
requiresRoot = True
requiresTor = False
@@ -36,19 +39,19 @@ class KeywordFiltering(scapyt.BaseScapyTest):
though this should not be an issue since we are testing all
the keywords in parallel.
"""
+ backend_ip, backend_port = self.localOptions['backend'].split(':')
+ timeout = int(self.localOptions['timeout'])
+ keyword_to_test = str(self.input)
+ packets = IP(dst=backend_ip, id=RandShort()) / TCP(sport=4000, dport=int(backend_port)) / keyword_to_test
+ d = self.sr(packets, timeout=timeout)
+
+ @d.addCallback
def finished(packets):
- log.debug("Finished running TCP traceroute test on port %s" % port)
answered, unanswered = packets
self.report['rst_packets'] = []
for snd, rcv in answered:
# The received packet has the RST flag
if rcv[TCP].flags == 4:
self.report['rst_packets'].append(rcv)
-
- backend_ip, backend_port = self.localOptions['backend']
- keyword_to_test = str(self.input)
- packets = IP(dst=backend_ip,id=RandShort())/TCP(dport=backend_port)/keyword_to_test
- d = self.sr(packets, timeout=timeout)
- d.addCallback(finished)
return d
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 8313822..7c3e9a2 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -12,6 +12,7 @@ from ooni.settings import config
from ooni.director import Director
from ooni.deck import Deck, nettest_to_path
from ooni.nettest import NetTestLoader
+from ooni.utils.txscapy import ScapyFactory
from ooni.utils import log, checkForRoot
@@ -111,14 +112,17 @@ def runWithDirector(logging=True, start_tor=True):
config.set_paths()
config.initialize_ooni_home()
config.read_config_file()
+
if global_options['verbose']:
config.advanced.debug = True
+
if not start_tor:
config.advanced.start_tor = False
if logging:
log.start(global_options['logfile'])
+ config.scapyFactory = ScapyFactory(config.advanced.interface)
if config.privacy.includepcap:
try:
checkForRoot()
diff --git a/ooni/settings.py b/ooni/settings.py
index 0d1275e..5bebc05 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -113,9 +113,4 @@ class OConfig(object):
pass
self.set_paths()
- def generate_pcap_filename(self, testDetails):
- test_name, start_time = testDetails['test_name'], testDetails['start_time']
- start_time = otime.epochToTimestamp(start_time)
- return "report-%s-%s.%s" % (test_name, start_time, "pcap")
-
config = OConfig()
diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py
index ee21508..d99ea3f 100644
--- a/ooni/templates/scapyt.py
+++ b/ooni/templates/scapyt.py
@@ -41,10 +41,6 @@ class BaseScapyTest(NetTestCase):
def _setUp(self):
super(BaseScapyTest, self)._setUp()
- if not config.scapyFactory:
- log.debug("Scapy factory not set, registering it.")
- config.scapyFactory = ScapyFactory(config.advanced.interface)
-
self.report['answer_flags'] = []
if self.localOptions['ipsrc']:
config.checkIPsrc = 0
@@ -93,12 +89,12 @@ class BaseScapyTest(NetTestCase):
self.report['answered_packets'].append(received_packet)
return packets
- def sr(self, packets, *arg, **kw):
+ def sr(self, packets, timeout=None, *arg, **kw):
"""
Wrapper around scapy.sendrecv.sr for sending and receiving of packets
at layer 3.
"""
- scapySender = ScapySender()
+ scapySender = ScapySender(timeout=timeout)
config.scapyFactory.registerProtocol(scapySender)
log.debug("Using sending with hash %s" % scapySender.__hash__)
diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py
index d80b606..87ea8da 100644
--- a/ooni/tests/test_oonibclient.py
+++ b/ooni/tests/test_oonibclient.py
@@ -28,10 +28,8 @@ class TestOONIBClient(ConfigTestCase):
data_dir = '/tmp/testooni'
config.advanced.data_dir = data_dir
- try:
+ if os.path.exists(data_dir):
shutil.rmtree(data_dir)
- except:
- pass
os.mkdir(data_dir)
os.mkdir(os.path.join(data_dir, 'inputs'))
os.mkdir(os.path.join(data_dir, 'decks'))
@@ -101,21 +99,11 @@ class TestOONIBClient(ConfigTestCase):
@defer.inlineCallbacks
def test_input_descriptor_not_found(self):
- try:
- yield self.oonibclient.queryBackend('GET', '/input/' + 'a'*64)
- except e.OONIBInputDescriptorNotFound:
- pass
- else:
- assert False
+ yield self.assertFailure(self.oonibclient.queryBackend('GET', '/input/' + 'a'*64), e.OONIBInputDescriptorNotFound)
@defer.inlineCallbacks
def test_http_errors(self):
- try:
- yield self.oonibclient.queryBackend('PUT', '/policy/input')
- except error.Error:
- pass
- else:
- assert False
+ yield self.assertFailure(self.oonibclient.queryBackend('PUT', '/policy/input'), error.Error)
@defer.inlineCallbacks
def test_create_report(self):
diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py
index bfd5d7f..89a0236 100644
--- a/ooni/utils/txscapy.py
+++ b/ooni/utils/txscapy.py
@@ -1,5 +1,3 @@
-import ipaddr
-import struct
import socket
import os
import sys
@@ -9,12 +7,10 @@ import random
from twisted.internet import protocol, base, fdesc
from twisted.internet import reactor, threads, error
from twisted.internet import defer, abstract
-from zope.interface import implements
from scapy.config import conf
from scapy.supersocket import L3RawSocket
-from scapy.all import RandShort, IP, IPerror, ICMP, ICMPerror
-from scapy.all import TCP, TCPerror, UDP, UDPerror
+from scapy.all import RandShort, IP, IPerror, ICMP, ICMPerror, TCP, TCPerror, UDP, UDPerror, read_routes
from ooni.utils import log
from ooni.settings import config
@@ -53,10 +49,8 @@ def pcapdnet_installed():
config.pcap_dnet = True
- except ImportError:
- log.err("pypcap or dnet not installed. "
- "Certain tests may not work.")
-
+ except ImportError as e:
+ log.err(e.message + ". Pypcap or dnet are not properly installed. Certain tests may not work.")
config.pcap_dnet = False
conf.use_pcap = False
conf.use_dnet = False
@@ -320,6 +314,9 @@ class ScapySniffer(ScapyProtocol):
def packetReceived(self, packet):
self.pcapwriter.write(packet)
+ def close(self):
+ self.pcapwriter.close()
+
class ParasiticTraceroute(ScapyProtocol):
def __init__(self):
@@ -363,11 +360,7 @@ class ParasiticTraceroute(ScapyProtocol):
def maxttl(packet=None):
if packet:
- return min(self.ttl_max,
- min(
- abs(64 - packet.ttl),
- abs(128 - packet.ttl),
- abs(256 - packet.ttl))) - 1
+ return min(self.ttl_max, *map(lambda x: x - packet.ttl, [64, 128, 256])) - 1
else:
return self.ttl_max
