commit 9b27ebd5ccd1b75e8f7fb2c5acc00d35da6bd70d Author: kudrom kudrom@riseup.net Date: Sun Jul 20 20:09:24 2014 +0200

Added tests to the sniffer subsystem --- ooni/director.py | 21 ++++++-------- ooni/reporter.py | 13 +++------ ooni/settings.py | 2 +- ooni/tests/test_director.py | 54 +++++++++++++++++++++++++++++++++++ ooni/tests/test_nettest.py | 4 +-- ooni/tests/test_oonicli.py | 65 +++++++++++++++++++++++++++++++++++++------ ooni/tests/test_reporter.py | 3 +- ooni/utils/__init__.py | 29 ++++++++++++++++--- 8 files changed, 154 insertions(+), 37 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py index 3634115..d48c18f 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -1,14 +1,13 @@ import os -import otime -from psutil import Process

from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report -from ooni.utils import log, pushFilenameStack +from ooni.utils import log, generate_filename from ooni.utils.net import randomFreePort from ooni.nettest import NetTest, getNetTestInformation from ooni.settings import config from ooni import errors +from ooni.nettest import test_class_name_to_name

from txtorcon import TorConfig, TorState, launch_tor, build_tor_connection

@@ -191,7 +190,7 @@ class Director(object): self.totalMeasurementRuntime += measurement.runtime self.successfulMeasurements += 1 measurement.result = result - test_name = measurement.testInstance.__class__.__name__ + test_name = test_class_name_to_name(measurement.testInstance.name) sniffer = self.sniffers[test_name] config.scapyFactory.unRegisterProtocol(sniffer) sniffer.close() @@ -264,16 +263,14 @@ class Director(object): """ from ooni.utils.txscapy import ScapySniffer

- test_name, start_time = testDetails['test_name'], testDetails['start_time'] - start_time = otime.epochToTimestamp(start_time) - suffix = "%s-%s.%s" % (test_name, start_time, "pcap") if not config.reports.pcap: - filename_pcap= "%s-%s" % ("report", suffix) + prefix = 'report' else: - filename_pcap = "%s-%s" % (config.reports.pcap, suffix) - - if len(self.sniffers) > 1: - pcap_filenames = set(sniffer.pcapwriter.filename for sniffer in self.sniffers) + prefix = config.reports.pcap + filename = config.global_options['reportfile'] if 'reportfile' in config.global_options.keys() else None + filename_pcap = generate_filename(testDetails, filename=filename, prefix=prefix, extension='pcap') + if len(self.sniffers) > 0: + pcap_filenames = set(sniffer.pcapwriter.filename for sniffer in self.sniffers.values()) pcap_filenames.add(filename_pcap) log.msg("pcap files %s can be messed up because several netTests are being executed in parallel." % ','.join(pcap_filenames)) diff --git a/ooni/reporter.py b/ooni/reporter.py index 44e6b96..5497ed5 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -30,7 +30,7 @@ except ImportError: from ooni import errors

from ooni import otime -from ooni.utils import pushFilenameStack +from ooni.utils import pushFilenameStack, generate_filename from ooni.utils.net import BodyReceiver, StringProducer

from ooni.settings import config @@ -171,17 +171,13 @@ class YAMLReporter(OReporter):

"""

- def __init__(self, test_details, report_destination='.', - report_filename=None): + def __init__(self, test_details, report_destination='.', report_filename=None): self.reportDestination = report_destination

if not os.path.isdir(report_destination): raise errors.InvalidDestination

- if not report_filename: - report_filename = "report-" + \ - test_details['test_name'] + "-" + \ - otime.timestamp() + ".yamloo" + report_filename = generate_filename(test_details, filename=report_filename, prefix='report', extension='yamloo')

report_path = os.path.join(self.reportDestination, report_filename)

@@ -553,8 +549,7 @@ class Report(object):

self.report_log = OONIBReportLog()

- self.yaml_reporter = YAMLReporter(test_details, - report_filename=report_filename) + self.yaml_reporter = YAMLReporter(test_details, report_filename=report_filename) self.report_filename = self.yaml_reporter.report_path

self.oonib_reporter = None diff --git a/ooni/settings.py b/ooni/settings.py index 5bebc05..07a0039 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -5,7 +5,7 @@ import getpass

from os.path import abspath, expanduser

-from ooni import otime, geoip +from ooni import geoip from ooni.utils import Storage

diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index da81731..38e8481 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -1,3 +1,5 @@ +import time + from mock import patch, MagicMock

from ooni.settings import config @@ -51,3 +53,55 @@ class TestDirector(ConfigTestCase): assert config.tor.control_port == 4242

return director_start_tor() + + +class TestStartSniffing(unittest.TestCase): + def setUp(self): + self.director = Director() + self.testDetails = { + 'test_name': 'foo', + 'start_time': time.time() + } + + # Each NetTestCase has a name attribute + class FooTestCase(object): + name = 'foo' + self.FooTestCase = FooTestCase + + def test_start_sniffing_once(self): + with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory: + with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: + self.director.startSniffing(self.testDetails) + sniffer = mock_scapy_sniffer.return_value + mock_scapy_factory.registerProtocol.assert_called_once_with(sniffer) + + def test_start_sniffing_twice(self): + with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory: + with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: + sniffer = mock_scapy_sniffer.return_value + sniffer.pcapwriter.filename = 'foo1_filename' + self.director.startSniffing(self.testDetails) + self.assertEqual(len(self.director.sniffers), 1) + + self.testDetails = { + 'test_name': 'bar', + 'start_time': time.time() + } + with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: + sniffer = mock_scapy_sniffer.return_value + sniffer.pcapwriter.filename = 'foo2_filename' + self.director.startSniffing(self.testDetails) + self.assertEqual(len(self.director.sniffers), 2) + + def test_measurement_succeeded(self): + with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory: + with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: + self.director.startSniffing(self.testDetails) + self.assertEqual(len(self.director.sniffers), 1) + measurement = MagicMock() + measurement.testInstance = self.FooTestCase() + self.director.measurementSucceeded('awesome', measurement) + self.assertEqual(len(self.director.sniffers), 0) + sniffer = mock_scapy_sniffer.return_value + mock_scapy_factory.unRegisterProtocol.assert_called_once_with(sniffer) + diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 7828c4b..5015c64 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -242,7 +242,7 @@ class TestNetTest(unittest.TestCase): ntl.checkOptions() director = Director()

- self.filename = 'dummy_report.yaml' + self.filename = 'dummy_report.yamloo' d = director.startNetTest(ntl, self.filename)

@d.addCallback @@ -306,7 +306,7 @@ class TestNettestTimeout(ConfigTestCase): ntl.checkOptions() director = Director()

- self.filename = 'dummy_report.yaml' + self.filename = 'dummy_report.yamloo' d = director.startNetTest(ntl, self.filename)

@d.addCallback diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index dd41c1d..3b37af0 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -2,15 +2,13 @@ import os import sys import yaml

-from twisted.internet import defer -from twisted.trial import unittest +from twisted.internet import defer, reactor

from ooni.tests import is_internet_connected from ooni.tests.bases import ConfigTestCase from ooni.settings import config from ooni.oonicli import runWithDirector

- def verify_header(header): assert 'input_hashes' in header.keys() assert 'options' in header.keys() @@ -26,6 +24,34 @@ def verify_header(header): def verify_entry(entry): assert 'input' in entry

+config_includepcap = """ +basic: + logfile: ~/.ooni/ooniprobe.log +privacy: + includeip: false + includeasn: true + includecountry: true + includecity: false + includepcap: true +reports: + pcap: null + collector: null +advanced: + geoip_data_dir: /usr/share/ooni/geoip + debug: false + interface: auto + start_tor: true + measurement_timeout: 60 + measurement_retries: 2 + measurement_concurrency: 10 + reporting_timeout: 80 + reporting_retries: 3 + reporting_concurrency: 15 + data_dir: /usr/share/ooni + oonid_api_port: 8042 +tor: +""" +

class TestRunDirector(ConfigTestCase): def setUp(self): @@ -43,14 +69,18 @@ class TestRunDirector(ConfigTestCase): super(TestRunDirector, self).tearDown() if len(self.filenames) > 0: for filename in self.filenames: - os.remove(filename) + if os.path.exists(filename): + os.remove(filename)

@defer.inlineCallbacks - def run_helper(self, test_name, args, verify_function): - output_file = 'test_report.yaml' + def run_helper(self, test_name, nettest_args, verify_function, ooni_args=[]): + output_file = 'test_report.yamloo' self.filenames.append(output_file) - sys.argv = ['', '-n', '-o', output_file, test_name] - sys.argv.extend(args) + oldargv = sys.argv + sys.argv = [''] + sys.argv.extend(ooni_args) + sys.argv.extend(['-n', '-o', output_file, test_name]) + sys.argv.extend(nettest_args) yield runWithDirector(False, False) with open(output_file) as f: entries = yaml.safe_load_all(f) @@ -62,6 +92,7 @@ class TestRunDirector(ConfigTestCase): verify_header(header) verify_entry(first_entry) verify_function(first_entry) + sys.argv = oldargv

@defer.inlineCallbacks def test_http_requests(self): @@ -125,3 +156,21 @@ class TestRunDirector(ConfigTestCase): yield self.run_helper('manipulation/http_header_field_manipulation', ['-b', 'http://64.9.225.221'], verify_function) + + @defer.inlineCallbacks + def test_sniffing_activated(self): + filename = 'test_report.pcap' + self.filenames.append(filename) + conf_file = 'fake_config.conf' + with open(conf_file, 'w') as cfg: + cfg.writelines(config_includepcap) + self.filenames.append(conf_file) + + def verify_function(_): + assert os.path.exists(filename) + self.assertGreater(os.stat(filename).st_size, 0) + yield self.run_helper('blocking/http_requests', + ['-f', 'example-input.txt'], + verify_function, ooni_args=['-f', conf_file]) + + config.scapyFactory.connectionLost('') diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py index a499eb3..06659fd 100644 --- a/ooni/tests/test_reporter.py +++ b/ooni/tests/test_reporter.py @@ -25,7 +25,8 @@ test_details = { 'software_name': 'spam', 'software_version': '1.0', 'input_hashes': [], - 'probe_asn': 'AS0' + 'probe_asn': 'AS0', + 'start_time': time.time() }

oonib_new_report_message = { diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 3f47bd1..767e6d2 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -1,12 +1,9 @@ -import logging import string import random import glob -import yaml -import imp import os

-from ooni import errors +from ooni import errors, otime

class Storage(dict): """ @@ -102,3 +99,27 @@ def pushFilenameStack(filename): new_filename = "%s.%s" % (c_filename, new_idx) os.rename(f, new_filename) os.rename(filename, filename+".1") + + +def generate_filename(testDetails, prefix=None, extension=None, filename=None): + """ + Returns a filename for every test execution. + + It's used to assure that all files of a certain test have a common basename but different + extension. + """ + if filename is None: + test_name, start_time = testDetails['test_name'], testDetails['start_time'] + start_time = otime.epochToTimestamp(start_time) + suffix = "%s-%s" % (test_name, start_time) + basename = '%s-%s' % (prefix, suffix) if prefix is not None else suffix + final_filename = '%s.%s' % (basename, extension) if extension is not None else basename + else: + if extension is not None: + basename = filename.split('.')[0] if '.' in filename else filename + final_filename = '%s.%s' % (basename, extension) + else: + final_filename = filename + + return final_filename +