[tor-commits] [ooni-probe/master] Added tests to the sniffer subsystem

art at torproject.org art at torproject.org
Tue Aug 5 10:33:36 UTC 2014


commit 9b27ebd5ccd1b75e8f7fb2c5acc00d35da6bd70d
Author: kudrom <kudrom at riseup.net>
Date:   Sun Jul 20 20:09:24 2014 +0200

    Added tests to the sniffer subsystem
---
 ooni/director.py            |   21 ++++++--------
 ooni/reporter.py            |   13 +++------
 ooni/settings.py            |    2 +-
 ooni/tests/test_director.py |   54 +++++++++++++++++++++++++++++++++++
 ooni/tests/test_nettest.py  |    4 +--
 ooni/tests/test_oonicli.py  |   65 +++++++++++++++++++++++++++++++++++++------
 ooni/tests/test_reporter.py |    3 +-
 ooni/utils/__init__.py      |   29 ++++++++++++++++---
 8 files changed, 154 insertions(+), 37 deletions(-)

diff --git a/ooni/director.py b/ooni/director.py
index 3634115..d48c18f 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -1,14 +1,13 @@
 import os
-import otime
-from psutil import Process
 
 from ooni.managers import ReportEntryManager, MeasurementManager
 from ooni.reporter import Report
-from ooni.utils import log, pushFilenameStack
+from ooni.utils import log, generate_filename
 from ooni.utils.net import randomFreePort
 from ooni.nettest import NetTest, getNetTestInformation
 from ooni.settings import config
 from ooni import errors
+from ooni.nettest import test_class_name_to_name
 
 from txtorcon import TorConfig, TorState, launch_tor, build_tor_connection
 
@@ -191,7 +190,7 @@ class Director(object):
         self.totalMeasurementRuntime += measurement.runtime
         self.successfulMeasurements += 1
         measurement.result = result
-        test_name = measurement.testInstance.__class__.__name__
+        test_name = test_class_name_to_name(measurement.testInstance.name)
         sniffer = self.sniffers[test_name]
         config.scapyFactory.unRegisterProtocol(sniffer)
         sniffer.close()
@@ -264,16 +263,14 @@ class Director(object):
         """
         from ooni.utils.txscapy import ScapySniffer
 
-        test_name, start_time = testDetails['test_name'], testDetails['start_time']
-        start_time = otime.epochToTimestamp(start_time)
-        suffix = "%s-%s.%s" % (test_name, start_time, "pcap")
         if not config.reports.pcap:
-            filename_pcap= "%s-%s" % ("report", suffix)
+            prefix = 'report'
         else:
-            filename_pcap = "%s-%s" % (config.reports.pcap, suffix)
-
-        if len(self.sniffers) > 1:
-            pcap_filenames = set(sniffer.pcapwriter.filename for sniffer in self.sniffers)
+            prefix = config.reports.pcap
+        filename = config.global_options['reportfile'] if 'reportfile' in config.global_options.keys() else None
+        filename_pcap = generate_filename(testDetails, filename=filename, prefix=prefix, extension='pcap')
+        if len(self.sniffers) > 0:
+            pcap_filenames = set(sniffer.pcapwriter.filename for sniffer in self.sniffers.values())
             pcap_filenames.add(filename_pcap)
             log.msg("pcap files %s can be messed up because several netTests are being executed in parallel." %
                     ','.join(pcap_filenames))
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 44e6b96..5497ed5 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -30,7 +30,7 @@ except ImportError:
 from ooni import errors
 
 from ooni import otime
-from ooni.utils import pushFilenameStack
+from ooni.utils import pushFilenameStack, generate_filename
 from ooni.utils.net import BodyReceiver, StringProducer
 
 from ooni.settings import config
@@ -171,17 +171,13 @@ class YAMLReporter(OReporter):
 
     """
 
-    def __init__(self, test_details, report_destination='.',
-                 report_filename=None):
+    def __init__(self, test_details, report_destination='.', report_filename=None):
         self.reportDestination = report_destination
 
         if not os.path.isdir(report_destination):
             raise errors.InvalidDestination
 
-        if not report_filename:
-            report_filename = "report-" + \
-                              test_details['test_name'] + "-" + \
-                              otime.timestamp() + ".yamloo"
+        report_filename = generate_filename(test_details, filename=report_filename, prefix='report', extension='yamloo')
 
         report_path = os.path.join(self.reportDestination, report_filename)
 
@@ -553,8 +549,7 @@ class Report(object):
 
         self.report_log = OONIBReportLog()
 
-        self.yaml_reporter = YAMLReporter(test_details,
-                                          report_filename=report_filename)
+        self.yaml_reporter = YAMLReporter(test_details, report_filename=report_filename)
         self.report_filename = self.yaml_reporter.report_path
 
         self.oonib_reporter = None
diff --git a/ooni/settings.py b/ooni/settings.py
index 5bebc05..07a0039 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -5,7 +5,7 @@ import getpass
 
 from os.path import abspath, expanduser
 
-from ooni import otime, geoip
+from ooni import geoip
 from ooni.utils import Storage
 
 
diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py
index da81731..38e8481 100644
--- a/ooni/tests/test_director.py
+++ b/ooni/tests/test_director.py
@@ -1,3 +1,5 @@
+import time
+
 from mock import patch, MagicMock
 
 from ooni.settings import config
@@ -51,3 +53,55 @@ class TestDirector(ConfigTestCase):
             assert config.tor.control_port == 4242
 
         return director_start_tor()
+
+
+class TestStartSniffing(unittest.TestCase):
+    def setUp(self):
+        self.director = Director()
+        self.testDetails = {
+            'test_name': 'foo',
+            'start_time': time.time()
+        }
+
+        # Each NetTestCase has a name attribute
+        class FooTestCase(object):
+            name = 'foo'
+        self.FooTestCase = FooTestCase
+
+    def test_start_sniffing_once(self):
+        with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory:
+            with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
+                self.director.startSniffing(self.testDetails)
+                sniffer = mock_scapy_sniffer.return_value
+                mock_scapy_factory.registerProtocol.assert_called_once_with(sniffer)
+
+    def test_start_sniffing_twice(self):
+        with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory:
+            with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
+                sniffer = mock_scapy_sniffer.return_value
+                sniffer.pcapwriter.filename = 'foo1_filename'
+                self.director.startSniffing(self.testDetails)
+                self.assertEqual(len(self.director.sniffers), 1)
+
+            self.testDetails = {
+                'test_name': 'bar',
+                'start_time': time.time()
+            }
+            with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
+                sniffer = mock_scapy_sniffer.return_value
+                sniffer.pcapwriter.filename = 'foo2_filename'
+                self.director.startSniffing(self.testDetails)
+                self.assertEqual(len(self.director.sniffers), 2)
+
+    def test_measurement_succeeded(self):
+        with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory:
+            with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer:
+                self.director.startSniffing(self.testDetails)
+                self.assertEqual(len(self.director.sniffers), 1)
+                measurement = MagicMock()
+                measurement.testInstance = self.FooTestCase()
+                self.director.measurementSucceeded('awesome', measurement)
+                self.assertEqual(len(self.director.sniffers), 0)
+                sniffer = mock_scapy_sniffer.return_value
+                mock_scapy_factory.unRegisterProtocol.assert_called_once_with(sniffer)
+
diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py
index 7828c4b..5015c64 100644
--- a/ooni/tests/test_nettest.py
+++ b/ooni/tests/test_nettest.py
@@ -242,7 +242,7 @@ class TestNetTest(unittest.TestCase):
         ntl.checkOptions()
         director = Director()
 
-        self.filename = 'dummy_report.yaml'
+        self.filename = 'dummy_report.yamloo'
         d = director.startNetTest(ntl, self.filename)
 
         @d.addCallback
@@ -306,7 +306,7 @@ class TestNettestTimeout(ConfigTestCase):
         ntl.checkOptions()
         director = Director()
 
-        self.filename = 'dummy_report.yaml'
+        self.filename = 'dummy_report.yamloo'
         d = director.startNetTest(ntl, self.filename)
 
         @d.addCallback
diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index dd41c1d..3b37af0 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -2,15 +2,13 @@ import os
 import sys
 import yaml
 
-from twisted.internet import defer
-from twisted.trial import unittest
+from twisted.internet import defer, reactor
 
 from ooni.tests import is_internet_connected
 from ooni.tests.bases import ConfigTestCase
 from ooni.settings import config
 from ooni.oonicli import runWithDirector
 
-
 def verify_header(header):
     assert 'input_hashes' in header.keys()
     assert 'options' in header.keys()
@@ -26,6 +24,34 @@ def verify_header(header):
 def verify_entry(entry):
     assert 'input' in entry
 
+config_includepcap = """
+basic:
+    logfile: ~/.ooni/ooniprobe.log
+privacy:
+    includeip: false
+    includeasn: true
+    includecountry: true
+    includecity: false
+    includepcap: true
+reports:
+    pcap: null
+    collector: null
+advanced:
+    geoip_data_dir: /usr/share/ooni/geoip
+    debug: false
+    interface: auto
+    start_tor: true
+    measurement_timeout: 60
+    measurement_retries: 2
+    measurement_concurrency: 10
+    reporting_timeout: 80
+    reporting_retries: 3
+    reporting_concurrency: 15
+    data_dir: /usr/share/ooni
+    oonid_api_port: 8042
+tor:
+"""
+
 
 class TestRunDirector(ConfigTestCase):
     def setUp(self):
@@ -43,14 +69,18 @@ class TestRunDirector(ConfigTestCase):
         super(TestRunDirector, self).tearDown()
         if len(self.filenames) > 0:
             for filename in self.filenames:
-                os.remove(filename)
+                if os.path.exists(filename):
+                    os.remove(filename)
 
     @defer.inlineCallbacks
-    def run_helper(self, test_name, args, verify_function):
-        output_file = 'test_report.yaml'
+    def run_helper(self, test_name, nettest_args, verify_function, ooni_args=[]):
+        output_file = 'test_report.yamloo'
         self.filenames.append(output_file)
-        sys.argv = ['', '-n', '-o', output_file, test_name]
-        sys.argv.extend(args)
+        oldargv = sys.argv
+        sys.argv = ['']
+        sys.argv.extend(ooni_args)
+        sys.argv.extend(['-n', '-o', output_file, test_name])
+        sys.argv.extend(nettest_args)
         yield runWithDirector(False, False)
         with open(output_file) as f:
             entries = yaml.safe_load_all(f)
@@ -62,6 +92,7 @@ class TestRunDirector(ConfigTestCase):
         verify_header(header)
         verify_entry(first_entry)
         verify_function(first_entry)
+        sys.argv = oldargv
 
     @defer.inlineCallbacks
     def test_http_requests(self):
@@ -125,3 +156,21 @@ class TestRunDirector(ConfigTestCase):
         yield self.run_helper('manipulation/http_header_field_manipulation',
                               ['-b', 'http://64.9.225.221'],
                               verify_function)
+
+    @defer.inlineCallbacks
+    def test_sniffing_activated(self):
+        filename = 'test_report.pcap'
+        self.filenames.append(filename)
+        conf_file = 'fake_config.conf'
+        with open(conf_file, 'w') as cfg:
+            cfg.writelines(config_includepcap)
+        self.filenames.append(conf_file)
+
+        def verify_function(_):
+            assert os.path.exists(filename)
+            self.assertGreater(os.stat(filename).st_size, 0)
+        yield self.run_helper('blocking/http_requests',
+                              ['-f', 'example-input.txt'],
+                              verify_function, ooni_args=['-f', conf_file])
+
+        config.scapyFactory.connectionLost('')
diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py
index a499eb3..06659fd 100644
--- a/ooni/tests/test_reporter.py
+++ b/ooni/tests/test_reporter.py
@@ -25,7 +25,8 @@ test_details = {
     'software_name': 'spam',
     'software_version': '1.0',
     'input_hashes': [],
-    'probe_asn': 'AS0'
+    'probe_asn': 'AS0',
+    'start_time': time.time()
 }
 
 oonib_new_report_message = {
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 3f47bd1..767e6d2 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -1,12 +1,9 @@
-import logging
 import string
 import random
 import glob
-import yaml
-import imp
 import os
 
-from ooni import errors
+from ooni import errors, otime
 
 class Storage(dict):
     """
@@ -102,3 +99,27 @@ def pushFilenameStack(filename):
         new_filename = "%s.%s" % (c_filename, new_idx)
         os.rename(f, new_filename)
     os.rename(filename, filename+".1")
+
+
+def generate_filename(testDetails, prefix=None, extension=None, filename=None):
+    """
+    Returns a filename for every test execution.
+
+    It's used to assure that all files of a certain test have a common basename but different
+    extension.
+    """
+    if filename is None:
+        test_name, start_time = testDetails['test_name'], testDetails['start_time']
+        start_time = otime.epochToTimestamp(start_time)
+        suffix = "%s-%s" % (test_name, start_time)
+        basename = '%s-%s' % (prefix, suffix) if prefix is not None else suffix
+        final_filename = '%s.%s' % (basename, extension) if extension is not None else basename
+    else:
+        if extension is not None:
+            basename = filename.split('.')[0] if '.' in filename else filename
+            final_filename = '%s.%s' % (basename, extension)
+        else:
+            final_filename = filename
+
+    return final_filename
+





More information about the tor-commits mailing list