[tor-commits] [ooni-probe/master] Add support for looking up test helpers via a bouncer.

art at torproject.org art at torproject.org
Tue Aug 27 09:21:51 UTC 2013


commit 3bface3bd4fab9e87bdc8c440d70e07e19b8af28
Author: Arturo Filastò <art at fuffa.org>
Date:   Fri Aug 23 16:46:15 2013 +0200

    Add support for looking up test helpers via a bouncer.
---
 data/nettests/blocking/dnsconsistency.py           |    4 +-
 data/nettests/manipulation/dnsspoof.py             |    1 +
 .../manipulation/http_header_field_manipulation.py |    1 +
 data/nettests/manipulation/http_host.py            |    1 +
 .../manipulation/http_invalid_request_line.py      |    4 +-
 data/nettests/manipulation/traceroute.py           |    3 +-
 decks/basic.deck                                   |    2 +-
 ooni/deck.py                                       |   50 +++++++++++++++---
 ooni/errors.py                                     |    2 +
 ooni/nettest.py                                    |   28 ++++++++--
 ooni/oonibclient.py                                |   55 ++++++++++++++------
 ooni/oonicli.py                                    |   42 ++++++++++-----
 ooni/reporter.py                                   |    8 ++-
 ooni/tests/test_oonibclient.py                     |   22 ++++++++
 ooni/utils/net.py                                  |    9 ++--
 15 files changed, 183 insertions(+), 49 deletions(-)

diff --git a/data/nettests/blocking/dnsconsistency.py b/data/nettests/blocking/dnsconsistency.py
index 7b6e7b9..3c88cd2 100644
--- a/data/nettests/blocking/dnsconsistency.py
+++ b/data/nettests/blocking/dnsconsistency.py
@@ -38,12 +38,14 @@ class DNSConsistencyTest(dnst.DNSTest):
 
     name = "DNS Consistency"
     description = "DNS censorship detection test"
-    version = "0.5"
+    version = "0.6"
     authors = "Arturo Filastò, Isis Lovecruft"
     requirements = None
 
     inputFile = ['file', 'f', None,
                  'Input file of list of hostnames to attempt to resolve']
+    
+    requiredTestHelpers = {'backend': 'dns'}
 
     usageOptions = UsageOptions
     requiredOptions = ['backend', 'file']
diff --git a/data/nettests/manipulation/dnsspoof.py b/data/nettests/manipulation/dnsspoof.py
index 5c50c2f..c9120a4 100644
--- a/data/nettests/manipulation/dnsspoof.py
+++ b/data/nettests/manipulation/dnsspoof.py
@@ -22,6 +22,7 @@ class DNSSpoof(scapyt.ScapyTest):
 
     usageOptions = UsageOptions
 
+    requiredTestHelpers = {'backend': 'dns'}
     requiredOptions = ['hostname', 'resolver']
 
     def setUp(self):
diff --git a/data/nettests/manipulation/http_header_field_manipulation.py b/data/nettests/manipulation/http_header_field_manipulation.py
index 509f4ef..3423442 100644
--- a/data/nettests/manipulation/http_header_field_manipulation.py
+++ b/data/nettests/manipulation/http_header_field_manipulation.py
@@ -48,6 +48,7 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest):
     randomizeUA = False
     usageOptions = UsageOptions
 
+    requiredTestHelpers = {'backend': 'http-return-json-headers'}
     requiredOptions = ['backend']
 
     def get_headers(self):
diff --git a/data/nettests/manipulation/http_host.py b/data/nettests/manipulation/http_host.py
index 3f1e0c6..2ec517c 100644
--- a/data/nettests/manipulation/http_host.py
+++ b/data/nettests/manipulation/http_host.py
@@ -43,6 +43,7 @@ class HTTPHost(httpt.HTTPTest):
     inputFile = ['file', 'f', None,
             'List of hostnames to test for censorship']
 
+    requiredTestHelpers = {'backend': 'http-return-json-headers'}
     requiredOptions = ['backend']
 
     def test_filtering_prepend_newline_to_method(self):
diff --git a/data/nettests/manipulation/http_invalid_request_line.py b/data/nettests/manipulation/http_invalid_request_line.py
index 2482282..64dbcac 100644
--- a/data/nettests/manipulation/http_invalid_request_line.py
+++ b/data/nettests/manipulation/http_invalid_request_line.py
@@ -20,10 +20,12 @@ class HTTPInvalidRequestLine(tcpt.TCPTest):
     ascii letters or numbers ('XxXx' will be 4).
     """
     name = "HTTP Invalid Request Line"
-    version = "0.1.4"
+    version = "0.2"
     authors = "Arturo Filastò"
 
     usageOptions = UsageOptions
+
+    requiredTestHelpers = {'backend': 'tcp-echo'}
     requiredOptions = ['backend']
 
     def setUp(self):
diff --git a/data/nettests/manipulation/traceroute.py b/data/nettests/manipulation/traceroute.py
index 3f6f17b..2db1826 100644
--- a/data/nettests/manipulation/traceroute.py
+++ b/data/nettests/manipulation/traceroute.py
@@ -23,8 +23,9 @@ class UsageOptions(usage.Options):
 class TracerouteTest(scapyt.BaseScapyTest):
     name = "Multi Protocol Traceroute Test"
     author = "Arturo Filastò"
-    version = "0.1.1"
+    version = "0.2"
 
+    requiredTestHelpers = {'backend': 'traceroute'}
     usageOptions = UsageOptions
     dst_ports = [0, 22, 23, 53, 80, 123, 443, 8080, 65535]
 
diff --git a/decks/basic.deck b/decks/basic.deck
index c784cd5..84a8f9d 100644
--- a/decks/basic.deck
+++ b/decks/basic.deck
@@ -7,7 +7,7 @@
     pcapfile: null
     reportfile: null
     resume: 0
-    subargs: [-f, 'httpo://3mr5phzltfzqgh6y.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1']
+    subargs: [-f, 'httpo://amcq5ldf3vwg22ze.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1']
     test_file: data/nettests/blocking/http_requests.py
     testdeck: null
 - options:
diff --git a/ooni/deck.py b/ooni/deck.py
index 8f1335d..1ad8377 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -4,7 +4,7 @@ from ooni.nettest import NetTestLoader
 from ooni.settings import config
 from ooni.utils import log
 from ooni.utils.txagentwithsocks import Agent
-from ooni.errors import UnableToLoadDeckInput
+from ooni import errors as e
 from ooni.oonibclient import OONIBClient
 
 from twisted.internet import reactor, defer
@@ -14,9 +14,12 @@ import re
 import yaml
 
 class Deck(object):
-    def __init__(self, deckFile=None):
+    def __init__(self, bouncer, deckFile=None):
+        self.bouncer = bouncer
         self.netTestLoaders = []
         self.inputs = []
+        self.testHelpers = {}
+        self.collector = None
 
         if deckFile: self.loadDeck(deckFile)
 
@@ -31,14 +34,20 @@ class Deck(object):
     def insert(self, net_test_loader):
         """ Add a NetTestLoader to this test deck """
         net_test_loader.checkOptions()
-        self.fetchAndVerifyNetTestInput(net_test_loader)
         self.netTestLoaders.append(net_test_loader)
- 
+
+    def getRequiredTestHelpers(self):
+        for net_test_loader in self.netTestLoaders:
+            for test_helper in net_test_loader.requiredTestHelpers:
+                self.testHelpers[test_helper['name']] = None
+    
     @defer.inlineCallbacks
-    def fetchAndVerifyDeckInputs(self):
+    def setup(self):
         """ fetch and verify inputs for all NetTests in the deck """
         for net_test_loader in self.netTestLoaders:
             yield self.fetchAndVerifyNetTestInput(net_test_loader)
+        self.getRequiredTestHelpers()
+        yield self.lookupTestHelpers()
 
     @defer.inlineCallbacks
     def fetchAndVerifyNetTestInput(self, net_test_loader):
@@ -47,12 +56,37 @@ class Deck(object):
         for i in net_test_loader.inputFiles:
             if 'url' in i:
                 log.debug("Downloading %s" % i['url'])
-                oonib = OONIBClient(i['address'])
+                oonibclient = OONIBClient(i['address'])
+                
+                try:
+                    input_file = yield oonibclient.downloadInput(i['hash'])
+                except:
+                    raise e.UnableToLoadDeckInput
 
-                input_file = yield oonib.downloadInput(i['hash'])
                 try:
                     input_file.verify()
                 except AssertionError:
-                    raise UnableToLoadDeckInput, cached_path
+                    raise e.UnableToLoadDeckInput, cached_path
                 
                 i['test_class'].localOptions[i['key']] = input_file.cached_file
+
+    def setNettestOptions(self):
+        for net_test_loader in self.netTestLoaders:
+            for th in net_test_loader.requiredTestHelpers:
+                test_helper_address = self.testHelpers[th['name']]
+                th['test_class'].localOptions[th['option']] = test_helper_address
+                net_test_loader.collector = self.collector
+                log.debug("Using %s: %s" % (test_helper_address, self.collector)) 
+
+    @defer.inlineCallbacks
+    def lookupTestHelpers(self):
+        log.msg("Looking up test helpers: %s" % self.testHelpers.keys())
+        
+        required_test_helpers = self.testHelpers.keys()
+        if required_test_helpers: 
+            oonibclient = OONIBClient(self.bouncer)
+            test_helpers = yield oonibclient.lookupTestHelpers(required_test_helpers)
+            self.collector = test_helpers['collector']
+            for name in self.testHelpers.keys():
+                self.testHelpers[name] = test_helpers[name]
+            self.setNettestOptions()
diff --git a/ooni/errors.py b/ooni/errors.py
index 94a83a4..5ad1940 100644
--- a/ooni/errors.py
+++ b/ooni/errors.py
@@ -170,3 +170,5 @@ class OONIBTestDetailsLookupError(OONIBReportError):
 
 class UnableToLoadDeckInput(Exception):
     pass
+class CouldNotFindTestHelper(Exception):
+    pass
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 1699f19..2b170d0 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -172,6 +172,7 @@ def getNetTestInformation(net_test_file):
 
 class NetTestLoader(object):
     method_prefix = 'test'
+    collector = None
 
     def __init__(self, options, test_file=None, test_string=None):
         self.onionInputRegex =  re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]{64})$")
@@ -185,7 +186,22 @@ class NetTestLoader(object):
 
         if test_cases:
             self.setupTestCases(test_cases)
-    
+   
+    @property
+    def requiredTestHelpers(self):
+        required_test_helpers = []
+        if not self.testCases:
+            return required_test_helpers
+
+        for test_class, test_methods in self.testCases:
+            for option, name in test_class.requiredTestHelpers.items():
+                required_test_helpers.append({
+                    'name': name,
+                    'option': option,
+                    'test_class': test_class
+                })
+        return required_test_helpers
+
     @property
     def inputFiles(self):
         input_files = []
@@ -255,6 +271,10 @@ class NetTestLoader(object):
         if (client_geodata and not config.privacy.includecountry) \
                 or ('countrycode' not in client_geodata):
             client_geodata['countrycode'] = None
+        
+        input_file_hashes = []
+        for input_file in self.inputFiles:
+            input_file_hashes.append(input_file['hash'])
 
         test_details = {'start_time': time.time(),
             'probe_asn': client_geodata['asn'],
@@ -264,7 +284,8 @@ class NetTestLoader(object):
             'test_version': self.testVersion,
             'software_name': 'ooniprobe',
             'software_version': software_version,
-            'options': self.options
+            'options': self.options,
+            'input_hashes': input_file_hashes
         }
         return test_details
 
@@ -613,7 +634,8 @@ class NetTestCase(object):
     optParameters = None
     baseParameters = None
     baseFlags = None
-
+    
+    requiredTestHelpers = {}
     requiredOptions = []
     requiresRoot = False
 
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
index 5861ff4..9bdce95 100644
--- a/ooni/oonibclient.py
+++ b/ooni/oonibclient.py
@@ -7,6 +7,7 @@ from twisted.internet import defer, reactor
 
 from ooni.utils.txagentwithsocks import Agent
 
+from ooni import errors as e
 from ooni.settings import config
 from ooni.utils import log
 from ooni.utils.net import BodyReceiver, StringProducer, Downloader
@@ -64,35 +65,46 @@ class InputFile(object):
             file_hash = sha256(f.read())
             assert file_hash.hexdigest() == digest
 
-    def validate(self, policy):
-        """
-        Validate this input file against the specified input policy.
-        """
-        for input_file
-
 class Collector(object):
-    def __init__(self):
+    def __init__(self, address):
+        self.address = address
+
         self.nettest_policy = None
         self.input_policy = None
+    
+    @defer.inlineCallbacks
+    def loadPolicy(self):
+        # XXX implement caching of policies
+        oonibclient = OONIBClient(self.address)
+        log.msg("Looking up nettest policy for %s" % self.address)
+        self.nettest_policy = yield oonibclient.getNettestPolicy()
+        log.msg("Looking up input policy for %s" % self.address)
+        self.input_policy = yield oonibclient.getInputPolicy()
 
     def validateInput(self, input_hash):
-        pass
+        for i in self.input_policy:
+            if i['id'] == input_hash:
+                return True
+        return False
 
-    def validateNettest(self, nettest):
-        pass
+    def validateNettest(self, nettest_name):
+        for i in self.nettest_policy:
+            if nettest_name == i['name']:
+                return True
+        return False
 
 class OONIBClient(object):
     def __init__(self, address):
         self.address = address
         self.agent = Agent(reactor, sockshost="127.0.0.1", 
                            socksport=config.tor.socks_port)
-        self.input_files = {}
 
     def _request(self, method, urn, genReceiver, bodyProducer=None):
         finished = defer.Deferred()
 
         uri = self.address + urn
-        d = self.agent.request(method, uri, bodyProducer)
+        headers = {}
+        d = self.agent.request(method, uri, bodyProducer=bodyProducer)
 
         @d.addCallback
         def callback(response):
@@ -108,7 +120,7 @@ class OONIBClient(object):
     def queryBackend(self, method, urn, query=None):
         bodyProducer = None
         if query:
-            bodyProducer = StringProducer(json.dumps(query), bodyProducer)
+            bodyProducer = StringProducer(json.dumps(query))
     
         def genReceiver(finished, content_length):
             return BodyReceiver(finished, content_length, json.loads)
@@ -125,9 +137,6 @@ class OONIBClient(object):
     def getNettestPolicy(self):
         pass
 
-    def queryBouncer(self, requested_helpers):
-        pass
-
     def getInput(self, input_hash):
         input_file = InputFile(input_hash)
         if input_file.descriptorCached:
@@ -176,3 +185,17 @@ class OONIBClient(object):
 
     def getNettestPolicy(self):
         return self.queryBackend('GET', '/policy/nettest')
+
+    @defer.inlineCallbacks
+    def lookupTestHelpers(self, test_helper_names):
+        try:
+            test_helpers = yield self.queryBackend('POST', '/bouncer', 
+                            query={'test-helpers': test_helper_names})
+        except Exception:
+            raise e.CouldNotFindTestHelper
+
+        if not test_helpers:
+            raise e.CouldNotFindTestHelper
+
+        defer.returnValue(test_helpers)
+
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index ef6ef84..cf7d34d 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -36,8 +36,10 @@ class Options(usage.Options):
     optParameters = [["reportfile", "o", None, "report file name"],
                      ["testdeck", "i", None,
                          "Specify as input a test deck: a yaml file containig the tests to run an their arguments"],
-                     ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion',
-                         "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"],
+                     ["collector", "c", None,
+                         "Address of the collector of test results. This option should not be used, but you should always use a bouncer."],
+                     ["bouncer", "b", 'httpo://nkvphnp3p6agi5qq.onion',
+                         "Address of the bouncer for test helpers. default: httpo://nkvphnp3p6agi5qq.onion"],
                      ["logfile", "l", None, "log file name"],
                      ["pcapfile", "O", None, "pcap file name"],
                      ["parallelism", "p", "10", "input parallelism"],
@@ -125,7 +127,7 @@ def runWithDirector():
     director = Director()
     d = director.start()
 
-    deck = Deck()
+    deck = Deck(global_options['bouncer'])
     if global_options['no-collector']:
         log.msg("Not reporting using a collector")
         collector = global_options['collector'] = None
@@ -146,10 +148,10 @@ def runWithDirector():
         log.err(e)
         print net_test_loader.usageOptions().getUsage()
         sys.exit(2)
-
-    def fetch_nettest_inputs(result):
+    
+    def setup_nettest(_):
         try: 
-            return deck.fetchAndVerifyDeckInputs()
+            return deck.setup()
         except errors.UnableToLoadDeckInput, e:
             return defer.failure.Failure(result)
 
@@ -157,18 +159,30 @@ def runWithDirector():
         log.err("Failed to start the director")
         r = failure.trap(errors.TorNotRunning,
                 errors.InvalidOONIBCollectorAddress,
-                errors.UnableToLoadDeckInput)
-        if r == errors.TorNotRunning:
+                errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper)
+
+        if isinstance(failure.value, errors.TorNotRunning):
             log.err("Tor does not appear to be running")
             log.err("Reporting with the collector %s is not possible" %
                     global_options['collector'])
             log.msg("Try with a different collector or disable collector reporting with -n")
-        elif r == errors.InvalidOONIBCollectorAddress:
+
+        elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
             log.err("Invalid format for oonib collector address.")
             log.msg("Should be in the format http://<collector_address>:<port>")
             log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
-        elif r == errors.UnableToLoadDeckInput:
-            log.err('Missing required input files: %s' % failure)
+
+        elif isinstance(failure.value, errors.UnableToLoadDeckInput):
+            log.err("Unable to fetch the required inputs for the test deck.")
+            log.msg("Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues")
+
+        elif isinstance(failure.value, errors.CouldNotFindTestHelper):
+            log.err("Unable to obtain the required test helpers.")
+            log.msg("Try with a different bouncer or check that Tor is running properly.")
+
+        if config.advanced.debug:
+            log.exception(failure)
+
         reactor.stop()
 
     # Wait until director has started up (including bootstrapping Tor)
@@ -188,8 +202,8 @@ def runWithDirector():
             if not global_options['no-collector']:
                 if global_options['collector']:
                     collector = global_options['collector']
-                elif net_test_loader.options['collector']:
-                    collector = net_test_loader.options['collector']
+                elif net_test_loader.collector:
+                    collector = net_test_loader.collector
 
             if collector and collector.startswith('httpo:') \
                     and (not (config.tor_state or config.tor.socks_port)):
@@ -213,7 +227,7 @@ def runWithDirector():
         director.allTestsDone.addBoth(shutdown)
 
     def start():
-        d.addCallback(fetch_nettest_inputs)
+        d.addCallback(setup_nettest)
         d.addCallback(post_director_start)
         d.addErrback(director_startup_failed)
 
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 0c13cf2..8a89c8a 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -308,6 +308,7 @@ class OONIBReporter(OReporter):
             'probe_asn': self.testDetails['probe_asn'],
             'test_name': self.testDetails['test_name'],
             'test_version': self.testDetails['test_version'],
+            'input_hashes': self.testDetails['input_hashes'],
             # XXX there is a bunch of redundancy in the arguments getting sent
             # to the backend. This may need to get changed in the client and the
             # backend.
@@ -328,7 +329,6 @@ class OONIBReporter(OReporter):
                                 bodyProducer=bodyProducer)
         except ConnectionRefusedError:
             log.err("Connection to reporting backend failed (ConnectionRefusedError)")
-            #yield defer.fail(OONIBReportCreationError())
             raise errors.OONIBReportCreationError
 
         except errors.HostUnreachable:
@@ -353,6 +353,12 @@ class OONIBReporter(OReporter):
             log.err("Failed to parse collector response")
             log.exception(e)
             raise errors.OONIBReportCreationError
+        
+        if response.code == 400:
+            # XXX make this more strict
+            log.err("The specified input or nettests cannot be submitted to this collector.")
+            log.msg("Try running a different test or try reporting to a different collector.")
+            raise errors.OONIBReportCreationError
 
         self.reportID = parsed_response['report_id']
         self.backendVersion = parsed_response['backend_version']
diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py
index c039c65..1a1c4bc 100644
--- a/ooni/tests/test_oonibclient.py
+++ b/ooni/tests/test_oonibclient.py
@@ -1,10 +1,17 @@
+import os
+import shutil
 import socket
 
 from twisted.trial import unittest
 from twisted.internet import defer
 
+from ooni import errors as e
+from ooni.utils import log
+from ooni.settings import config
 from ooni.oonibclient import OONIBClient
 
+data_dir = '/tmp/testooni'
+config.advanced.data_dir = data_dir
 input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'
 
 class TestOONIBClient(unittest.TestCase):
@@ -15,6 +22,10 @@ class TestOONIBClient(unittest.TestCase):
         try:
             s.connect((host, port))
             s.shutdown(2)
+            try: shutil.rmtree(data_dir)
+            except: pass
+            os.mkdir(data_dir)
+            os.mkdir(os.path.join(data_dir, 'inputs'))
         except Exception as ex:
             self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)")
         self.oonibclient = OONIBClient('http://' + host + ':' + str(port))
@@ -51,6 +62,17 @@ class TestOONIBClient(unittest.TestCase):
     def test_download_deck(self):
         pass
 
+    def test_lookup_invalid_helpers(self):
+        return self.failUnlessFailure(
+                self.oonibclient.lookupTestHelpers(
+                    ['dns', 'http-return-json-headers', 'sdadsadsa']
+                ), e.CouldNotFindTestHelper)
+
+    @defer.inlineCallbacks
+    def test_lookup_test_helpers(self):
+        helpers = yield self.oonibclient.lookupTestHelpers(['dns', 'http-return-json-headers'])
+        self.assertTrue(len(helpers) == 1)
+
     @defer.inlineCallbacks
     def test_get_nettest_list(self):
         input_list = yield self.oonibclient.getInputList()
diff --git a/ooni/utils/net.py b/ooni/utils/net.py
index 1ec9608..845cf4c 100644
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@ -77,9 +77,12 @@ class BodyReceiver(protocol.Protocol):
                 self.bytes_remaining -= len(b)
 
     def connectionLost(self, reason):
-        if self.body_processor:
-            self.data = self.body_processor(self.data)
-        self.finished.callback(self.data)
+        try:
+            if self.body_processor:
+                self.data = self.body_processor(self.data)
+            self.finished.callback(self.data)
+        except Exception as exc:
+            self.finished.errback(exc)
 
 class Downloader(protocol.Protocol):
     def __init__(self,  download_path,





More information about the tor-commits mailing list