commit b80a5e2e23483d5c55749341730a743c6930e5af Author: Arturo Filastò art@fuffa.org Date: Tue Aug 27 13:37:29 2013 +0200

Update ooniprobe to reflect changes in the backend specification

We now also support running test decks that have test helpers that can't reside on the same machine. --- ooni/deck.py | 39 ++++++++++++++++++++++----------------- ooni/errors.py | 11 ++++++++++- ooni/oonibclient.py | 17 ++++++++++++----- ooni/tests/test_oonibclient.py | 11 ++++++----- 4 files changed, 50 insertions(+), 28 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py index a8aee39..23f4cb1 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -41,29 +41,34 @@ class Deck(object): def setup(self): """ fetch and verify inputs for all NetTests in the deck """ if self.bouncer: + log.msg("Looking up test helpers...") + yield self.lookupTestHelpers() for net_test_loader in self.netTestLoaders: + log.msg("Fetching required net test inputs...") yield self.fetchAndVerifyNetTestInput(net_test_loader) - yield self.lookupTestHelper(net_test_loader) - yield self.lookupTestCollector(net_test_loader)

@defer.inlineCallbacks - def lookupTestHelper(self, net_test_loader): + def lookupTestHelpers(self): oonibclient = OONIBClient(self.bouncer) - for th in net_test_loader.requiredTestHelpers: - # {'name':'', 'option':'', 'test_class':''} - response = yield oonibclient.lookupTestHelper(th['name']) - th['test_class'].localOptions[th['option']] = response['test-helper'] - #XXX: collector is only set once! - if 'collector' in response.keys(): - net_test_loader.collector = response['collector'].encode('utf-8') + required_test_helpers = [] + for net_test_loader in self.netTestLoaders: + for th in net_test_loader.requiredTestHelpers: + # {'name':'', 'option':'', 'test_class':''} + required_test_helpers.append(th['name'])

- @defer.inlineCallbacks - def lookupTestCollector(self, net_test_loader): - oonibclient = OONIBClient(self.bouncer) - if net_test_loader.collector is None: - response = yield oonibclient.lookupTestCollector(net_test_loader.testName) - if 'collector' in response.keys(): - net_test_loader.collector = response['collector'].encode('utf-8') + response = yield oonibclient.lookupTestHelpers(required_test_helpers) + + for net_test_loader in self.netTestLoaders: + log.msg("Setting collector and test helpers for %s" % net_test_loader.testDetails['test_name']) + if not net_test_loader.requiredTestHelpers: + log.msg("Using the default collector: %s" % response['default']['collector']) + net_test_loader.collector = response['default']['collector'] + + for th in net_test_loader.requiredTestHelpers: + test_helper = response[th['name']] + log.msg("Using this helper: %s" % test_helper) + th['test_class'].localOptions[th['option']] = test_helper['address'] + net_test_loader.collector = test_helper['collector'].encode('utf-8')

@defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): diff --git a/ooni/errors.py b/ooni/errors.py index 200491e..b29b778 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -165,7 +165,10 @@ class NoMoreReporters(Exception): class TorNotRunning(Exception): pass

-class OONIBReportError(Exception): +class OONIBError(Exception): + pass + +class OONIBReportError(OONIBError): pass

class OONIBReportUpdateError(OONIBReportError): @@ -185,3 +188,9 @@ class CouldNotFindTestHelper(Exception):

class CouldNotFindTestCollector(Exception): pass + +def get_error(error_key): + if error_key == 'test-helpers-key-missing': + return CouldNotFindTestHelper + else: + return OONIBError diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 227c775..9dada5f 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -121,9 +121,16 @@ class OONIBClient(object): bodyProducer = None if query: bodyProducer = StringProducer(json.dumps(query)) - + def genReceiver(finished, content_length): - return BodyReceiver(finished, content_length, json.loads) + def process_response(s): + response = json.loads(s) + if 'error' in response: + print "Got this backend error message %s" % response + log.err("Got this backend error message %s" % response) + raise e.get_error(response['error']) + return response + return BodyReceiver(finished, content_length, process_response)

return self._request(method, urn, genReceiver, bodyProducer)

@@ -197,15 +204,15 @@ class OONIBClient(object): defer.returnValue(test_collector)

@defer.inlineCallbacks - def lookupTestHelper(self, test_helper_name): + def lookupTestHelpers(self, test_helper_names): try:

test_helper = yield self.queryBackend('POST', '/bouncer', - query={'test-helper': test_helper_name}) + query={'test-helpers': test_helper_names}) except Exception: raise e.CouldNotFindTestHelper

- if not test_helpers: + if not test_helper: raise e.CouldNotFindTestHelper

defer.returnValue(test_helper) diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 1a1c4bc..0f22786 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -64,14 +64,15 @@ class TestOONIBClient(unittest.TestCase):

def test_lookup_invalid_helpers(self): return self.failUnlessFailure( - self.oonibclient.lookupTestHelpers( - ['dns', 'http-return-json-headers', 'sdadsadsa'] - ), e.CouldNotFindTestHelper) + self.oonibclient.lookupTestHelpers([ + 'sdadsadsa', 'dns' + ]), e.CouldNotFindTestHelper)

@defer.inlineCallbacks def test_lookup_test_helpers(self): - helpers = yield self.oonibclient.lookupTestHelpers(['dns', 'http-return-json-headers']) - self.assertTrue(len(helpers) == 1) + required_helpers = [u'http-return-json-headers', u'dns'] + helpers = yield self.oonibclient.lookupTestHelpers(required_helpers) + self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default']))

@defer.inlineCallbacks def test_get_nettest_list(self):