[tor-commits] [ooni-probe/master] Update ooniprobe to reflect changes in the backend specification
art at torproject.org
art at torproject.org
Thu Sep 5 22:14:23 UTC 2013
commit b80a5e2e23483d5c55749341730a743c6930e5af
Author: Arturo Filastò <art at fuffa.org>
Date: Tue Aug 27 13:37:29 2013 +0200
Update ooniprobe to reflect changes in the backend specification
We now also support running test decks that have test helpers that can't reside on the same machine.
---
ooni/deck.py | 39 ++++++++++++++++++++++-----------------
ooni/errors.py | 11 ++++++++++-
ooni/oonibclient.py | 17 ++++++++++++-----
ooni/tests/test_oonibclient.py | 11 ++++++-----
4 files changed, 50 insertions(+), 28 deletions(-)
diff --git a/ooni/deck.py b/ooni/deck.py
index a8aee39..23f4cb1 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -41,29 +41,34 @@ class Deck(object):
def setup(self):
""" fetch and verify inputs for all NetTests in the deck """
if self.bouncer:
+ log.msg("Looking up test helpers...")
+ yield self.lookupTestHelpers()
for net_test_loader in self.netTestLoaders:
+ log.msg("Fetching required net test inputs...")
yield self.fetchAndVerifyNetTestInput(net_test_loader)
- yield self.lookupTestHelper(net_test_loader)
- yield self.lookupTestCollector(net_test_loader)
@defer.inlineCallbacks
- def lookupTestHelper(self, net_test_loader):
+ def lookupTestHelpers(self):
oonibclient = OONIBClient(self.bouncer)
- for th in net_test_loader.requiredTestHelpers:
- # {'name':'', 'option':'', 'test_class':''}
- response = yield oonibclient.lookupTestHelper(th['name'])
- th['test_class'].localOptions[th['option']] = response['test-helper']
- #XXX: collector is only set once!
- if 'collector' in response.keys():
- net_test_loader.collector = response['collector'].encode('utf-8')
+ required_test_helpers = []
+ for net_test_loader in self.netTestLoaders:
+ for th in net_test_loader.requiredTestHelpers:
+ # {'name':'', 'option':'', 'test_class':''}
+ required_test_helpers.append(th['name'])
- @defer.inlineCallbacks
- def lookupTestCollector(self, net_test_loader):
- oonibclient = OONIBClient(self.bouncer)
- if net_test_loader.collector is None:
- response = yield oonibclient.lookupTestCollector(net_test_loader.testName)
- if 'collector' in response.keys():
- net_test_loader.collector = response['collector'].encode('utf-8')
+ response = yield oonibclient.lookupTestHelpers(required_test_helpers)
+
+ for net_test_loader in self.netTestLoaders:
+ log.msg("Setting collector and test helpers for %s" % net_test_loader.testDetails['test_name'])
+ if not net_test_loader.requiredTestHelpers:
+ log.msg("Using the default collector: %s" % response['default']['collector'])
+ net_test_loader.collector = response['default']['collector']
+
+ for th in net_test_loader.requiredTestHelpers:
+ test_helper = response[th['name']]
+ log.msg("Using this helper: %s" % test_helper)
+ th['test_class'].localOptions[th['option']] = test_helper['address']
+ net_test_loader.collector = test_helper['collector'].encode('utf-8')
@defer.inlineCallbacks
def fetchAndVerifyNetTestInput(self, net_test_loader):
diff --git a/ooni/errors.py b/ooni/errors.py
index 200491e..b29b778 100644
--- a/ooni/errors.py
+++ b/ooni/errors.py
@@ -165,7 +165,10 @@ class NoMoreReporters(Exception):
class TorNotRunning(Exception):
pass
-class OONIBReportError(Exception):
+class OONIBError(Exception):
+ pass
+
+class OONIBReportError(OONIBError):
pass
class OONIBReportUpdateError(OONIBReportError):
@@ -185,3 +188,9 @@ class CouldNotFindTestHelper(Exception):
class CouldNotFindTestCollector(Exception):
pass
+
+def get_error(error_key):
+ if error_key == 'test-helpers-key-missing':
+ return CouldNotFindTestHelper
+ else:
+ return OONIBError
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
index 227c775..9dada5f 100644
--- a/ooni/oonibclient.py
+++ b/ooni/oonibclient.py
@@ -121,9 +121,16 @@ class OONIBClient(object):
bodyProducer = None
if query:
bodyProducer = StringProducer(json.dumps(query))
-
+
def genReceiver(finished, content_length):
- return BodyReceiver(finished, content_length, json.loads)
+ def process_response(s):
+ response = json.loads(s)
+ if 'error' in response:
+ print "Got this backend error message %s" % response
+ log.err("Got this backend error message %s" % response)
+ raise e.get_error(response['error'])
+ return response
+ return BodyReceiver(finished, content_length, process_response)
return self._request(method, urn, genReceiver, bodyProducer)
@@ -197,15 +204,15 @@ class OONIBClient(object):
defer.returnValue(test_collector)
@defer.inlineCallbacks
- def lookupTestHelper(self, test_helper_name):
+ def lookupTestHelpers(self, test_helper_names):
try:
test_helper = yield self.queryBackend('POST', '/bouncer',
- query={'test-helper': test_helper_name})
+ query={'test-helpers': test_helper_names})
except Exception:
raise e.CouldNotFindTestHelper
- if not test_helpers:
+ if not test_helper:
raise e.CouldNotFindTestHelper
defer.returnValue(test_helper)
diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py
index 1a1c4bc..0f22786 100644
--- a/ooni/tests/test_oonibclient.py
+++ b/ooni/tests/test_oonibclient.py
@@ -64,14 +64,15 @@ class TestOONIBClient(unittest.TestCase):
def test_lookup_invalid_helpers(self):
return self.failUnlessFailure(
- self.oonibclient.lookupTestHelpers(
- ['dns', 'http-return-json-headers', 'sdadsadsa']
- ), e.CouldNotFindTestHelper)
+ self.oonibclient.lookupTestHelpers([
+ 'sdadsadsa', 'dns'
+ ]), e.CouldNotFindTestHelper)
@defer.inlineCallbacks
def test_lookup_test_helpers(self):
- helpers = yield self.oonibclient.lookupTestHelpers(['dns', 'http-return-json-headers'])
- self.assertTrue(len(helpers) == 1)
+ required_helpers = [u'http-return-json-headers', u'dns']
+ helpers = yield self.oonibclient.lookupTestHelpers(required_helpers)
+ self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default']))
@defer.inlineCallbacks
def test_get_nettest_list(self):
More information about the tor-commits
mailing list