commit 0b62769ebed74e62882b06ee4579e78799df3d64 Author: Arturo Filastò <art@fuffa.org> Date: Tue Aug 27 17:09:05 2013 +0200 Add support for looking up decks and deck descriptors in oonibclient --- ooni/deck.py | 70 ++++++++++++++++++++++++++++++++++-- ooni/oonibclient.py | 99 +++++++++++++++++++++++---------------------------- 2 files changed, 112 insertions(+), 57 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 23f4cb1..3b0716c 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -5,20 +5,82 @@ from ooni.settings import config from ooni.utils import log from ooni.utils.txagentwithsocks import Agent from ooni import errors as e -from ooni.oonibclient import OONIBClient from twisted.internet import reactor, defer import os import re import yaml +import json +from hashlib import sha256 -class Deck(object): - def __init__(self, deckFile=None): +class InputFile(object): + def __init__(self, input_hash): + self.id = input_hash + cached_input_dir = os.path.join(config.advanced.data_dir, + 'inputs') + cache_path = os.path.join(cached_input_dir, input_hash) + self.cached_file = cache_path + self.cached_descriptor = cache_path + '.desc' + + @property + def descriptorCached(self): + if os.path.exists(self.cached_descriptor): + with open(self.cached_descriptor) as f: + descriptor = json.load(f) + self.load(descriptor) + return True + return False + + @property + def fileCached(self): + if os.path.exists(self.cached_file): + try: + self.verify() + except AssertionError: + log.err("The input %s failed validation. Going to consider it not cached." % self.id) + return False + return True + return False + + def save(self): + with open(self.cached_descriptor, 'w+') as f: + json.dump({ + 'name': self.name, + 'id': self.id, + 'version': self.version, + 'author': self.author, + 'date': self.date, + 'description': self.description + }, f) + + def load(self, descriptor): + self.name = descriptor['name'] + self.version = descriptor['version'] + self.author = descriptor['author'] + self.date = descriptor['date'] + self.description = descriptor['description'] + + def verify(self): + digest = os.path.basename(self.cached_file) + with open(self.cached_file) as f: + file_hash = sha256(f.read()) + assert file_hash.hexdigest() == digest + +class Deck(InputFile): + def __init__(self, deck_hash=None, deckFile=None): + self.id = deck_hash self.bouncer = None self.netTestLoaders = [] self.inputs = [] self.testHelpers = {} + + cached_deck_dir = os.path.join(config.advanced.data_dir, + 'decks') + cache_path = os.path.join(cached_deck_dir, deck_hash) + self.cached_file = cache_path + self.cached_descriptor = cache_path + '.desc' + if deckFile: self.loadDeck(deckFile) def loadDeck(self, deckFile): @@ -49,6 +111,7 @@ class Deck(object): @defer.inlineCallbacks def lookupTestHelpers(self): + from ooni.oonibclient import OONIBClient oonibclient = OONIBClient(self.bouncer) required_test_helpers = [] for net_test_loader in self.netTestLoaders: @@ -73,6 +136,7 @@ class Deck(object): @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): """ fetch and verify a single NetTest's inputs """ + from ooni.oonibclient import OONIBClient log.debug("Fetching and verifying inputs") for i in net_test_loader.inputFiles: if 'url' in i: diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 9dada5f..24dc374 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -7,64 +7,12 @@ from twisted.internet import defer, reactor from ooni.utils.txagentwithsocks import Agent +from ooni.deck import Deck, InputFile from ooni import errors as e from ooni.settings import config from ooni.utils import log from ooni.utils.net import BodyReceiver, StringProducer, Downloader -class InputFile(object): - def __init__(self, input_hash): - self.id = input_hash - cached_input_dir = os.path.join(config.advanced.data_dir, - 'inputs') - cache_path = os.path.join(cached_input_dir, input_hash) - self.cached_file = cache_path - self.cached_descriptor = cache_path + '.desc' - - @property - def descriptorCached(self): - if os.path.exists(self.cached_descriptor): - with open(self.cached_descriptor) as f: - descriptor = json.load(f) - self.load(descriptor) - return True - return False - - @property - def fileCached(self): - if os.path.exists(self.cached_file): - try: - self.verify() - except AssertionError: - log.err("The input %s failed validation. Going to consider it not cached." % self.id) - return False - return True - return False - - def save(self): - with open(self.cached_descriptor, 'w+') as f: - json.dump({ - 'name': self.name, - 'id': self.id, - 'version': self.version, - 'author': self.author, - 'date': self.date, - 'description': self.description - }, f) - - def load(self, descriptor): - self.name = descriptor['name'] - self.version = descriptor['version'] - self.author = descriptor['author'] - self.date = descriptor['date'] - self.description = descriptor['description'] - - def verify(self): - digest = os.path.basename(self.cached_file) - with open(self.cached_file) as f: - file_hash = sha256(f.read()) - assert file_hash.hexdigest() == digest - class Collector(object): def __init__(self, address): self.address = address @@ -193,6 +141,50 @@ class OONIBClient(object): def getNettestPolicy(self): return self.queryBackend('GET', '/policy/nettest') + def getDeckList(self): + return self.queryBackend('GET', '/deck') + + def getDeck(self, deck_hash): + deck = Deck(deck_hash) + if deck.descriptorCached: + return defer.succeed(deck) + else: + d = self.queryBackend('GET', '/deck/' + deck_hash) + + @d.addCallback + def cb(descriptor): + deck.load(descriptor) + deck.save() + return deck + + @d.addErrback + def err(err): + log.err("Failed to get descriptor for deck %s" % deck_hash) + print err + log.exception(err) + + return d + + def downloadDeck(self, deck_hash): + deck = Deck(deck_hash) + if deck.fileCached: + return defer.succeed(deck) + else: + d = self.download('/deck/'+deck_hash+'/file', deck.cached_file) + + @d.addCallback + def cb(res): + deck.verify() + return deck + + @d.addErrback + def err(err): + log.err("Failed to download the deck %s" % deck_hash) + print err + log.exception(err) + + return d + @defer.inlineCallbacks def lookupTestCollector(self, test_name): try: @@ -216,4 +208,3 @@ class OONIBClient(object): raise e.CouldNotFindTestHelper defer.returnValue(test_helper) -