commit fb8a90c6f2ec136eb477bbb2bf91d61456e66c80 Author: Arturo Filastò art@fuffa.org Date: Thu Aug 22 17:38:38 2013 +0200
Refactor verification of input file and implement caching --- ooni/deck.py | 28 +++--------- ooni/oonibclient.py | 93 ++++++++++++++++++++++++++++++++-------- ooni/tests/test_oonibclient.py | 2 +- 3 files changed, 83 insertions(+), 40 deletions(-)
diff --git a/ooni/deck.py b/ooni/deck.py index f99916b..ef1b58a 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -1,6 +1,5 @@ #-*- coding: utf-8 -*-
-from hashlib import sha1 from ooni.nettest import NetTestLoader from ooni.settings import config from ooni.utils.txagentwithsocks import Agent @@ -10,14 +9,6 @@ import os import re import yaml
-def verifyFile(filePath): - # get the filename component of the file path - digest = os.path.basename(filePath) - with open(filePath) as f: - sha1digest = sha1(f.read()) - return sha1digest.hexdigest() == digest - return False - class Deck(object): def __init__(self, oonibclient, deckFile=None): self.netTestLoaders = [] @@ -51,17 +42,10 @@ class Deck(object): if 'url' in input_file: oonib = OONIBClient(input_file['address'])
- cached_input_dir = os.path.join(config.advanced.data_dir, - 'inputs') - cached_path = os.path.join(cached_input_dir, input_file['hash']) - self.inputs.append(cached_path) - - if os.path.exists(cached_path) and verifyFile(cached_path): - test_class.localOptions[inputArg] = cached_path - continue - yield oonib.downloadInput(input_file['hash'], cached_path) - if verifyFile(cached_path): - test_class.localOptions[input_file['key']] = cached_path - continue + input_file = yield oonib.downloadInput(input_file['hash']) + try: + input_file.verify() + except AssertionError: + raise UnableToLoadDeckInput, cached_path
- raise UnableToLoadDeckInput, cached_path + test_class.localOptions[input_file['key']] = input_file.cached_file diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 4f555b2..9665ee1 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -1,30 +1,67 @@ -from hashlib import sha256 - +import os import json
+from hashlib import sha256 + from twisted.internet import defer, reactor from twisted.web.client import Agent
+from ooni.settings import config from ooni.utils import log from ooni.utils.net import BodyReceiver, StringProducer, Downloader
class InputFile(object): - def __init__(self, descriptor): - self.id = descriptor['id'] + def __init__(self, input_hash): + self.id = input_hash + cached_input_dir = os.path.join(config.advanced.data_dir, + 'inputs') + cache_path = os.path.join(cached_input_dir, input_hash) + self.cached_file = cache_path + self.cached_descriptor = cache_path + '.desc' + + @property + def descriptorCached(self): + if os.path.exists(self.cached_descriptor): + with open(self.cached_descriptor) as f: + descriptor = json.load(f) + self.load(descriptor) + return True + return False + + @property + def fileCached(self): + if os.path.exists(self.cached_file): + try: + self.verify() + except AssertionError: + log.err("The input %s failed validation. Going to consider it not cached." % self.id) + return False + return True + return False + + def save(self): + with open(self.cached_descriptor, 'w+') as f: + json.dump({ + 'name': self.name, + 'id': self.id, + 'version': self.version, + 'author': self.author, + 'date': self.date, + 'description': self.description + }, f) + + def load(self, descriptor): self.name = descriptor['name'] self.version = descriptor['version'] self.author = descriptor['author'] self.date = descriptor['date'] self.description = descriptor['description']
- self.file_path = None - def verify(self): - digest = os.path.basename(self.file_path) - with open(self.file_path) as f: + digest = os.path.basename(self.cached_file) + with open(self.cached_file) as f: file_hash = sha256(f.read()) - return file_hash.hexdigest() == digest - return False + assert file_hash.hexdigest() == digest
class OONIBClient(object): def __init__(self, address): @@ -73,25 +110,47 @@ class OONIBClient(object): pass
def getInput(self, input_hash): - try: - return defer.succeed(self.input_files[input_hash]) - except KeyError: + input_file = InputFile(input_hash) + if input_file.descriptorCached: + return defer.succeed(input_file) + else: d = self.queryBackend('GET', '/input/' + input_hash) + @d.addCallback def cb(descriptor): - self.input_files[input_hash] = InputFile(descriptor) - return self.input_files[input_hash] + input_file.load(descriptor) + input_file.save() + return input_file + @d.addErrback def err(err): log.err("Failed to get descriptor for input %s" % input_hash) log.exception(err) + return d
def getInputList(self): return self.queryBackend('GET', '/input')
- def downloadInput(self, input_hash, download_path): - return self.download('/input/'+input_hash+'/file', download_path) + def downloadInput(self, input_hash): + input_file = InputFile(input_hash) + + if input_file.fileCached: + return defer.succeed(input_file) + else: + d = self.download('/input/'+input_hash+'/file', input_file.cached_file) + + @d.addCallback + def cb(res): + input_file.verify() + return input_file + + @d.addErrback + def err(err): + log.err("Failed to download the input file %s" % input_hash) + log.exception(err) + + return d
def getInputPolicy(self): pass diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 02f9f8f..26e2f62 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -28,7 +28,7 @@ class TestOONIBClient(unittest.TestCase):
@defer.inlineCallbacks def test_download_input(self): - yield self.oonibclient.downloadInput(input_id, input_id) + yield self.oonibclient.downloadInput(input_id)
@defer.inlineCallbacks def test_get_deck_list(self):