[tor-commits] [ooni-probe/master] Add support for looking up decks and deck descriptors in oonibclient
art at torproject.org
art at torproject.org
Thu Sep 5 22:14:23 UTC 2013
commit 0b62769ebed74e62882b06ee4579e78799df3d64
Author: Arturo Filastò <art at fuffa.org>
Date: Tue Aug 27 17:09:05 2013 +0200
Add support for looking up decks and deck descriptors in oonibclient
---
ooni/deck.py | 70 ++++++++++++++++++++++++++++++++++--
ooni/oonibclient.py | 99 +++++++++++++++++++++++----------------------------
2 files changed, 112 insertions(+), 57 deletions(-)
diff --git a/ooni/deck.py b/ooni/deck.py
index 23f4cb1..3b0716c 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -5,20 +5,82 @@ from ooni.settings import config
from ooni.utils import log
from ooni.utils.txagentwithsocks import Agent
from ooni import errors as e
-from ooni.oonibclient import OONIBClient
from twisted.internet import reactor, defer
import os
import re
import yaml
+import json
+from hashlib import sha256
-class Deck(object):
- def __init__(self, deckFile=None):
+class InputFile(object):
+ def __init__(self, input_hash):
+ self.id = input_hash
+ cached_input_dir = os.path.join(config.advanced.data_dir,
+ 'inputs')
+ cache_path = os.path.join(cached_input_dir, input_hash)
+ self.cached_file = cache_path
+ self.cached_descriptor = cache_path + '.desc'
+
+ @property
+ def descriptorCached(self):
+ if os.path.exists(self.cached_descriptor):
+ with open(self.cached_descriptor) as f:
+ descriptor = json.load(f)
+ self.load(descriptor)
+ return True
+ return False
+
+ @property
+ def fileCached(self):
+ if os.path.exists(self.cached_file):
+ try:
+ self.verify()
+ except AssertionError:
+ log.err("The input %s failed validation. Going to consider it not cached." % self.id)
+ return False
+ return True
+ return False
+
+ def save(self):
+ with open(self.cached_descriptor, 'w+') as f:
+ json.dump({
+ 'name': self.name,
+ 'id': self.id,
+ 'version': self.version,
+ 'author': self.author,
+ 'date': self.date,
+ 'description': self.description
+ }, f)
+
+ def load(self, descriptor):
+ self.name = descriptor['name']
+ self.version = descriptor['version']
+ self.author = descriptor['author']
+ self.date = descriptor['date']
+ self.description = descriptor['description']
+
+ def verify(self):
+ digest = os.path.basename(self.cached_file)
+ with open(self.cached_file) as f:
+ file_hash = sha256(f.read())
+ assert file_hash.hexdigest() == digest
+
+class Deck(InputFile):
+ def __init__(self, deck_hash=None, deckFile=None):
+ self.id = deck_hash
self.bouncer = None
self.netTestLoaders = []
self.inputs = []
self.testHelpers = {}
+
+ cached_deck_dir = os.path.join(config.advanced.data_dir,
+ 'decks')
+ cache_path = os.path.join(cached_deck_dir, deck_hash)
+ self.cached_file = cache_path
+ self.cached_descriptor = cache_path + '.desc'
+
if deckFile: self.loadDeck(deckFile)
def loadDeck(self, deckFile):
@@ -49,6 +111,7 @@ class Deck(object):
@defer.inlineCallbacks
def lookupTestHelpers(self):
+ from ooni.oonibclient import OONIBClient
oonibclient = OONIBClient(self.bouncer)
required_test_helpers = []
for net_test_loader in self.netTestLoaders:
@@ -73,6 +136,7 @@ class Deck(object):
@defer.inlineCallbacks
def fetchAndVerifyNetTestInput(self, net_test_loader):
""" fetch and verify a single NetTest's inputs """
+ from ooni.oonibclient import OONIBClient
log.debug("Fetching and verifying inputs")
for i in net_test_loader.inputFiles:
if 'url' in i:
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
index 9dada5f..24dc374 100644
--- a/ooni/oonibclient.py
+++ b/ooni/oonibclient.py
@@ -7,64 +7,12 @@ from twisted.internet import defer, reactor
from ooni.utils.txagentwithsocks import Agent
+from ooni.deck import Deck, InputFile
from ooni import errors as e
from ooni.settings import config
from ooni.utils import log
from ooni.utils.net import BodyReceiver, StringProducer, Downloader
-class InputFile(object):
- def __init__(self, input_hash):
- self.id = input_hash
- cached_input_dir = os.path.join(config.advanced.data_dir,
- 'inputs')
- cache_path = os.path.join(cached_input_dir, input_hash)
- self.cached_file = cache_path
- self.cached_descriptor = cache_path + '.desc'
-
- @property
- def descriptorCached(self):
- if os.path.exists(self.cached_descriptor):
- with open(self.cached_descriptor) as f:
- descriptor = json.load(f)
- self.load(descriptor)
- return True
- return False
-
- @property
- def fileCached(self):
- if os.path.exists(self.cached_file):
- try:
- self.verify()
- except AssertionError:
- log.err("The input %s failed validation. Going to consider it not cached." % self.id)
- return False
- return True
- return False
-
- def save(self):
- with open(self.cached_descriptor, 'w+') as f:
- json.dump({
- 'name': self.name,
- 'id': self.id,
- 'version': self.version,
- 'author': self.author,
- 'date': self.date,
- 'description': self.description
- }, f)
-
- def load(self, descriptor):
- self.name = descriptor['name']
- self.version = descriptor['version']
- self.author = descriptor['author']
- self.date = descriptor['date']
- self.description = descriptor['description']
-
- def verify(self):
- digest = os.path.basename(self.cached_file)
- with open(self.cached_file) as f:
- file_hash = sha256(f.read())
- assert file_hash.hexdigest() == digest
-
class Collector(object):
def __init__(self, address):
self.address = address
@@ -193,6 +141,50 @@ class OONIBClient(object):
def getNettestPolicy(self):
return self.queryBackend('GET', '/policy/nettest')
+ def getDeckList(self):
+ return self.queryBackend('GET', '/deck')
+
+ def getDeck(self, deck_hash):
+ deck = Deck(deck_hash)
+ if deck.descriptorCached:
+ return defer.succeed(deck)
+ else:
+ d = self.queryBackend('GET', '/deck/' + deck_hash)
+
+ @d.addCallback
+ def cb(descriptor):
+ deck.load(descriptor)
+ deck.save()
+ return deck
+
+ @d.addErrback
+ def err(err):
+ log.err("Failed to get descriptor for deck %s" % deck_hash)
+ print err
+ log.exception(err)
+
+ return d
+
+ def downloadDeck(self, deck_hash):
+ deck = Deck(deck_hash)
+ if deck.fileCached:
+ return defer.succeed(deck)
+ else:
+ d = self.download('/deck/'+deck_hash+'/file', deck.cached_file)
+
+ @d.addCallback
+ def cb(res):
+ deck.verify()
+ return deck
+
+ @d.addErrback
+ def err(err):
+ log.err("Failed to download the deck %s" % deck_hash)
+ print err
+ log.exception(err)
+
+ return d
+
@defer.inlineCallbacks
def lookupTestCollector(self, test_name):
try:
@@ -216,4 +208,3 @@ class OONIBClient(object):
raise e.CouldNotFindTestHelper
defer.returnValue(test_helper)
-
More information about the tor-commits
mailing list