[tor-commits] [ooni-probe/master] Start implementing support for downloading of test decks
art at torproject.org
art at torproject.org
Tue Aug 27 09:21:51 UTC 2013
commit fcae18e1cc5b6387bf4b609d62deec477414cd92
Author: Arturo Filastò <art at fuffa.org>
Date: Wed Aug 21 22:21:33 2013 +0200
Start implementing support for downloading of test decks
The work here was mainly merging together my branch with the one of @aagbsn and
refactoring a lot of the code.
---
ooni/deck.py | 83 +++++++++++----------------------------------------
ooni/nettest.py | 27 ++++++++++++++++-
ooni/oonibclient.py | 36 ++++++++++++++--------
ooni/oonicli.py | 14 ++++-----
ooni/utils/net.py | 19 ++++++++++++
5 files changed, 93 insertions(+), 86 deletions(-)
diff --git a/ooni/deck.py b/ooni/deck.py
index 02c8fc1..f99916b 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -10,23 +10,6 @@ import os
import re
import yaml
-onionInputRegex = re.compile("httpo://[a-z0-9]{16}\.onion/input/([a-z0-9]){40}$")
-
- at defer.inlineCallbacks
-def downloadFile(fileURL, filePath):
-
- def writeFile(response_body, filePath):
- f = open(filePath, 'w+')
- f.write(response_body)
-
- finished = defer.Deferred()
- finished.addCallback(writeFile, filePath)
-
- agent = Agent(reactor, sockshost="127.0.0.1",
- socksport=int(config.tor.socks_port))
- response = yield agent.request("GET", fileURL)
- response.deliverBody(BodyReceiver(finished))
-
def verifyFile(filePath):
# get the filename component of the file path
digest = os.path.basename(filePath)
@@ -35,10 +18,11 @@ def verifyFile(filePath):
return sha1digest.hexdigest() == digest
return False
-class TestDeck(object):
- def __init__(self, deckFile=None):
+class Deck(object):
+ def __init__(self, oonibclient, deckFile=None):
self.netTestLoaders = []
self.inputs = []
+
if deckFile: self.loadDeck(deckFile)
def loadDeck(self, deckFile):
@@ -63,54 +47,21 @@ class TestDeck(object):
@defer.inlineCallbacks
def fetchAndVerifyNetTestInput(self, net_test_loader):
""" fetch and verify a single NetTest's inputs """
- for test_class, test_methods in net_test_loader.testCases:
- if test_class.inputFile:
- inputArg = test_class.inputFile[0]
- inputFileURL = test_class.localOptions[inputArg]
-
- m = onionInputRegex.match(inputFileURL)
- if m:
- fileDigest = m.group(1)
- else:
- fileDigest = os.path.basename(inputFileURL)
+ for input_file in net_test_loader.inputFiles:
+ if 'url' in input_file:
+ oonib = OONIBClient(input_file['address'])
- cachedInputDir = os.path.join(config.advanced.data_dir,
+ cached_input_dir = os.path.join(config.advanced.data_dir,
'inputs')
- cachedPath = os.path.join(cachedInputDir, fileDigest)
- self.inputs.append(cachedPath)
+ cached_path = os.path.join(cached_input_dir, input_file['hash'])
+ self.inputs.append(cached_path)
- if os.path.exists(cachedPath) and verifyFile(cachedPath):
- test_class.localOptions[inputArg] = cachedPath
- continue
- if m:
- yield downloadFile(inputFileURL, cachedPath)
- if verifyFile(cachedPath):
- test_class.localOptions[inputArg] = cachedPath
+ if os.path.exists(cached_path) and verifyFile(cached_path):
+ test_class.localOptions[inputArg] = cached_path
continue
-
- raise UnableToLoadDeckInput, cachedPath
-
-def test_verify_file_success():
- f = open('/dev/urandom')
- r = f.read(1024*1024)
- z = sha1(f).hexdigest()
- f.close()
- fn = '/tmp/%s' % z
- f = open(fn)
- f.write(r)
- f.close()
- verifyFile(fn)
- os.unlink(fn)
-
-def test_verify_file_failure():
- pass
-def test_load_deck_with_no_inputs(deck):
- pass
-def test_load_deck_with_cached_input(deckFile):
- pass
-def test_load_deck_with_evil_path(deckFile):
- pass
-def test_load_deck_with_download_success(deckFile):
- pass
-def test_load_deck_with_download_failure(deckFile):
- pass
+ yield oonib.downloadInput(input_file['hash'], cached_path)
+ if verifyFile(cached_path):
+ test_class.localOptions[input_file['key']] = cached_path
+ continue
+
+ raise UnableToLoadDeckInput, cached_path
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 51c6074..4806bdd 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -173,8 +173,9 @@ class NetTestLoader(object):
method_prefix = 'test'
def __init__(self, options, test_file=None, test_string=None):
+ self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]){40}$")
self.options = options
- test_cases = None
+ self.testCases, test_cases = None, None
if test_file:
test_cases = loadNetTestFile(test_file)
@@ -183,6 +184,30 @@ class NetTestLoader(object):
if test_cases:
self.setupTestCases(test_cases)
+
+ @property
+ def inputFiles(self):
+ input_files = []
+ if not self.testCases:
+ return input_files
+
+ for test_class, test_methods in self.testCases:
+ if test_class.inputFile:
+ key = test_class.inputFile[0]
+ filename = test_class.localOptions[key]
+ input_file = {
+ 'id': key
+ }
+ m = self.onionInputRegex.match(filename)
+ if m:
+ input_file['url'] = filename
+ input_file['address'] = m.group(1)
+ input_file['hash'] = m.group(2)
+ else:
+ input_file['filename'] = filename
+ input_files.append(input_file)
+
+ return input_files
@property
def testDetails(self):
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
index 28e42a7..3f9ed56 100644
--- a/ooni/oonibclient.py
+++ b/ooni/oonibclient.py
@@ -17,27 +17,21 @@ class InputFile(object):
self._file = None
-class Deck(object):
- pass
-
class OONIBClient(object):
def __init__(self, address):
self.address = address
self.agent = Agent(reactor)
- def queryBackend(self, method, urn, query=None):
+ def _request(self, method, urn, genReceiver, bodyProducer=None):
finished = defer.Deferred()
- bodyProducer = None
- if query:
- bodyProducer = StringProducer(json.dumps(query))
-
uri = self.address + urn
d = self.agent.request(method, uri, bodyProducer)
+
@d.addCallback
- def cb(response):
+ def callback(response):
content_length = response.headers.getRawHeaders('content-length')
- response.deliverBody(BodyReceiver(finished, content_length, json.loads))
+ response.deliverBody(genReceiver(finished, content_length))
@d.addErrback
def eb(err):
@@ -45,6 +39,26 @@ class OONIBClient(object):
return finished
+ def queryBackend(self, method, urn, query=None):
+ bodyProducer = None
+ if query:
+ bodyProducer = StringProducer(json.dumps(query), bodyProducer)
+
+ def genReceiver(finished, content_length):
+ return BodyReceiver(finished, content_length, json.loads)
+
+ return self._request(method, urn, genReceiver, bodyProducer)
+
+ def download(self, urn, download_path):
+
+ def genReceiver(finished, content_length):
+ return Downloader(download_path, finished, content_length)
+
+ return self._request('GET', urn, genReceiver)
+
+ def downloadInput(self, input_hash, download_path):
+ return self.download('/input/'+input_hash, download_path)
+
def getNettestPolicy(self):
pass
@@ -53,5 +67,3 @@ class OONIBClient(object):
def getInputPolicy(self):
pass
-
-
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 4c7dd8b..8708642 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -14,7 +14,7 @@ from ooni import errors
from ooni.settings import config
from ooni.director import Director
-from ooni.deck import TestDeck
+from ooni.deck import Deck
from ooni.reporter import YAMLReporter, OONIBReporter
from ooni.nettest import NetTestLoader, MissingRequiredOption
@@ -125,19 +125,19 @@ def runWithDirector():
director = Director()
d = director.start()
- test_deck = TestDeck()
+ deck = Deck()
if global_options['no-collector']:
log.msg("Not reporting using a collector")
collector = global_options['collector'] = None
try:
if global_options['testdeck']:
- test_deck.loadDeck(global_options['testdeck'])
+ deck.loadDeck(global_options['testdeck'])
else:
log.debug("No test deck detected")
net_test_loader = NetTestLoader(global_options['subargs'],
test_file=global_options['test_file'])
- test_deck.insert(net_test_loader)
+ deck.insert(net_test_loader)
except MissingRequiredOption, option_name:
log.err('Missing required option: "%s"' % option_name)
print net_test_loader.usageOptions().getUsage()
@@ -149,7 +149,7 @@ def runWithDirector():
def fetch_nettest_inputs(result):
try:
- test_deck.fetchAndVerifyDeckInputs()
+ deck.fetchAndVerifyDeckInputs()
except errors.UnableToLoadDeckInput, e:
return defer.failure.Failure(result)
@@ -174,8 +174,8 @@ def runWithDirector():
# Wait until director has started up (including bootstrapping Tor)
# before adding tests
def post_director_start(_):
- for net_test_loader in test_deck.netTestLoaders:
- # TestDecks can specify different collectors
+ for net_test_loader in deck.netTestLoaders:
+ # Decks can specify different collectors
# for each net test, so that each NetTest
# may be paired with a test_helper and its collector
# However, a user can override this behavior by
diff --git a/ooni/utils/net.py b/ooni/utils/net.py
index b63a173..db2ea18 100644
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@ -81,6 +81,25 @@ class BodyReceiver(protocol.Protocol):
self.data = self.body_processor(self.data)
self.finished.callback(self.data)
+class Downloader(protocol.Protocol):
+ def __init__(self, download_path,
+ finished, content_length=None):
+ self.finished = finished
+ self.bytes_remaining = content_length
+ self.fp = open(download_path, 'w+')
+
+ def dataReceived(self, b):
+ self.fp.write(b)
+ if self.bytes_remaining:
+ if self.bytes_remaining == 0:
+ self.connectionLost(None)
+ else:
+ self.bytes_remaining -= len(b)
+
+ def connectionLost(self, reason):
+ self.fp.close()
+ self.finished.callback(self.download_path)
+
def getSystemResolver():
"""
XXX implement a function that returns the resolver that is currently
More information about the tor-commits
mailing list