commit fcae18e1cc5b6387bf4b609d62deec477414cd92 Author: Arturo Filastò art@fuffa.org Date: Wed Aug 21 22:21:33 2013 +0200

Start implementing support for downloading of test decks

The work here was mainly merging together my branch with the one of @aagbsn and refactoring a lot of the code. --- ooni/deck.py | 83 +++++++++++---------------------------------------- ooni/nettest.py | 27 ++++++++++++++++- ooni/oonibclient.py | 36 ++++++++++++++-------- ooni/oonicli.py | 14 ++++----- ooni/utils/net.py | 19 ++++++++++++ 5 files changed, 93 insertions(+), 86 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py index 02c8fc1..f99916b 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -10,23 +10,6 @@ import os import re import yaml

-onionInputRegex = re.compile("httpo://[a-z0-9]{16}.onion/input/([a-z0-9]){40}$") - -@defer.inlineCallbacks -def downloadFile(fileURL, filePath): - - def writeFile(response_body, filePath): - f = open(filePath, 'w+') - f.write(response_body) - - finished = defer.Deferred() - finished.addCallback(writeFile, filePath) - - agent = Agent(reactor, sockshost="127.0.0.1", - socksport=int(config.tor.socks_port)) - response = yield agent.request("GET", fileURL) - response.deliverBody(BodyReceiver(finished)) - def verifyFile(filePath): # get the filename component of the file path digest = os.path.basename(filePath) @@ -35,10 +18,11 @@ def verifyFile(filePath): return sha1digest.hexdigest() == digest return False

-class TestDeck(object): - def __init__(self, deckFile=None): +class Deck(object): + def __init__(self, oonibclient, deckFile=None): self.netTestLoaders = [] self.inputs = [] + if deckFile: self.loadDeck(deckFile)

def loadDeck(self, deckFile): @@ -63,54 +47,21 @@ class TestDeck(object): @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): """ fetch and verify a single NetTest's inputs """ - for test_class, test_methods in net_test_loader.testCases: - if test_class.inputFile: - inputArg = test_class.inputFile[0] - inputFileURL = test_class.localOptions[inputArg] - - m = onionInputRegex.match(inputFileURL) - if m: - fileDigest = m.group(1) - else: - fileDigest = os.path.basename(inputFileURL) + for input_file in net_test_loader.inputFiles: + if 'url' in input_file: + oonib = OONIBClient(input_file['address'])

- cachedInputDir = os.path.join(config.advanced.data_dir, + cached_input_dir = os.path.join(config.advanced.data_dir, 'inputs') - cachedPath = os.path.join(cachedInputDir, fileDigest) - self.inputs.append(cachedPath) + cached_path = os.path.join(cached_input_dir, input_file['hash']) + self.inputs.append(cached_path)

- if os.path.exists(cachedPath) and verifyFile(cachedPath): - test_class.localOptions[inputArg] = cachedPath - continue - if m: - yield downloadFile(inputFileURL, cachedPath) - if verifyFile(cachedPath): - test_class.localOptions[inputArg] = cachedPath + if os.path.exists(cached_path) and verifyFile(cached_path): + test_class.localOptions[inputArg] = cached_path continue - - raise UnableToLoadDeckInput, cachedPath - -def test_verify_file_success(): - f = open('/dev/urandom') - r = f.read(1024*1024) - z = sha1(f).hexdigest() - f.close() - fn = '/tmp/%s' % z - f = open(fn) - f.write(r) - f.close() - verifyFile(fn) - os.unlink(fn) - -def test_verify_file_failure(): - pass -def test_load_deck_with_no_inputs(deck): - pass -def test_load_deck_with_cached_input(deckFile): - pass -def test_load_deck_with_evil_path(deckFile): - pass -def test_load_deck_with_download_success(deckFile): - pass -def test_load_deck_with_download_failure(deckFile): - pass + yield oonib.downloadInput(input_file['hash'], cached_path) + if verifyFile(cached_path): + test_class.localOptions[input_file['key']] = cached_path + continue + + raise UnableToLoadDeckInput, cached_path diff --git a/ooni/nettest.py b/ooni/nettest.py index 51c6074..4806bdd 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -173,8 +173,9 @@ class NetTestLoader(object): method_prefix = 'test'

def __init__(self, options, test_file=None, test_string=None): + self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}.onion)/input/([a-z0-9]){40}$") self.options = options - test_cases = None + self.testCases, test_cases = None, None

if test_file: test_cases = loadNetTestFile(test_file) @@ -183,6 +184,30 @@ class NetTestLoader(object):

if test_cases: self.setupTestCases(test_cases) + + @property + def inputFiles(self): + input_files = [] + if not self.testCases: + return input_files + + for test_class, test_methods in self.testCases: + if test_class.inputFile: + key = test_class.inputFile[0] + filename = test_class.localOptions[key] + input_file = { + 'id': key + } + m = self.onionInputRegex.match(filename) + if m: + input_file['url'] = filename + input_file['address'] = m.group(1) + input_file['hash'] = m.group(2) + else: + input_file['filename'] = filename + input_files.append(input_file) + + return input_files

@property def testDetails(self): diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 28e42a7..3f9ed56 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -17,27 +17,21 @@ class InputFile(object):

self._file = None

-class Deck(object): - pass - class OONIBClient(object): def __init__(self, address): self.address = address self.agent = Agent(reactor)

- def queryBackend(self, method, urn, query=None): + def _request(self, method, urn, genReceiver, bodyProducer=None): finished = defer.Deferred()

- bodyProducer = None - if query: - bodyProducer = StringProducer(json.dumps(query)) - uri = self.address + urn d = self.agent.request(method, uri, bodyProducer) + @d.addCallback - def cb(response): + def callback(response): content_length = response.headers.getRawHeaders('content-length') - response.deliverBody(BodyReceiver(finished, content_length, json.loads)) + response.deliverBody(genReceiver(finished, content_length))

@d.addErrback def eb(err): @@ -45,6 +39,26 @@ class OONIBClient(object):

return finished

+ def queryBackend(self, method, urn, query=None): + bodyProducer = None + if query: + bodyProducer = StringProducer(json.dumps(query), bodyProducer) + + def genReceiver(finished, content_length): + return BodyReceiver(finished, content_length, json.loads) + + return self._request(method, urn, genReceiver, bodyProducer) + + def download(self, urn, download_path): + + def genReceiver(finished, content_length): + return Downloader(download_path, finished, content_length) + + return self._request('GET', urn, genReceiver) + + def downloadInput(self, input_hash, download_path): + return self.download('/input/'+input_hash, download_path) + def getNettestPolicy(self): pass

@@ -53,5 +67,3 @@ class OONIBClient(object):

def getInputPolicy(self): pass - - diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 4c7dd8b..8708642 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -14,7 +14,7 @@ from ooni import errors

from ooni.settings import config from ooni.director import Director -from ooni.deck import TestDeck +from ooni.deck import Deck from ooni.reporter import YAMLReporter, OONIBReporter from ooni.nettest import NetTestLoader, MissingRequiredOption

@@ -125,19 +125,19 @@ def runWithDirector(): director = Director() d = director.start()

- test_deck = TestDeck() + deck = Deck() if global_options['no-collector']: log.msg("Not reporting using a collector") collector = global_options['collector'] = None

try: if global_options['testdeck']: - test_deck.loadDeck(global_options['testdeck']) + deck.loadDeck(global_options['testdeck']) else: log.debug("No test deck detected") net_test_loader = NetTestLoader(global_options['subargs'], test_file=global_options['test_file']) - test_deck.insert(net_test_loader) + deck.insert(net_test_loader) except MissingRequiredOption, option_name: log.err('Missing required option: "%s"' % option_name) print net_test_loader.usageOptions().getUsage() @@ -149,7 +149,7 @@ def runWithDirector():

def fetch_nettest_inputs(result): try: - test_deck.fetchAndVerifyDeckInputs() + deck.fetchAndVerifyDeckInputs() except errors.UnableToLoadDeckInput, e: return defer.failure.Failure(result)

@@ -174,8 +174,8 @@ def runWithDirector(): # Wait until director has started up (including bootstrapping Tor) # before adding tests def post_director_start(_): - for net_test_loader in test_deck.netTestLoaders: - # TestDecks can specify different collectors + for net_test_loader in deck.netTestLoaders: + # Decks can specify different collectors # for each net test, so that each NetTest # may be paired with a test_helper and its collector # However, a user can override this behavior by diff --git a/ooni/utils/net.py b/ooni/utils/net.py index b63a173..db2ea18 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -81,6 +81,25 @@ class BodyReceiver(protocol.Protocol): self.data = self.body_processor(self.data) self.finished.callback(self.data)

+class Downloader(protocol.Protocol): + def __init__(self, download_path, + finished, content_length=None): + self.finished = finished + self.bytes_remaining = content_length + self.fp = open(download_path, 'w+') + + def dataReceived(self, b): + self.fp.write(b) + if self.bytes_remaining: + if self.bytes_remaining == 0: + self.connectionLost(None) + else: + self.bytes_remaining -= len(b) + + def connectionLost(self, reason): + self.fp.close() + self.finished.callback(self.download_path) + def getSystemResolver(): """ XXX implement a function that returns the resolver that is currently