commit f62f822b926eab78153c3081ced38b1075b16e10 Author: Arturo Filastò art@fuffa.org Date: Thu Aug 22 18:10:10 2013 +0200

Implement downloading of input files based on URL in deck file --- ooni/deck.py | 21 ++++++++++++++------- ooni/nettest.py | 5 +++-- ooni/oonibclient.py | 8 +++++--- ooni/oonicli.py | 2 +- ooni/utils/net.py | 1 + 5 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py index ef1b58a..8f1335d 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -2,15 +2,19 @@

from ooni.nettest import NetTestLoader from ooni.settings import config +from ooni.utils import log from ooni.utils.txagentwithsocks import Agent from ooni.errors import UnableToLoadDeckInput +from ooni.oonibclient import OONIBClient + from twisted.internet import reactor, defer + import os import re import yaml

class Deck(object): - def __init__(self, oonibclient, deckFile=None): + def __init__(self, deckFile=None): self.netTestLoaders = [] self.inputs = []

@@ -30,22 +34,25 @@ class Deck(object): self.fetchAndVerifyNetTestInput(net_test_loader) self.netTestLoaders.append(net_test_loader)

+ @defer.inlineCallbacks def fetchAndVerifyDeckInputs(self): """ fetch and verify inputs for all NetTests in the deck """ for net_test_loader in self.netTestLoaders: - self.fetchAndVerifyNetTestInput(net_test_loader) + yield self.fetchAndVerifyNetTestInput(net_test_loader)

@defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): """ fetch and verify a single NetTest's inputs """ - for input_file in net_test_loader.inputFiles: - if 'url' in input_file: - oonib = OONIBClient(input_file['address']) + log.debug("Fetching and verifying inputs") + for i in net_test_loader.inputFiles: + if 'url' in i: + log.debug("Downloading %s" % i['url']) + oonib = OONIBClient(i['address'])

- input_file = yield oonib.downloadInput(input_file['hash']) + input_file = yield oonib.downloadInput(i['hash']) try: input_file.verify() except AssertionError: raise UnableToLoadDeckInput, cached_path

- test_class.localOptions[input_file['key']] = input_file.cached_file + i['test_class'].localOptions[i['key']] = input_file.cached_file diff --git a/ooni/nettest.py b/ooni/nettest.py index 4806bdd..b8cdf65 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -173,7 +173,7 @@ class NetTestLoader(object): method_prefix = 'test'

def __init__(self, options, test_file=None, test_string=None): - self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}.onion)/input/([a-z0-9]){40}$") + self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}.onion)/input/([a-z0-9]{64})$") self.options = options self.testCases, test_cases = None, None

@@ -196,7 +196,8 @@ class NetTestLoader(object): key = test_class.inputFile[0] filename = test_class.localOptions[key] input_file = { - 'id': key + 'key': key, + 'test_class': test_class } m = self.onionInputRegex.match(filename) if m: diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 9665ee1..9d2085b 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -4,7 +4,8 @@ import json from hashlib import sha256

from twisted.internet import defer, reactor -from twisted.web.client import Agent + +from ooni.utils.txagentwithsocks import Agent

from ooni.settings import config from ooni.utils import log @@ -66,7 +67,8 @@ class InputFile(object): class OONIBClient(object): def __init__(self, address): self.address = address - self.agent = Agent(reactor) + self.agent = Agent(reactor, sockshost="127.0.0.1", + socksport=config.tor.socks_port) self.input_files = {}

def _request(self, method, urn, genReceiver, bodyProducer=None): @@ -77,7 +79,7 @@ class OONIBClient(object):

@d.addCallback def callback(response): - content_length = response.headers.getRawHeaders('content-length') + content_length = int(response.headers.getRawHeaders('content-length')[0]) response.deliverBody(genReceiver(finished, content_length))

@d.addErrback diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 8708642..a7df18f 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -149,7 +149,7 @@ def runWithDirector():

def fetch_nettest_inputs(result): try: - deck.fetchAndVerifyDeckInputs() + return deck.fetchAndVerifyDeckInputs() except errors.UnableToLoadDeckInput, e: return defer.failure.Failure(result)

diff --git a/ooni/utils/net.py b/ooni/utils/net.py index ab33d00..1ec9608 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -97,6 +97,7 @@ class Downloader(protocol.Protocol): self.bytes_remaining -= len(b)

def connectionLost(self, reason): + self.fp.flush() self.fp.close() self.finished.callback(None)