August 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Make the return value of getInputProcessor be a deferred.
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 55c0c68b5097fe7e26ef692cee845ecb45f40cef Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jul 25 13:29:02 2013 +0200 Make the return value of getInputProcessor be a deferred. --- ooni/nettest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 92c9864..f940ae9 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -499,7 +499,7 @@ class NetTest(object): for test_class, test_methods in self.testCases: # load the input processor as late as possible - inputs = test_class().getInputProcessor() + inputs = yield defer.maybeDeferred(test_class().getInputProcessor()) if not inputs: inputs = [None] test_class.inputs = inputs @@ -661,6 +661,11 @@ class NetTestCase(object): We check to see if it's possible to have an input file and if the user has specified such file. + + + If the operations to be done here are network related or blocking, they + should be wrapped in a deferred. That is the return value of this + method should be a :class:`twisted.internet.defer.Deferred`. Returns: a generator that will yield one item from the file based on the

1 0

[ooni-probe/master] Update Unit Tests
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 6f5493dd7b6bd99d34eddb66ad7ae632edfd775d Author: aagbsn <aagbsn(a)extc.org> Date: Fri Aug 16 22:53:12 2013 +0200 Update Unit Tests Adds a call to the initializeInputProcessor method --- ooni/tests/test_nettest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index eb909d9..718ad89 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -189,6 +189,8 @@ class TestNetTest(unittest.TestCase): ntl.loadNetTestString(net_test_string_with_file) ntl.checkOptions() + nt = NetTest(ntl,None) + nt.initializeInputProcessor() # XXX: if you use the same test_class twice you will have consumed all # of its inputs! @@ -214,6 +216,7 @@ class TestNetTest(unittest.TestCase): ntl.checkOptions() net_test = NetTest(ntl, None) + net_test.initializeInputProcessor() measurements = list(net_test.generateMeasurements()) self.assertEqual(len(measurements), 20)

1 0

[ooni-probe/master] Only run unittests if oonib is imported
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 219f0ba3f503c5084e955f240482c06460ea6b3d Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Aug 21 14:54:47 2013 +0200 Only run unittests if oonib is imported --- ooni/tests/test_oonibclient.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 17259a6..06008bb 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -1,6 +1,11 @@ from twisted.trial import unittest from twisted.internet import defer +try: + import oonib +except ImportError: + oonib = None + from ooni.oonibclient import OONIBClient input_id = 'e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290' @@ -8,6 +13,8 @@ input_id = 'e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290' class TestOONIBClient(unittest.TestCase): def setUp(self): self.oonibclient = OONIBClient('http://127.0.0.1:8888') + if not oonib: + self.skipTest("OONIB is not running") @defer.inlineCallbacks def test_query(self):

1 0

[ooni-probe/master] Add support for running body_processor
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 51caef2bf217216b9330d1f4b449b59cf869c9f0 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Aug 21 14:55:05 2013 +0200 Add support for running body_processor --- ooni/utils/net.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index e97d8c3..b63a173 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -62,10 +62,11 @@ class StringProducer(object): pass class BodyReceiver(protocol.Protocol): - def __init__(self, finished, content_length=None): + def __init__(self, finished, content_length=None, body_processor=None): self.finished = finished self.data = "" self.bytes_remaining = content_length + self.body_processor = body_processor def dataReceived(self, b): self.data += b @@ -76,6 +77,8 @@ class BodyReceiver(protocol.Protocol): self.bytes_remaining -= len(b) def connectionLost(self, reason): + if self.body_processor: + self.data = self.body_processor(self.data) self.finished.callback(self.data) def getSystemResolver():

1 0

[ooni-probe/master] Get all the unittests to pass.
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 553cb673b39fce91b0829ebe71caba94a817eadc Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Aug 21 23:22:55 2013 +0200 Get all the unittests to pass. --- ooni/oonibclient.py | 26 +++++++++++++++++++++----- ooni/tests/test_oonibclient.py | 22 ++++++++++------------ ooni/utils/net.py | 2 +- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 3f9ed56..1a581c0 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -1,9 +1,11 @@ +from hashlib import sha256 + import json from twisted.internet import defer, reactor from twisted.web.client import Agent -from ooni.utils.net import BodyReceiver, StringProducer +from ooni.utils.net import BodyReceiver, StringProducer, Downloader class InputFile(object): def __init__(self, id, name=None, description=None, @@ -15,12 +17,20 @@ class InputFile(object): self.author = author self.date = date - self._file = None + self.file_path = None + + def verify(self): + digest = os.path.basename(self.file_path) + with open(self.file_path) as f: + file_hash = sha256(f.read()) + return file_hash.hexdigest() == digest + return False class OONIBClient(object): def __init__(self, address): self.address = address self.agent = Agent(reactor) + self.input_files = {} def _request(self, method, urn, genReceiver, bodyProducer=None): finished = defer.Deferred() @@ -56,14 +66,20 @@ class OONIBClient(object): return self._request('GET', urn, genReceiver) - def downloadInput(self, input_hash, download_path): - return self.download('/input/'+input_hash, download_path) - def getNettestPolicy(self): pass def queryBouncer(self, requested_helpers): pass + def getInput(self, input_hash): + return self.queryBackend('GET', '/input/' + input_hash) + + def getInputList(self): + return self.queryBackend('GET', '/input') + + def downloadInput(self, input_hash, download_path): + return self.download('/input/'+input_hash+'/file', download_path) + def getInputPolicy(self): pass diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 06008bb..65e9e7e 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -1,42 +1,39 @@ from twisted.trial import unittest from twisted.internet import defer -try: - import oonib -except ImportError: - oonib = None - from ooni.oonibclient import OONIBClient -input_id = 'e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290' +input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1' class TestOONIBClient(unittest.TestCase): def setUp(self): self.oonibclient = OONIBClient('http://127.0.0.1:8888') - if not oonib: - self.skipTest("OONIB is not running") @defer.inlineCallbacks def test_query(self): res = yield self.oonibclient.queryBackend('GET', '/policy/input') self.assertTrue(isinstance(res, list)) + @defer.inlineCallbacks def test_get_input_list(self): input_list = yield self.oonibclient.getInputList() - self.assertTrue(isinstance(inputList, list)) + self.assertTrue(isinstance(input_list, list)) + @defer.inlineCallbacks def test_get_input_descriptor(self): input_descriptor = yield self.oonibclient.getInput(input_id) for key in ['name', 'description', 'version', 'author', 'date']: self.assertTrue(key in input_descriptor.keys()) + @defer.inlineCallbacks def test_download_input(self): - pass + yield self.oonibclient.downloadInput(input_id, input_id) + @defer.inlineCallbacks def test_get_deck_list(self): input_list = yield self.oonibclient.getInputList() - self.assertTrue(isinstance(inputList, list)) + self.assertTrue(isinstance(input_list, list)) def test_get_deck_descriptor(self): pass @@ -44,9 +41,10 @@ class TestOONIBClient(unittest.TestCase): def test_download_deck(self): pass + @defer.inlineCallbacks def test_get_nettest_list(self): input_list = yield self.oonibclient.getInputList() - self.assertTrue(isinstance(inputList, list)) + self.assertTrue(isinstance(input_list, list)) def test_get_nettest_descriptor(self): pass diff --git a/ooni/utils/net.py b/ooni/utils/net.py index db2ea18..ab33d00 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -98,7 +98,7 @@ class Downloader(protocol.Protocol): def connectionLost(self, reason): self.fp.close() - self.finished.callback(self.download_path) + self.finished.callback(None) def getSystemResolver(): """

1 0

[ooni-probe/master] Start outlining the oonibclient object.
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 7efd771be71e654cf8d961ae33b13bd7a3cad6e5 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 20 16:36:30 2013 +0200 Start outlining the oonibclient object. Write the minimum unittests that it must pass to be complete. --- ooni/oonibclient.py | 57 ++++++++++++++++++++++++++++++++++++++++ ooni/tests/test_oonibclient.py | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py new file mode 100644 index 0000000..28e42a7 --- /dev/null +++ b/ooni/oonibclient.py @@ -0,0 +1,57 @@ +import json + +from twisted.internet import defer, reactor +from twisted.web.client import Agent + +from ooni.utils.net import BodyReceiver, StringProducer + +class InputFile(object): + def __init__(self, id, name=None, description=None, + version=None, author=None, date=None): + self.id = id + self.name = name + self.description = description + self.version = version + self.author = author + self.date = date + + self._file = None + +class Deck(object): + pass + +class OONIBClient(object): + def __init__(self, address): + self.address = address + self.agent = Agent(reactor) + + def queryBackend(self, method, urn, query=None): + finished = defer.Deferred() + + bodyProducer = None + if query: + bodyProducer = StringProducer(json.dumps(query)) + + uri = self.address + urn + d = self.agent.request(method, uri, bodyProducer) + @d.addCallback + def cb(response): + content_length = response.headers.getRawHeaders('content-length') + response.deliverBody(BodyReceiver(finished, content_length, json.loads)) + + @d.addErrback + def eb(err): + finished.errback(err) + + return finished + + def getNettestPolicy(self): + pass + + def queryBouncer(self, requested_helpers): + pass + + def getInputPolicy(self): + pass + + diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py new file mode 100644 index 0000000..17259a6 --- /dev/null +++ b/ooni/tests/test_oonibclient.py @@ -0,0 +1,48 @@ +from twisted.trial import unittest +from twisted.internet import defer + +from ooni.oonibclient import OONIBClient + +input_id = 'e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290' + +class TestOONIBClient(unittest.TestCase): + def setUp(self): + self.oonibclient = OONIBClient('http://127.0.0.1:8888') + + @defer.inlineCallbacks + def test_query(self): + res = yield self.oonibclient.queryBackend('GET', '/policy/input') + self.assertTrue(isinstance(res, list)) + + def test_get_input_list(self): + input_list = yield self.oonibclient.getInputList() + self.assertTrue(isinstance(inputList, list)) + + def test_get_input_descriptor(self): + input_descriptor = yield self.oonibclient.getInput(input_id) + for key in ['name', 'description', + 'version', 'author', 'date']: + self.assertTrue(key in input_descriptor.keys()) + + def test_download_input(self): + pass + + def test_get_deck_list(self): + input_list = yield self.oonibclient.getInputList() + self.assertTrue(isinstance(inputList, list)) + + def test_get_deck_descriptor(self): + pass + + def test_download_deck(self): + pass + + def test_get_nettest_list(self): + input_list = yield self.oonibclient.getInputList() + self.assertTrue(isinstance(inputList, list)) + + def test_get_nettest_descriptor(self): + pass + + def test_download_nettest(self): + pass

1 0

[ooni-probe/master] Merge remote-tracking branch 'aagbsn/feature/add_nonblocking_inputprocessor_setup' into feature/oonib-support
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 81cf3c89a1d8d21c75000e92f90ed3bc63558521 Merge: 8d77ff5 6f5493d Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Aug 21 16:14:53 2013 +0200 Merge remote-tracking branch 'aagbsn/feature/add_nonblocking_inputprocessor_setup' into feature/oonib-support * aagbsn/feature/add_nonblocking_inputprocessor_setup: Update Unit Tests Make inputProcessor setup be non-blocking Make the return value of getInputProcessor be a deferred. Get the input processor as late as possible Conflicts: ooni/nettest.py ooni/director.py | 1 + ooni/nettest.py | 17 ++++++++++++----- ooni/tests/test_nettest.py | 3 +++ 3 files changed, 16 insertions(+), 5 deletions(-)

1 0

[ooni-probe/master] Merge remote-tracking branch 'aagbsn/feature/add_deck_input_fetching' into feature/oonib-support
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 8d77ff51fe8b636027dc4c199730357a56b63a51 Merge: 51caef2 0fd667f Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Aug 21 15:07:42 2013 +0200 Merge remote-tracking branch 'aagbsn/feature/add_deck_input_fetching' into feature/oonib-support * aagbsn/feature/add_deck_input_fetching: Make oonicli download inputs before starting tests. Adds class TestDeck Move input checking out of checkOptions Add UnableToLoadDeckInput Exception Conflicts: ooni/oonicli.py ooni/deck.py | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ ooni/errors.py | 3 +- ooni/nettest.py | 12 +++--- ooni/oonicli.py | 96 +++++++++++++++++++++++++++------------------ 4 files changed, 184 insertions(+), 43 deletions(-) diff --cc ooni/oonicli.py index 32fc762,2a141f6..4c7dd8b --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@@ -110,43 -111,6 +111,17 @@@ def runWithDirector() log.start(global_options['logfile']) + if config.privacy.includepcap: + try: + checkForRoot() + except errors.InsufficientPrivileges: + log.err("Insufficient Privileges to capture packets." + " See ooniprobe.conf privacy.includepcap") + sys.exit(2) + + # contains (test_cases, options, cmd_line_options) + test_list = [] - if global_options['no-collector']: - log.msg("Not reporting using a collector") - global_options['collector'] = None - - if global_options['testdeck']: - test_deck = yaml.safe_load(open(global_options['testdeck'])) - for test in test_deck: - test_list.append(NetTestLoader(test['options']['subargs'], - test_file=test['options']['test_file'])) - else: - log.debug("No test deck detected") - test_list.append(NetTestLoader(global_options['subargs'], - test_file=global_options['test_file'])) - - # check each test's usageOptions - for net_test_loader in test_list: - try: - net_test_loader.checkOptions() - except MissingRequiredOption, option_name: - log.err('Missing required option: "%s"' % option_name) - print net_test_loader.usageOptions().getUsage() - sys.exit(2) - except usage.UsageError, e: - log.err(e) - print net_test_loader.usageOptions().getUsage() - sys.exit(2) + director = Director() d = director.start()

1 0

[ooni-probe/master] Start implementing support for downloading of test decks
by art＠torproject.org 27 Aug '13

27 Aug '13

commit fcae18e1cc5b6387bf4b609d62deec477414cd92 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Aug 21 22:21:33 2013 +0200 Start implementing support for downloading of test decks The work here was mainly merging together my branch with the one of @aagbsn and refactoring a lot of the code. --- ooni/deck.py | 83 +++++++++++---------------------------------------- ooni/nettest.py | 27 ++++++++++++++++- ooni/oonibclient.py | 36 ++++++++++++++-------- ooni/oonicli.py | 14 ++++----- ooni/utils/net.py | 19 ++++++++++++ 5 files changed, 93 insertions(+), 86 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 02c8fc1..f99916b 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -10,23 +10,6 @@ import os import re import yaml -onionInputRegex = re.compile("httpo://[a-z0-9]{16}\.onion/input/([a-z0-9]){40}$") - -(a)defer.inlineCallbacks -def downloadFile(fileURL, filePath): - - def writeFile(response_body, filePath): - f = open(filePath, 'w+') - f.write(response_body) - - finished = defer.Deferred() - finished.addCallback(writeFile, filePath) - - agent = Agent(reactor, sockshost="127.0.0.1", - socksport=int(config.tor.socks_port)) - response = yield agent.request("GET", fileURL) - response.deliverBody(BodyReceiver(finished)) - def verifyFile(filePath): # get the filename component of the file path digest = os.path.basename(filePath) @@ -35,10 +18,11 @@ def verifyFile(filePath): return sha1digest.hexdigest() == digest return False -class TestDeck(object): - def __init__(self, deckFile=None): +class Deck(object): + def __init__(self, oonibclient, deckFile=None): self.netTestLoaders = [] self.inputs = [] + if deckFile: self.loadDeck(deckFile) def loadDeck(self, deckFile): @@ -63,54 +47,21 @@ class TestDeck(object): @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): """ fetch and verify a single NetTest's inputs """ - for test_class, test_methods in net_test_loader.testCases: - if test_class.inputFile: - inputArg = test_class.inputFile[0] - inputFileURL = test_class.localOptions[inputArg] - - m = onionInputRegex.match(inputFileURL) - if m: - fileDigest = m.group(1) - else: - fileDigest = os.path.basename(inputFileURL) + for input_file in net_test_loader.inputFiles: + if 'url' in input_file: + oonib = OONIBClient(input_file['address']) - cachedInputDir = os.path.join(config.advanced.data_dir, + cached_input_dir = os.path.join(config.advanced.data_dir, 'inputs') - cachedPath = os.path.join(cachedInputDir, fileDigest) - self.inputs.append(cachedPath) + cached_path = os.path.join(cached_input_dir, input_file['hash']) + self.inputs.append(cached_path) - if os.path.exists(cachedPath) and verifyFile(cachedPath): - test_class.localOptions[inputArg] = cachedPath - continue - if m: - yield downloadFile(inputFileURL, cachedPath) - if verifyFile(cachedPath): - test_class.localOptions[inputArg] = cachedPath + if os.path.exists(cached_path) and verifyFile(cached_path): + test_class.localOptions[inputArg] = cached_path continue - - raise UnableToLoadDeckInput, cachedPath - -def test_verify_file_success(): - f = open('/dev/urandom') - r = f.read(1024*1024) - z = sha1(f).hexdigest() - f.close() - fn = '/tmp/%s' % z - f = open(fn) - f.write(r) - f.close() - verifyFile(fn) - os.unlink(fn) - -def test_verify_file_failure(): - pass -def test_load_deck_with_no_inputs(deck): - pass -def test_load_deck_with_cached_input(deckFile): - pass -def test_load_deck_with_evil_path(deckFile): - pass -def test_load_deck_with_download_success(deckFile): - pass -def test_load_deck_with_download_failure(deckFile): - pass + yield oonib.downloadInput(input_file['hash'], cached_path) + if verifyFile(cached_path): + test_class.localOptions[input_file['key']] = cached_path + continue + + raise UnableToLoadDeckInput, cached_path diff --git a/ooni/nettest.py b/ooni/nettest.py index 51c6074..4806bdd 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -173,8 +173,9 @@ class NetTestLoader(object): method_prefix = 'test' def __init__(self, options, test_file=None, test_string=None): + self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]){40}$") self.options = options - test_cases = None + self.testCases, test_cases = None, None if test_file: test_cases = loadNetTestFile(test_file) @@ -183,6 +184,30 @@ class NetTestLoader(object): if test_cases: self.setupTestCases(test_cases) + + @property + def inputFiles(self): + input_files = [] + if not self.testCases: + return input_files + + for test_class, test_methods in self.testCases: + if test_class.inputFile: + key = test_class.inputFile[0] + filename = test_class.localOptions[key] + input_file = { + 'id': key + } + m = self.onionInputRegex.match(filename) + if m: + input_file['url'] = filename + input_file['address'] = m.group(1) + input_file['hash'] = m.group(2) + else: + input_file['filename'] = filename + input_files.append(input_file) + + return input_files @property def testDetails(self): diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 28e42a7..3f9ed56 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -17,27 +17,21 @@ class InputFile(object): self._file = None -class Deck(object): - pass - class OONIBClient(object): def __init__(self, address): self.address = address self.agent = Agent(reactor) - def queryBackend(self, method, urn, query=None): + def _request(self, method, urn, genReceiver, bodyProducer=None): finished = defer.Deferred() - bodyProducer = None - if query: - bodyProducer = StringProducer(json.dumps(query)) - uri = self.address + urn d = self.agent.request(method, uri, bodyProducer) + @d.addCallback - def cb(response): + def callback(response): content_length = response.headers.getRawHeaders('content-length') - response.deliverBody(BodyReceiver(finished, content_length, json.loads)) + response.deliverBody(genReceiver(finished, content_length)) @d.addErrback def eb(err): @@ -45,6 +39,26 @@ class OONIBClient(object): return finished + def queryBackend(self, method, urn, query=None): + bodyProducer = None + if query: + bodyProducer = StringProducer(json.dumps(query), bodyProducer) + + def genReceiver(finished, content_length): + return BodyReceiver(finished, content_length, json.loads) + + return self._request(method, urn, genReceiver, bodyProducer) + + def download(self, urn, download_path): + + def genReceiver(finished, content_length): + return Downloader(download_path, finished, content_length) + + return self._request('GET', urn, genReceiver) + + def downloadInput(self, input_hash, download_path): + return self.download('/input/'+input_hash, download_path) + def getNettestPolicy(self): pass @@ -53,5 +67,3 @@ class OONIBClient(object): def getInputPolicy(self): pass - - diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 4c7dd8b..8708642 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -14,7 +14,7 @@ from ooni import errors from ooni.settings import config from ooni.director import Director -from ooni.deck import TestDeck +from ooni.deck import Deck from ooni.reporter import YAMLReporter, OONIBReporter from ooni.nettest import NetTestLoader, MissingRequiredOption @@ -125,19 +125,19 @@ def runWithDirector(): director = Director() d = director.start() - test_deck = TestDeck() + deck = Deck() if global_options['no-collector']: log.msg("Not reporting using a collector") collector = global_options['collector'] = None try: if global_options['testdeck']: - test_deck.loadDeck(global_options['testdeck']) + deck.loadDeck(global_options['testdeck']) else: log.debug("No test deck detected") net_test_loader = NetTestLoader(global_options['subargs'], test_file=global_options['test_file']) - test_deck.insert(net_test_loader) + deck.insert(net_test_loader) except MissingRequiredOption, option_name: log.err('Missing required option: "%s"' % option_name) print net_test_loader.usageOptions().getUsage() @@ -149,7 +149,7 @@ def runWithDirector(): def fetch_nettest_inputs(result): try: - test_deck.fetchAndVerifyDeckInputs() + deck.fetchAndVerifyDeckInputs() except errors.UnableToLoadDeckInput, e: return defer.failure.Failure(result) @@ -174,8 +174,8 @@ def runWithDirector(): # Wait until director has started up (including bootstrapping Tor) # before adding tests def post_director_start(_): - for net_test_loader in test_deck.netTestLoaders: - # TestDecks can specify different collectors + for net_test_loader in deck.netTestLoaders: + # Decks can specify different collectors # for each net test, so that each NetTest # may be paired with a test_helper and its collector # However, a user can override this behavior by diff --git a/ooni/utils/net.py b/ooni/utils/net.py index b63a173..db2ea18 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -81,6 +81,25 @@ class BodyReceiver(protocol.Protocol): self.data = self.body_processor(self.data) self.finished.callback(self.data) +class Downloader(protocol.Protocol): + def __init__(self, download_path, + finished, content_length=None): + self.finished = finished + self.bytes_remaining = content_length + self.fp = open(download_path, 'w+') + + def dataReceived(self, b): + self.fp.write(b) + if self.bytes_remaining: + if self.bytes_remaining == 0: + self.connectionLost(None) + else: + self.bytes_remaining -= len(b) + + def connectionLost(self, reason): + self.fp.close() + self.finished.callback(self.download_path) + def getSystemResolver(): """ XXX implement a function that returns the resolver that is currently

1 0

[ooni-probe/master] Fix some bugs in unittests
by art＠torproject.org 27 Aug '13

27 Aug '13

commit a3c6e2aacdffc0fe59f4b5c3f959b04ddf6fa1dd Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 16:24:14 2013 +0200 Fix some bugs in unittests --- ooni/oonibclient.py | 2 ++ ooni/tests/test_oonibclient.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index d8a8bcd..4f555b2 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -5,6 +5,7 @@ import json from twisted.internet import defer, reactor from twisted.web.client import Agent +from ooni.utils import log from ooni.utils.net import BodyReceiver, StringProducer, Downloader class InputFile(object): @@ -14,6 +15,7 @@ class InputFile(object): self.version = descriptor['version'] self.author = descriptor['author'] self.date = descriptor['date'] + self.description = descriptor['description'] self.file_path = None diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 65e9e7e..02f9f8f 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -23,8 +23,8 @@ class TestOONIBClient(unittest.TestCase): def test_get_input_descriptor(self): input_descriptor = yield self.oonibclient.getInput(input_id) for key in ['name', 'description', - 'version', 'author', 'date']: - self.assertTrue(key in input_descriptor.keys()) + 'version', 'author', 'date', 'id']: + self.assertTrue(hasattr(input_descriptor, key)) @defer.inlineCallbacks def test_download_input(self):

1 0