August 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Store a cache of input descriptor lookups
by art＠torproject.org 27 Aug '13

27 Aug '13

commit a00231ca5aad7992b85ff57d7ca4e1fd5607b65f Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 14:58:41 2013 +0200 Store a cache of input descriptor lookups --- ooni/oonibclient.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 1a581c0..d8a8bcd 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -8,14 +8,12 @@ from twisted.web.client import Agent from ooni.utils.net import BodyReceiver, StringProducer, Downloader class InputFile(object): - def __init__(self, id, name=None, description=None, - version=None, author=None, date=None): - self.id = id - self.name = name - self.description = description - self.version = version - self.author = author - self.date = date + def __init__(self, descriptor): + self.id = descriptor['id'] + self.name = descriptor['name'] + self.version = descriptor['version'] + self.author = descriptor['author'] + self.date = descriptor['date'] self.file_path = None @@ -73,7 +71,19 @@ class OONIBClient(object): pass def getInput(self, input_hash): - return self.queryBackend('GET', '/input/' + input_hash) + try: + return defer.succeed(self.input_files[input_hash]) + except KeyError: + d = self.queryBackend('GET', '/input/' + input_hash) + @d.addCallback + def cb(descriptor): + self.input_files[input_hash] = InputFile(descriptor) + return self.input_files[input_hash] + @d.addErrback + def err(err): + log.err("Failed to get descriptor for input %s" % input_hash) + log.exception(err) + return d def getInputList(self): return self.queryBackend('GET', '/input')

1 0

[ooni-probe/master] Start outlining the classes for policy handling
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 3ddc95bd9841770dfe08cbf92265563cf17baf45 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 19:02:06 2013 +0200 Start outlining the classes for policy handling --- .travis.yml | 1 - ooni/nettest.py | 6 ++++++ ooni/oonibclient.py | 24 ++++++++++++++++++++++-- ooni/tests/test_oonibclient.py | 2 +- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index fed5d7c..568f42d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ before_install: - sudo apt-get install tor libpcap-dev python: - "2.7" - - "2.6" # command to install dependencies # the first is for testing pip and the second for setuptools install: diff --git a/ooni/nettest.py b/ooni/nettest.py index b8cdf65..1699f19 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -1,6 +1,7 @@ import os import re import time +from hashlib import sha256 from twisted.internet import defer, reactor from twisted.trial.runner import filenameToModule @@ -206,6 +207,11 @@ class NetTestLoader(object): input_file['hash'] = m.group(2) else: input_file['filename'] = filename + with open(filename) as f: + h = sha256() + for l in f: + h.update(l) + input_file['hash'] = h.hexdigest() input_files.append(input_file) return input_files diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 9d2085b..5861ff4 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -50,7 +50,7 @@ class InputFile(object): 'date': self.date, 'description': self.description }, f) - + def load(self, descriptor): self.name = descriptor['name'] self.version = descriptor['version'] @@ -64,6 +64,23 @@ class InputFile(object): file_hash = sha256(f.read()) assert file_hash.hexdigest() == digest + def validate(self, policy): + """ + Validate this input file against the specified input policy. + """ + for input_file + +class Collector(object): + def __init__(self): + self.nettest_policy = None + self.input_policy = None + + def validateInput(self, input_hash): + pass + + def validateNettest(self, nettest): + pass + class OONIBClient(object): def __init__(self, address): self.address = address @@ -155,4 +172,7 @@ class OONIBClient(object): return d def getInputPolicy(self): - pass + return self.queryBackend('GET', '/policy/input') + + def getNettestPolicy(self): + return self.queryBackend('GET', '/policy/nettest') diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 47a3d47..c039c65 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -17,7 +17,7 @@ class TestOONIBClient(unittest.TestCase): s.shutdown(2) except Exception as ex: self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") - self.oonibclient = OONIBClient('http://'+host+':'+str(port)) + self.oonibclient = OONIBClient('http://' + host + ':' + str(port)) @defer.inlineCallbacks def test_query(self):

1 0

[ooni-probe/master] Implement downloading of input files based on URL in deck file
by art＠torproject.org 27 Aug '13

27 Aug '13

commit f62f822b926eab78153c3081ced38b1075b16e10 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 18:10:10 2013 +0200 Implement downloading of input files based on URL in deck file --- ooni/deck.py | 21 ++++++++++++++------- ooni/nettest.py | 5 +++-- ooni/oonibclient.py | 8 +++++--- ooni/oonicli.py | 2 +- ooni/utils/net.py | 1 + 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index ef1b58a..8f1335d 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -2,15 +2,19 @@ from ooni.nettest import NetTestLoader from ooni.settings import config +from ooni.utils import log from ooni.utils.txagentwithsocks import Agent from ooni.errors import UnableToLoadDeckInput +from ooni.oonibclient import OONIBClient + from twisted.internet import reactor, defer + import os import re import yaml class Deck(object): - def __init__(self, oonibclient, deckFile=None): + def __init__(self, deckFile=None): self.netTestLoaders = [] self.inputs = [] @@ -30,22 +34,25 @@ class Deck(object): self.fetchAndVerifyNetTestInput(net_test_loader) self.netTestLoaders.append(net_test_loader) + @defer.inlineCallbacks def fetchAndVerifyDeckInputs(self): """ fetch and verify inputs for all NetTests in the deck """ for net_test_loader in self.netTestLoaders: - self.fetchAndVerifyNetTestInput(net_test_loader) + yield self.fetchAndVerifyNetTestInput(net_test_loader) @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): """ fetch and verify a single NetTest's inputs """ - for input_file in net_test_loader.inputFiles: - if 'url' in input_file: - oonib = OONIBClient(input_file['address']) + log.debug("Fetching and verifying inputs") + for i in net_test_loader.inputFiles: + if 'url' in i: + log.debug("Downloading %s" % i['url']) + oonib = OONIBClient(i['address']) - input_file = yield oonib.downloadInput(input_file['hash']) + input_file = yield oonib.downloadInput(i['hash']) try: input_file.verify() except AssertionError: raise UnableToLoadDeckInput, cached_path - test_class.localOptions[input_file['key']] = input_file.cached_file + i['test_class'].localOptions[i['key']] = input_file.cached_file diff --git a/ooni/nettest.py b/ooni/nettest.py index 4806bdd..b8cdf65 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -173,7 +173,7 @@ class NetTestLoader(object): method_prefix = 'test' def __init__(self, options, test_file=None, test_string=None): - self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]){40}$") + self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]{64})$") self.options = options self.testCases, test_cases = None, None @@ -196,7 +196,8 @@ class NetTestLoader(object): key = test_class.inputFile[0] filename = test_class.localOptions[key] input_file = { - 'id': key + 'key': key, + 'test_class': test_class } m = self.onionInputRegex.match(filename) if m: diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 9665ee1..9d2085b 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -4,7 +4,8 @@ import json from hashlib import sha256 from twisted.internet import defer, reactor -from twisted.web.client import Agent + +from ooni.utils.txagentwithsocks import Agent from ooni.settings import config from ooni.utils import log @@ -66,7 +67,8 @@ class InputFile(object): class OONIBClient(object): def __init__(self, address): self.address = address - self.agent = Agent(reactor) + self.agent = Agent(reactor, sockshost="127.0.0.1", + socksport=config.tor.socks_port) self.input_files = {} def _request(self, method, urn, genReceiver, bodyProducer=None): @@ -77,7 +79,7 @@ class OONIBClient(object): @d.addCallback def callback(response): - content_length = response.headers.getRawHeaders('content-length') + content_length = int(response.headers.getRawHeaders('content-length')[0]) response.deliverBody(genReceiver(finished, content_length)) @d.addErrback diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 8708642..a7df18f 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -149,7 +149,7 @@ def runWithDirector(): def fetch_nettest_inputs(result): try: - deck.fetchAndVerifyDeckInputs() + return deck.fetchAndVerifyDeckInputs() except errors.UnableToLoadDeckInput, e: return defer.failure.Failure(result) diff --git a/ooni/utils/net.py b/ooni/utils/net.py index ab33d00..1ec9608 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -97,6 +97,7 @@ class Downloader(protocol.Protocol): self.bytes_remaining -= len(b) def connectionLost(self, reason): + self.fp.flush() self.fp.close() self.finished.callback(None)

1 0

[ooni-probe/master] Add a basic deck that includes inputs as URLs
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 2601d75c1efa9c1cc0e474f5bc1ef5955cda7703 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 18:30:32 2013 +0200 Add a basic deck that includes inputs as URLs --- decks/basic.deck | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/decks/basic.deck b/decks/basic.deck new file mode 100644 index 0000000..c784cd5 --- /dev/null +++ b/decks/basic.deck @@ -0,0 +1,24 @@ +- options: + collector: null + help: 0 + logfile: null + no-default-reporter: 0 + parallelism: '10' + pcapfile: null + reportfile: null + resume: 0 + subargs: [-f, 'httpo://3mr5phzltfzqgh6y.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] + test_file: data/nettests/blocking/http_requests.py + testdeck: null +- options: + collector: null + help: 0 + logfile: null + no-default-reporter: 0 + parallelism: '10' + pcapfile: null + reportfile: null + resume: 0 + subargs: [-b, 'http://93.95.227.200'] + test_file: data/nettests/manipulation/http_header_field_manipulation.py + testdeck: null

1 0

[ooni-probe/master] Refactor verification of input file and implement caching
by art＠torproject.org 27 Aug '13

27 Aug '13

commit fb8a90c6f2ec136eb477bbb2bf91d61456e66c80 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 17:38:38 2013 +0200 Refactor verification of input file and implement caching --- ooni/deck.py | 28 +++--------- ooni/oonibclient.py | 93 ++++++++++++++++++++++++++++++++-------- ooni/tests/test_oonibclient.py | 2 +- 3 files changed, 83 insertions(+), 40 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index f99916b..ef1b58a 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -1,6 +1,5 @@ #-*- coding: utf-8 -*- -from hashlib import sha1 from ooni.nettest import NetTestLoader from ooni.settings import config from ooni.utils.txagentwithsocks import Agent @@ -10,14 +9,6 @@ import os import re import yaml -def verifyFile(filePath): - # get the filename component of the file path - digest = os.path.basename(filePath) - with open(filePath) as f: - sha1digest = sha1(f.read()) - return sha1digest.hexdigest() == digest - return False - class Deck(object): def __init__(self, oonibclient, deckFile=None): self.netTestLoaders = [] @@ -51,17 +42,10 @@ class Deck(object): if 'url' in input_file: oonib = OONIBClient(input_file['address']) - cached_input_dir = os.path.join(config.advanced.data_dir, - 'inputs') - cached_path = os.path.join(cached_input_dir, input_file['hash']) - self.inputs.append(cached_path) - - if os.path.exists(cached_path) and verifyFile(cached_path): - test_class.localOptions[inputArg] = cached_path - continue - yield oonib.downloadInput(input_file['hash'], cached_path) - if verifyFile(cached_path): - test_class.localOptions[input_file['key']] = cached_path - continue + input_file = yield oonib.downloadInput(input_file['hash']) + try: + input_file.verify() + except AssertionError: + raise UnableToLoadDeckInput, cached_path - raise UnableToLoadDeckInput, cached_path + test_class.localOptions[input_file['key']] = input_file.cached_file diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 4f555b2..9665ee1 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -1,30 +1,67 @@ -from hashlib import sha256 - +import os import json +from hashlib import sha256 + from twisted.internet import defer, reactor from twisted.web.client import Agent +from ooni.settings import config from ooni.utils import log from ooni.utils.net import BodyReceiver, StringProducer, Downloader class InputFile(object): - def __init__(self, descriptor): - self.id = descriptor['id'] + def __init__(self, input_hash): + self.id = input_hash + cached_input_dir = os.path.join(config.advanced.data_dir, + 'inputs') + cache_path = os.path.join(cached_input_dir, input_hash) + self.cached_file = cache_path + self.cached_descriptor = cache_path + '.desc' + + @property + def descriptorCached(self): + if os.path.exists(self.cached_descriptor): + with open(self.cached_descriptor) as f: + descriptor = json.load(f) + self.load(descriptor) + return True + return False + + @property + def fileCached(self): + if os.path.exists(self.cached_file): + try: + self.verify() + except AssertionError: + log.err("The input %s failed validation. Going to consider it not cached." % self.id) + return False + return True + return False + + def save(self): + with open(self.cached_descriptor, 'w+') as f: + json.dump({ + 'name': self.name, + 'id': self.id, + 'version': self.version, + 'author': self.author, + 'date': self.date, + 'description': self.description + }, f) + + def load(self, descriptor): self.name = descriptor['name'] self.version = descriptor['version'] self.author = descriptor['author'] self.date = descriptor['date'] self.description = descriptor['description'] - self.file_path = None - def verify(self): - digest = os.path.basename(self.file_path) - with open(self.file_path) as f: + digest = os.path.basename(self.cached_file) + with open(self.cached_file) as f: file_hash = sha256(f.read()) - return file_hash.hexdigest() == digest - return False + assert file_hash.hexdigest() == digest class OONIBClient(object): def __init__(self, address): @@ -73,25 +110,47 @@ class OONIBClient(object): pass def getInput(self, input_hash): - try: - return defer.succeed(self.input_files[input_hash]) - except KeyError: + input_file = InputFile(input_hash) + if input_file.descriptorCached: + return defer.succeed(input_file) + else: d = self.queryBackend('GET', '/input/' + input_hash) + @d.addCallback def cb(descriptor): - self.input_files[input_hash] = InputFile(descriptor) - return self.input_files[input_hash] + input_file.load(descriptor) + input_file.save() + return input_file + @d.addErrback def err(err): log.err("Failed to get descriptor for input %s" % input_hash) log.exception(err) + return d def getInputList(self): return self.queryBackend('GET', '/input') - def downloadInput(self, input_hash, download_path): - return self.download('/input/'+input_hash+'/file', download_path) + def downloadInput(self, input_hash): + input_file = InputFile(input_hash) + + if input_file.fileCached: + return defer.succeed(input_file) + else: + d = self.download('/input/'+input_hash+'/file', input_file.cached_file) + + @d.addCallback + def cb(res): + input_file.verify() + return input_file + + @d.addErrback + def err(err): + log.err("Failed to download the input file %s" % input_hash) + log.exception(err) + + return d def getInputPolicy(self): pass diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 02f9f8f..26e2f62 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -28,7 +28,7 @@ class TestOONIBClient(unittest.TestCase): @defer.inlineCallbacks def test_download_input(self): - yield self.oonibclient.downloadInput(input_id, input_id) + yield self.oonibclient.downloadInput(input_id) @defer.inlineCallbacks def test_get_deck_list(self):

1 0

[ooni-probe/master] Add support for looking up test helpers via a bouncer.
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 3bface3bd4fab9e87bdc8c440d70e07e19b8af28 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Aug 23 16:46:15 2013 +0200 Add support for looking up test helpers via a bouncer. --- data/nettests/blocking/dnsconsistency.py | 4 +- data/nettests/manipulation/dnsspoof.py | 1 + .../manipulation/http_header_field_manipulation.py | 1 + data/nettests/manipulation/http_host.py | 1 + .../manipulation/http_invalid_request_line.py | 4 +- data/nettests/manipulation/traceroute.py | 3 +- decks/basic.deck | 2 +- ooni/deck.py | 50 +++++++++++++++--- ooni/errors.py | 2 + ooni/nettest.py | 28 ++++++++-- ooni/oonibclient.py | 55 ++++++++++++++------ ooni/oonicli.py | 42 ++++++++++----- ooni/reporter.py | 8 ++- ooni/tests/test_oonibclient.py | 22 ++++++++ ooni/utils/net.py | 9 ++-- 15 files changed, 183 insertions(+), 49 deletions(-) diff --git a/data/nettests/blocking/dnsconsistency.py b/data/nettests/blocking/dnsconsistency.py index 7b6e7b9..3c88cd2 100644 --- a/data/nettests/blocking/dnsconsistency.py +++ b/data/nettests/blocking/dnsconsistency.py @@ -38,12 +38,14 @@ class DNSConsistencyTest(dnst.DNSTest): name = "DNS Consistency" description = "DNS censorship detection test" - version = "0.5" + version = "0.6" authors = "Arturo Filastò, Isis Lovecruft" requirements = None inputFile = ['file', 'f', None, 'Input file of list of hostnames to attempt to resolve'] + + requiredTestHelpers = {'backend': 'dns'} usageOptions = UsageOptions requiredOptions = ['backend', 'file'] diff --git a/data/nettests/manipulation/dnsspoof.py b/data/nettests/manipulation/dnsspoof.py index 5c50c2f..c9120a4 100644 --- a/data/nettests/manipulation/dnsspoof.py +++ b/data/nettests/manipulation/dnsspoof.py @@ -22,6 +22,7 @@ class DNSSpoof(scapyt.ScapyTest): usageOptions = UsageOptions + requiredTestHelpers = {'backend': 'dns'} requiredOptions = ['hostname', 'resolver'] def setUp(self): diff --git a/data/nettests/manipulation/http_header_field_manipulation.py b/data/nettests/manipulation/http_header_field_manipulation.py index 509f4ef..3423442 100644 --- a/data/nettests/manipulation/http_header_field_manipulation.py +++ b/data/nettests/manipulation/http_header_field_manipulation.py @@ -48,6 +48,7 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): randomizeUA = False usageOptions = UsageOptions + requiredTestHelpers = {'backend': 'http-return-json-headers'} requiredOptions = ['backend'] def get_headers(self): diff --git a/data/nettests/manipulation/http_host.py b/data/nettests/manipulation/http_host.py index 3f1e0c6..2ec517c 100644 --- a/data/nettests/manipulation/http_host.py +++ b/data/nettests/manipulation/http_host.py @@ -43,6 +43,7 @@ class HTTPHost(httpt.HTTPTest): inputFile = ['file', 'f', None, 'List of hostnames to test for censorship'] + requiredTestHelpers = {'backend': 'http-return-json-headers'} requiredOptions = ['backend'] def test_filtering_prepend_newline_to_method(self): diff --git a/data/nettests/manipulation/http_invalid_request_line.py b/data/nettests/manipulation/http_invalid_request_line.py index 2482282..64dbcac 100644 --- a/data/nettests/manipulation/http_invalid_request_line.py +++ b/data/nettests/manipulation/http_invalid_request_line.py @@ -20,10 +20,12 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): ascii letters or numbers ('XxXx' will be 4). """ name = "HTTP Invalid Request Line" - version = "0.1.4" + version = "0.2" authors = "Arturo Filastò" usageOptions = UsageOptions + + requiredTestHelpers = {'backend': 'tcp-echo'} requiredOptions = ['backend'] def setUp(self): diff --git a/data/nettests/manipulation/traceroute.py b/data/nettests/manipulation/traceroute.py index 3f6f17b..2db1826 100644 --- a/data/nettests/manipulation/traceroute.py +++ b/data/nettests/manipulation/traceroute.py @@ -23,8 +23,9 @@ class UsageOptions(usage.Options): class TracerouteTest(scapyt.BaseScapyTest): name = "Multi Protocol Traceroute Test" author = "Arturo Filastò" - version = "0.1.1" + version = "0.2" + requiredTestHelpers = {'backend': 'traceroute'} usageOptions = UsageOptions dst_ports = [0, 22, 23, 53, 80, 123, 443, 8080, 65535] diff --git a/decks/basic.deck b/decks/basic.deck index c784cd5..84a8f9d 100644 --- a/decks/basic.deck +++ b/decks/basic.deck @@ -7,7 +7,7 @@ pcapfile: null reportfile: null resume: 0 - subargs: [-f, 'httpo://3mr5phzltfzqgh6y.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] + subargs: [-f, 'httpo://amcq5ldf3vwg22ze.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] test_file: data/nettests/blocking/http_requests.py testdeck: null - options: diff --git a/ooni/deck.py b/ooni/deck.py index 8f1335d..1ad8377 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -4,7 +4,7 @@ from ooni.nettest import NetTestLoader from ooni.settings import config from ooni.utils import log from ooni.utils.txagentwithsocks import Agent -from ooni.errors import UnableToLoadDeckInput +from ooni import errors as e from ooni.oonibclient import OONIBClient from twisted.internet import reactor, defer @@ -14,9 +14,12 @@ import re import yaml class Deck(object): - def __init__(self, deckFile=None): + def __init__(self, bouncer, deckFile=None): + self.bouncer = bouncer self.netTestLoaders = [] self.inputs = [] + self.testHelpers = {} + self.collector = None if deckFile: self.loadDeck(deckFile) @@ -31,14 +34,20 @@ class Deck(object): def insert(self, net_test_loader): """ Add a NetTestLoader to this test deck """ net_test_loader.checkOptions() - self.fetchAndVerifyNetTestInput(net_test_loader) self.netTestLoaders.append(net_test_loader) - + + def getRequiredTestHelpers(self): + for net_test_loader in self.netTestLoaders: + for test_helper in net_test_loader.requiredTestHelpers: + self.testHelpers[test_helper['name']] = None + @defer.inlineCallbacks - def fetchAndVerifyDeckInputs(self): + def setup(self): """ fetch and verify inputs for all NetTests in the deck """ for net_test_loader in self.netTestLoaders: yield self.fetchAndVerifyNetTestInput(net_test_loader) + self.getRequiredTestHelpers() + yield self.lookupTestHelpers() @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): @@ -47,12 +56,37 @@ class Deck(object): for i in net_test_loader.inputFiles: if 'url' in i: log.debug("Downloading %s" % i['url']) - oonib = OONIBClient(i['address']) + oonibclient = OONIBClient(i['address']) + + try: + input_file = yield oonibclient.downloadInput(i['hash']) + except: + raise e.UnableToLoadDeckInput - input_file = yield oonib.downloadInput(i['hash']) try: input_file.verify() except AssertionError: - raise UnableToLoadDeckInput, cached_path + raise e.UnableToLoadDeckInput, cached_path i['test_class'].localOptions[i['key']] = input_file.cached_file + + def setNettestOptions(self): + for net_test_loader in self.netTestLoaders: + for th in net_test_loader.requiredTestHelpers: + test_helper_address = self.testHelpers[th['name']] + th['test_class'].localOptions[th['option']] = test_helper_address + net_test_loader.collector = self.collector + log.debug("Using %s: %s" % (test_helper_address, self.collector)) + + @defer.inlineCallbacks + def lookupTestHelpers(self): + log.msg("Looking up test helpers: %s" % self.testHelpers.keys()) + + required_test_helpers = self.testHelpers.keys() + if required_test_helpers: + oonibclient = OONIBClient(self.bouncer) + test_helpers = yield oonibclient.lookupTestHelpers(required_test_helpers) + self.collector = test_helpers['collector'] + for name in self.testHelpers.keys(): + self.testHelpers[name] = test_helpers[name] + self.setNettestOptions() diff --git a/ooni/errors.py b/ooni/errors.py index 94a83a4..5ad1940 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -170,3 +170,5 @@ class OONIBTestDetailsLookupError(OONIBReportError): class UnableToLoadDeckInput(Exception): pass +class CouldNotFindTestHelper(Exception): + pass diff --git a/ooni/nettest.py b/ooni/nettest.py index 1699f19..2b170d0 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -172,6 +172,7 @@ def getNetTestInformation(net_test_file): class NetTestLoader(object): method_prefix = 'test' + collector = None def __init__(self, options, test_file=None, test_string=None): self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]{64})$") @@ -185,7 +186,22 @@ class NetTestLoader(object): if test_cases: self.setupTestCases(test_cases) - + + @property + def requiredTestHelpers(self): + required_test_helpers = [] + if not self.testCases: + return required_test_helpers + + for test_class, test_methods in self.testCases: + for option, name in test_class.requiredTestHelpers.items(): + required_test_helpers.append({ + 'name': name, + 'option': option, + 'test_class': test_class + }) + return required_test_helpers + @property def inputFiles(self): input_files = [] @@ -255,6 +271,10 @@ class NetTestLoader(object): if (client_geodata and not config.privacy.includecountry) \ or ('countrycode' not in client_geodata): client_geodata['countrycode'] = None + + input_file_hashes = [] + for input_file in self.inputFiles: + input_file_hashes.append(input_file['hash']) test_details = {'start_time': time.time(), 'probe_asn': client_geodata['asn'], @@ -264,7 +284,8 @@ class NetTestLoader(object): 'test_version': self.testVersion, 'software_name': 'ooniprobe', 'software_version': software_version, - 'options': self.options + 'options': self.options, + 'input_hashes': input_file_hashes } return test_details @@ -613,7 +634,8 @@ class NetTestCase(object): optParameters = None baseParameters = None baseFlags = None - + + requiredTestHelpers = {} requiredOptions = [] requiresRoot = False diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 5861ff4..9bdce95 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -7,6 +7,7 @@ from twisted.internet import defer, reactor from ooni.utils.txagentwithsocks import Agent +from ooni import errors as e from ooni.settings import config from ooni.utils import log from ooni.utils.net import BodyReceiver, StringProducer, Downloader @@ -64,35 +65,46 @@ class InputFile(object): file_hash = sha256(f.read()) assert file_hash.hexdigest() == digest - def validate(self, policy): - """ - Validate this input file against the specified input policy. - """ - for input_file - class Collector(object): - def __init__(self): + def __init__(self, address): + self.address = address + self.nettest_policy = None self.input_policy = None + + @defer.inlineCallbacks + def loadPolicy(self): + # XXX implement caching of policies + oonibclient = OONIBClient(self.address) + log.msg("Looking up nettest policy for %s" % self.address) + self.nettest_policy = yield oonibclient.getNettestPolicy() + log.msg("Looking up input policy for %s" % self.address) + self.input_policy = yield oonibclient.getInputPolicy() def validateInput(self, input_hash): - pass + for i in self.input_policy: + if i['id'] == input_hash: + return True + return False - def validateNettest(self, nettest): - pass + def validateNettest(self, nettest_name): + for i in self.nettest_policy: + if nettest_name == i['name']: + return True + return False class OONIBClient(object): def __init__(self, address): self.address = address self.agent = Agent(reactor, sockshost="127.0.0.1", socksport=config.tor.socks_port) - self.input_files = {} def _request(self, method, urn, genReceiver, bodyProducer=None): finished = defer.Deferred() uri = self.address + urn - d = self.agent.request(method, uri, bodyProducer) + headers = {} + d = self.agent.request(method, uri, bodyProducer=bodyProducer) @d.addCallback def callback(response): @@ -108,7 +120,7 @@ class OONIBClient(object): def queryBackend(self, method, urn, query=None): bodyProducer = None if query: - bodyProducer = StringProducer(json.dumps(query), bodyProducer) + bodyProducer = StringProducer(json.dumps(query)) def genReceiver(finished, content_length): return BodyReceiver(finished, content_length, json.loads) @@ -125,9 +137,6 @@ class OONIBClient(object): def getNettestPolicy(self): pass - def queryBouncer(self, requested_helpers): - pass - def getInput(self, input_hash): input_file = InputFile(input_hash) if input_file.descriptorCached: @@ -176,3 +185,17 @@ class OONIBClient(object): def getNettestPolicy(self): return self.queryBackend('GET', '/policy/nettest') + + @defer.inlineCallbacks + def lookupTestHelpers(self, test_helper_names): + try: + test_helpers = yield self.queryBackend('POST', '/bouncer', + query={'test-helpers': test_helper_names}) + except Exception: + raise e.CouldNotFindTestHelper + + if not test_helpers: + raise e.CouldNotFindTestHelper + + defer.returnValue(test_helpers) + diff --git a/ooni/oonicli.py b/ooni/oonicli.py index ef6ef84..cf7d34d 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -36,8 +36,10 @@ class Options(usage.Options): optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], - ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion', - "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"], + ["collector", "c", None, + "Address of the collector of test results. This option should not be used, but you should always use a bouncer."], + ["bouncer", "b", 'httpo://nkvphnp3p6agi5qq.onion', + "Address of the bouncer for test helpers. default: httpo://nkvphnp3p6agi5qq.onion"], ["logfile", "l", None, "log file name"], ["pcapfile", "O", None, "pcap file name"], ["parallelism", "p", "10", "input parallelism"], @@ -125,7 +127,7 @@ def runWithDirector(): director = Director() d = director.start() - deck = Deck() + deck = Deck(global_options['bouncer']) if global_options['no-collector']: log.msg("Not reporting using a collector") collector = global_options['collector'] = None @@ -146,10 +148,10 @@ def runWithDirector(): log.err(e) print net_test_loader.usageOptions().getUsage() sys.exit(2) - - def fetch_nettest_inputs(result): + + def setup_nettest(_): try: - return deck.fetchAndVerifyDeckInputs() + return deck.setup() except errors.UnableToLoadDeckInput, e: return defer.failure.Failure(result) @@ -157,18 +159,30 @@ def runWithDirector(): log.err("Failed to start the director") r = failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress, - errors.UnableToLoadDeckInput) - if r == errors.TorNotRunning: + errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper) + + if isinstance(failure.value, errors.TorNotRunning): log.err("Tor does not appear to be running") log.err("Reporting with the collector %s is not possible" % global_options['collector']) log.msg("Try with a different collector or disable collector reporting with -n") - elif r == errors.InvalidOONIBCollectorAddress: + + elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress): log.err("Invalid format for oonib collector address.") log.msg("Should be in the format http://<collector_address>:<port>") log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") - elif r == errors.UnableToLoadDeckInput: - log.err('Missing required input files: %s' % failure) + + elif isinstance(failure.value, errors.UnableToLoadDeckInput): + log.err("Unable to fetch the required inputs for the test deck.") + log.msg("Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues") + + elif isinstance(failure.value, errors.CouldNotFindTestHelper): + log.err("Unable to obtain the required test helpers.") + log.msg("Try with a different bouncer or check that Tor is running properly.") + + if config.advanced.debug: + log.exception(failure) + reactor.stop() # Wait until director has started up (including bootstrapping Tor) @@ -188,8 +202,8 @@ def runWithDirector(): if not global_options['no-collector']: if global_options['collector']: collector = global_options['collector'] - elif net_test_loader.options['collector']: - collector = net_test_loader.options['collector'] + elif net_test_loader.collector: + collector = net_test_loader.collector if collector and collector.startswith('httpo:') \ and (not (config.tor_state or config.tor.socks_port)): @@ -213,7 +227,7 @@ def runWithDirector(): director.allTestsDone.addBoth(shutdown) def start(): - d.addCallback(fetch_nettest_inputs) + d.addCallback(setup_nettest) d.addCallback(post_director_start) d.addErrback(director_startup_failed) diff --git a/ooni/reporter.py b/ooni/reporter.py index 0c13cf2..8a89c8a 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -308,6 +308,7 @@ class OONIBReporter(OReporter): 'probe_asn': self.testDetails['probe_asn'], 'test_name': self.testDetails['test_name'], 'test_version': self.testDetails['test_version'], + 'input_hashes': self.testDetails['input_hashes'], # XXX there is a bunch of redundancy in the arguments getting sent # to the backend. This may need to get changed in the client and the # backend. @@ -328,7 +329,6 @@ class OONIBReporter(OReporter): bodyProducer=bodyProducer) except ConnectionRefusedError: log.err("Connection to reporting backend failed (ConnectionRefusedError)") - #yield defer.fail(OONIBReportCreationError()) raise errors.OONIBReportCreationError except errors.HostUnreachable: @@ -353,6 +353,12 @@ class OONIBReporter(OReporter): log.err("Failed to parse collector response") log.exception(e) raise errors.OONIBReportCreationError + + if response.code == 400: + # XXX make this more strict + log.err("The specified input or nettests cannot be submitted to this collector.") + log.msg("Try running a different test or try reporting to a different collector.") + raise errors.OONIBReportCreationError self.reportID = parsed_response['report_id'] self.backendVersion = parsed_response['backend_version'] diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index c039c65..1a1c4bc 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -1,10 +1,17 @@ +import os +import shutil import socket from twisted.trial import unittest from twisted.internet import defer +from ooni import errors as e +from ooni.utils import log +from ooni.settings import config from ooni.oonibclient import OONIBClient +data_dir = '/tmp/testooni' +config.advanced.data_dir = data_dir input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1' class TestOONIBClient(unittest.TestCase): @@ -15,6 +22,10 @@ class TestOONIBClient(unittest.TestCase): try: s.connect((host, port)) s.shutdown(2) + try: shutil.rmtree(data_dir) + except: pass + os.mkdir(data_dir) + os.mkdir(os.path.join(data_dir, 'inputs')) except Exception as ex: self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") self.oonibclient = OONIBClient('http://' + host + ':' + str(port)) @@ -51,6 +62,17 @@ class TestOONIBClient(unittest.TestCase): def test_download_deck(self): pass + def test_lookup_invalid_helpers(self): + return self.failUnlessFailure( + self.oonibclient.lookupTestHelpers( + ['dns', 'http-return-json-headers', 'sdadsadsa'] + ), e.CouldNotFindTestHelper) + + @defer.inlineCallbacks + def test_lookup_test_helpers(self): + helpers = yield self.oonibclient.lookupTestHelpers(['dns', 'http-return-json-headers']) + self.assertTrue(len(helpers) == 1) + @defer.inlineCallbacks def test_get_nettest_list(self): input_list = yield self.oonibclient.getInputList() diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 1ec9608..845cf4c 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -77,9 +77,12 @@ class BodyReceiver(protocol.Protocol): self.bytes_remaining -= len(b) def connectionLost(self, reason): - if self.body_processor: - self.data = self.body_processor(self.data) - self.finished.callback(self.data) + try: + if self.body_processor: + self.data = self.body_processor(self.data) + self.finished.callback(self.data) + except Exception as exc: + self.finished.errback(exc) class Downloader(protocol.Protocol): def __init__(self, download_path,

1 0

[ooni-probe/master] Only run unittests when there is something on port 8888
by art＠torproject.org 27 Aug '13

27 Aug '13

commit efa38f208504975054e233aaba3dd92aa3b8b51b Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 18:36:11 2013 +0200 Only run unittests when there is something on port 8888 --- ooni/tests/test_oonibclient.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index 26e2f62..47a3d47 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -1,3 +1,5 @@ +import socket + from twisted.trial import unittest from twisted.internet import defer @@ -7,7 +9,15 @@ input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1' class TestOONIBClient(unittest.TestCase): def setUp(self): - self.oonibclient = OONIBClient('http://127.0.0.1:8888') + host = '127.0.0.1' + port = 8888 + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect((host, port)) + s.shutdown(2) + except Exception as ex: + self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") + self.oonibclient = OONIBClient('http://'+host+':'+str(port)) @defer.inlineCallbacks def test_query(self):

1 0

[ooni-probe/master] Use error-code 406 to indicate policy rejection
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 1ad69bfd531c1d6cd6c39272848b85b7c825356d Author: aagbsn <aagbsn(a)extc.org> Date: Sat Aug 24 18:32:12 2013 +0200 Use error-code 406 to indicate policy rejection --- ooni/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 8a89c8a..67678a6 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -354,7 +354,7 @@ class OONIBReporter(OReporter): log.exception(e) raise errors.OONIBReportCreationError - if response.code == 400: + if response.code == 406: # XXX make this more strict log.err("The specified input or nettests cannot be submitted to this collector.") log.msg("Try running a different test or try reporting to a different collector.")

1 0

[ooni-probe/master] Fix bug that leads decks not being started properly
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 8cdae801fb9414eeba62c1e2c6215d0eef7754b6 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 18:29:56 2013 +0200 Fix bug that leads decks not being started properly --- ooni/oonicli.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index a7df18f..ef6ef84 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -208,9 +208,8 @@ def runWithDirector(): raise e log.debug("adding callback for startNetTest") - @d.addCallback - def cb(res): - director.startNetTest(net_test_loader, reporters) + director.startNetTest(net_test_loader, reporters) + director.allTestsDone.addBoth(shutdown) def start():

1 0

[ooni-probe/master] Reschedule failures immediately
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 7ea42ac6815d46329d979cf6e93f750a440b4166 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 25 14:50:41 2013 +0200 Reschedule failures immediately Reschedule failures immediately so they do not accrue at the end of the task list and eat memory. --- ooni/managers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/managers.py b/ooni/managers.py index cc2d067..36b791e 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -37,8 +37,8 @@ class TaskManager(object): if task.failures <= self.retries: log.debug("Rescheduling...") - self._tasks = itertools.chain(self._tasks, - makeIterable(task)) + self._tasks = itertools.chain(makeIterable(task), self._tasks) + else: # This fires the errback when the task is done but has failed. log.err('Permanent failure for %s' % task)

1 0