commit 3bface3bd4fab9e87bdc8c440d70e07e19b8af28 Author: Arturo Filastò art@fuffa.org Date: Fri Aug 23 16:46:15 2013 +0200

Add support for looking up test helpers via a bouncer. --- data/nettests/blocking/dnsconsistency.py | 4 +- data/nettests/manipulation/dnsspoof.py | 1 + .../manipulation/http_header_field_manipulation.py | 1 + data/nettests/manipulation/http_host.py | 1 + .../manipulation/http_invalid_request_line.py | 4 +- data/nettests/manipulation/traceroute.py | 3 +- decks/basic.deck | 2 +- ooni/deck.py | 50 +++++++++++++++--- ooni/errors.py | 2 + ooni/nettest.py | 28 ++++++++-- ooni/oonibclient.py | 55 ++++++++++++++------ ooni/oonicli.py | 42 ++++++++++----- ooni/reporter.py | 8 ++- ooni/tests/test_oonibclient.py | 22 ++++++++ ooni/utils/net.py | 9 ++-- 15 files changed, 183 insertions(+), 49 deletions(-)

diff --git a/data/nettests/blocking/dnsconsistency.py b/data/nettests/blocking/dnsconsistency.py index 7b6e7b9..3c88cd2 100644 --- a/data/nettests/blocking/dnsconsistency.py +++ b/data/nettests/blocking/dnsconsistency.py @@ -38,12 +38,14 @@ class DNSConsistencyTest(dnst.DNSTest):

name = "DNS Consistency" description = "DNS censorship detection test" - version = "0.5" + version = "0.6" authors = "Arturo Filastò, Isis Lovecruft" requirements = None

inputFile = ['file', 'f', None, 'Input file of list of hostnames to attempt to resolve'] + + requiredTestHelpers = {'backend': 'dns'}

usageOptions = UsageOptions requiredOptions = ['backend', 'file'] diff --git a/data/nettests/manipulation/dnsspoof.py b/data/nettests/manipulation/dnsspoof.py index 5c50c2f..c9120a4 100644 --- a/data/nettests/manipulation/dnsspoof.py +++ b/data/nettests/manipulation/dnsspoof.py @@ -22,6 +22,7 @@ class DNSSpoof(scapyt.ScapyTest):

usageOptions = UsageOptions

+ requiredTestHelpers = {'backend': 'dns'} requiredOptions = ['hostname', 'resolver']

def setUp(self): diff --git a/data/nettests/manipulation/http_header_field_manipulation.py b/data/nettests/manipulation/http_header_field_manipulation.py index 509f4ef..3423442 100644 --- a/data/nettests/manipulation/http_header_field_manipulation.py +++ b/data/nettests/manipulation/http_header_field_manipulation.py @@ -48,6 +48,7 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): randomizeUA = False usageOptions = UsageOptions

+ requiredTestHelpers = {'backend': 'http-return-json-headers'} requiredOptions = ['backend']

def get_headers(self): diff --git a/data/nettests/manipulation/http_host.py b/data/nettests/manipulation/http_host.py index 3f1e0c6..2ec517c 100644 --- a/data/nettests/manipulation/http_host.py +++ b/data/nettests/manipulation/http_host.py @@ -43,6 +43,7 @@ class HTTPHost(httpt.HTTPTest): inputFile = ['file', 'f', None, 'List of hostnames to test for censorship']

+ requiredTestHelpers = {'backend': 'http-return-json-headers'} requiredOptions = ['backend']

def test_filtering_prepend_newline_to_method(self): diff --git a/data/nettests/manipulation/http_invalid_request_line.py b/data/nettests/manipulation/http_invalid_request_line.py index 2482282..64dbcac 100644 --- a/data/nettests/manipulation/http_invalid_request_line.py +++ b/data/nettests/manipulation/http_invalid_request_line.py @@ -20,10 +20,12 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): ascii letters or numbers ('XxXx' will be 4). """ name = "HTTP Invalid Request Line" - version = "0.1.4" + version = "0.2" authors = "Arturo Filastò"

usageOptions = UsageOptions + + requiredTestHelpers = {'backend': 'tcp-echo'} requiredOptions = ['backend']

def setUp(self): diff --git a/data/nettests/manipulation/traceroute.py b/data/nettests/manipulation/traceroute.py index 3f6f17b..2db1826 100644 --- a/data/nettests/manipulation/traceroute.py +++ b/data/nettests/manipulation/traceroute.py @@ -23,8 +23,9 @@ class UsageOptions(usage.Options): class TracerouteTest(scapyt.BaseScapyTest): name = "Multi Protocol Traceroute Test" author = "Arturo Filastò" - version = "0.1.1" + version = "0.2"

+ requiredTestHelpers = {'backend': 'traceroute'} usageOptions = UsageOptions dst_ports = [0, 22, 23, 53, 80, 123, 443, 8080, 65535]

diff --git a/decks/basic.deck b/decks/basic.deck index c784cd5..84a8f9d 100644 --- a/decks/basic.deck +++ b/decks/basic.deck @@ -7,7 +7,7 @@ pcapfile: null reportfile: null resume: 0 - subargs: [-f, 'httpo://3mr5phzltfzqgh6y.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] + subargs: [-f, 'httpo://amcq5ldf3vwg22ze.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'] test_file: data/nettests/blocking/http_requests.py testdeck: null - options: diff --git a/ooni/deck.py b/ooni/deck.py index 8f1335d..1ad8377 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -4,7 +4,7 @@ from ooni.nettest import NetTestLoader from ooni.settings import config from ooni.utils import log from ooni.utils.txagentwithsocks import Agent -from ooni.errors import UnableToLoadDeckInput +from ooni import errors as e from ooni.oonibclient import OONIBClient

from twisted.internet import reactor, defer @@ -14,9 +14,12 @@ import re import yaml

class Deck(object): - def __init__(self, deckFile=None): + def __init__(self, bouncer, deckFile=None): + self.bouncer = bouncer self.netTestLoaders = [] self.inputs = [] + self.testHelpers = {} + self.collector = None

if deckFile: self.loadDeck(deckFile)

@@ -31,14 +34,20 @@ class Deck(object): def insert(self, net_test_loader): """ Add a NetTestLoader to this test deck """ net_test_loader.checkOptions() - self.fetchAndVerifyNetTestInput(net_test_loader) self.netTestLoaders.append(net_test_loader) - + + def getRequiredTestHelpers(self): + for net_test_loader in self.netTestLoaders: + for test_helper in net_test_loader.requiredTestHelpers: + self.testHelpers[test_helper['name']] = None + @defer.inlineCallbacks - def fetchAndVerifyDeckInputs(self): + def setup(self): """ fetch and verify inputs for all NetTests in the deck """ for net_test_loader in self.netTestLoaders: yield self.fetchAndVerifyNetTestInput(net_test_loader) + self.getRequiredTestHelpers() + yield self.lookupTestHelpers()

@defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): @@ -47,12 +56,37 @@ class Deck(object): for i in net_test_loader.inputFiles: if 'url' in i: log.debug("Downloading %s" % i['url']) - oonib = OONIBClient(i['address']) + oonibclient = OONIBClient(i['address']) + + try: + input_file = yield oonibclient.downloadInput(i['hash']) + except: + raise e.UnableToLoadDeckInput

- input_file = yield oonib.downloadInput(i['hash']) try: input_file.verify() except AssertionError: - raise UnableToLoadDeckInput, cached_path + raise e.UnableToLoadDeckInput, cached_path

i['test_class'].localOptions[i['key']] = input_file.cached_file + + def setNettestOptions(self): + for net_test_loader in self.netTestLoaders: + for th in net_test_loader.requiredTestHelpers: + test_helper_address = self.testHelpers[th['name']] + th['test_class'].localOptions[th['option']] = test_helper_address + net_test_loader.collector = self.collector + log.debug("Using %s: %s" % (test_helper_address, self.collector)) + + @defer.inlineCallbacks + def lookupTestHelpers(self): + log.msg("Looking up test helpers: %s" % self.testHelpers.keys()) + + required_test_helpers = self.testHelpers.keys() + if required_test_helpers: + oonibclient = OONIBClient(self.bouncer) + test_helpers = yield oonibclient.lookupTestHelpers(required_test_helpers) + self.collector = test_helpers['collector'] + for name in self.testHelpers.keys(): + self.testHelpers[name] = test_helpers[name] + self.setNettestOptions() diff --git a/ooni/errors.py b/ooni/errors.py index 94a83a4..5ad1940 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -170,3 +170,5 @@ class OONIBTestDetailsLookupError(OONIBReportError):

class UnableToLoadDeckInput(Exception): pass +class CouldNotFindTestHelper(Exception): + pass diff --git a/ooni/nettest.py b/ooni/nettest.py index 1699f19..2b170d0 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -172,6 +172,7 @@ def getNetTestInformation(net_test_file):

class NetTestLoader(object): method_prefix = 'test' + collector = None

def __init__(self, options, test_file=None, test_string=None): self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}.onion)/input/([a-z0-9]{64})$") @@ -185,7 +186,22 @@ class NetTestLoader(object):

if test_cases: self.setupTestCases(test_cases) - + + @property + def requiredTestHelpers(self): + required_test_helpers = [] + if not self.testCases: + return required_test_helpers + + for test_class, test_methods in self.testCases: + for option, name in test_class.requiredTestHelpers.items(): + required_test_helpers.append({ + 'name': name, + 'option': option, + 'test_class': test_class + }) + return required_test_helpers + @property def inputFiles(self): input_files = [] @@ -255,6 +271,10 @@ class NetTestLoader(object): if (client_geodata and not config.privacy.includecountry) \ or ('countrycode' not in client_geodata): client_geodata['countrycode'] = None + + input_file_hashes = [] + for input_file in self.inputFiles: + input_file_hashes.append(input_file['hash'])

test_details = {'start_time': time.time(), 'probe_asn': client_geodata['asn'], @@ -264,7 +284,8 @@ class NetTestLoader(object): 'test_version': self.testVersion, 'software_name': 'ooniprobe', 'software_version': software_version, - 'options': self.options + 'options': self.options, + 'input_hashes': input_file_hashes } return test_details

@@ -613,7 +634,8 @@ class NetTestCase(object): optParameters = None baseParameters = None baseFlags = None - + + requiredTestHelpers = {} requiredOptions = [] requiresRoot = False

diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 5861ff4..9bdce95 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -7,6 +7,7 @@ from twisted.internet import defer, reactor

from ooni.utils.txagentwithsocks import Agent

+from ooni import errors as e from ooni.settings import config from ooni.utils import log from ooni.utils.net import BodyReceiver, StringProducer, Downloader @@ -64,35 +65,46 @@ class InputFile(object): file_hash = sha256(f.read()) assert file_hash.hexdigest() == digest

- def validate(self, policy): - """ - Validate this input file against the specified input policy. - """ - for input_file - class Collector(object): - def __init__(self): + def __init__(self, address): + self.address = address + self.nettest_policy = None self.input_policy = None + + @defer.inlineCallbacks + def loadPolicy(self): + # XXX implement caching of policies + oonibclient = OONIBClient(self.address) + log.msg("Looking up nettest policy for %s" % self.address) + self.nettest_policy = yield oonibclient.getNettestPolicy() + log.msg("Looking up input policy for %s" % self.address) + self.input_policy = yield oonibclient.getInputPolicy()

def validateInput(self, input_hash): - pass + for i in self.input_policy: + if i['id'] == input_hash: + return True + return False

- def validateNettest(self, nettest): - pass + def validateNettest(self, nettest_name): + for i in self.nettest_policy: + if nettest_name == i['name']: + return True + return False

class OONIBClient(object): def __init__(self, address): self.address = address self.agent = Agent(reactor, sockshost="127.0.0.1", socksport=config.tor.socks_port) - self.input_files = {}

def _request(self, method, urn, genReceiver, bodyProducer=None): finished = defer.Deferred()

uri = self.address + urn - d = self.agent.request(method, uri, bodyProducer) + headers = {} + d = self.agent.request(method, uri, bodyProducer=bodyProducer)

@d.addCallback def callback(response): @@ -108,7 +120,7 @@ class OONIBClient(object): def queryBackend(self, method, urn, query=None): bodyProducer = None if query: - bodyProducer = StringProducer(json.dumps(query), bodyProducer) + bodyProducer = StringProducer(json.dumps(query))

def genReceiver(finished, content_length): return BodyReceiver(finished, content_length, json.loads) @@ -125,9 +137,6 @@ class OONIBClient(object): def getNettestPolicy(self): pass

- def queryBouncer(self, requested_helpers): - pass - def getInput(self, input_hash): input_file = InputFile(input_hash) if input_file.descriptorCached: @@ -176,3 +185,17 @@ class OONIBClient(object):

def getNettestPolicy(self): return self.queryBackend('GET', '/policy/nettest') + + @defer.inlineCallbacks + def lookupTestHelpers(self, test_helper_names): + try: + test_helpers = yield self.queryBackend('POST', '/bouncer', + query={'test-helpers': test_helper_names}) + except Exception: + raise e.CouldNotFindTestHelper + + if not test_helpers: + raise e.CouldNotFindTestHelper + + defer.returnValue(test_helpers) + diff --git a/ooni/oonicli.py b/ooni/oonicli.py index ef6ef84..cf7d34d 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -36,8 +36,10 @@ class Options(usage.Options): optParameters = [["reportfile", "o", None, "report file name"], ["testdeck", "i", None, "Specify as input a test deck: a yaml file containig the tests to run an their arguments"], - ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion', - "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"], + ["collector", "c", None, + "Address of the collector of test results. This option should not be used, but you should always use a bouncer."], + ["bouncer", "b", 'httpo://nkvphnp3p6agi5qq.onion', + "Address of the bouncer for test helpers. default: httpo://nkvphnp3p6agi5qq.onion"], ["logfile", "l", None, "log file name"], ["pcapfile", "O", None, "pcap file name"], ["parallelism", "p", "10", "input parallelism"], @@ -125,7 +127,7 @@ def runWithDirector(): director = Director() d = director.start()

- deck = Deck() + deck = Deck(global_options['bouncer']) if global_options['no-collector']: log.msg("Not reporting using a collector") collector = global_options['collector'] = None @@ -146,10 +148,10 @@ def runWithDirector(): log.err(e) print net_test_loader.usageOptions().getUsage() sys.exit(2) - - def fetch_nettest_inputs(result): + + def setup_nettest(_): try: - return deck.fetchAndVerifyDeckInputs() + return deck.setup() except errors.UnableToLoadDeckInput, e: return defer.failure.Failure(result)

@@ -157,18 +159,30 @@ def runWithDirector(): log.err("Failed to start the director") r = failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress, - errors.UnableToLoadDeckInput) - if r == errors.TorNotRunning: + errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper) + + if isinstance(failure.value, errors.TorNotRunning): log.err("Tor does not appear to be running") log.err("Reporting with the collector %s is not possible" % global_options['collector']) log.msg("Try with a different collector or disable collector reporting with -n") - elif r == errors.InvalidOONIBCollectorAddress: + + elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress): log.err("Invalid format for oonib collector address.") log.msg("Should be in the format http://<collector_address>:<port>") log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") - elif r == errors.UnableToLoadDeckInput: - log.err('Missing required input files: %s' % failure) + + elif isinstance(failure.value, errors.UnableToLoadDeckInput): + log.err("Unable to fetch the required inputs for the test deck.") + log.msg("Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues") + + elif isinstance(failure.value, errors.CouldNotFindTestHelper): + log.err("Unable to obtain the required test helpers.") + log.msg("Try with a different bouncer or check that Tor is running properly.") + + if config.advanced.debug: + log.exception(failure) + reactor.stop()

# Wait until director has started up (including bootstrapping Tor) @@ -188,8 +202,8 @@ def runWithDirector(): if not global_options['no-collector']: if global_options['collector']: collector = global_options['collector'] - elif net_test_loader.options['collector']: - collector = net_test_loader.options['collector'] + elif net_test_loader.collector: + collector = net_test_loader.collector

if collector and collector.startswith('httpo:') \ and (not (config.tor_state or config.tor.socks_port)): @@ -213,7 +227,7 @@ def runWithDirector(): director.allTestsDone.addBoth(shutdown)

def start(): - d.addCallback(fetch_nettest_inputs) + d.addCallback(setup_nettest) d.addCallback(post_director_start) d.addErrback(director_startup_failed)

diff --git a/ooni/reporter.py b/ooni/reporter.py index 0c13cf2..8a89c8a 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -308,6 +308,7 @@ class OONIBReporter(OReporter): 'probe_asn': self.testDetails['probe_asn'], 'test_name': self.testDetails['test_name'], 'test_version': self.testDetails['test_version'], + 'input_hashes': self.testDetails['input_hashes'], # XXX there is a bunch of redundancy in the arguments getting sent # to the backend. This may need to get changed in the client and the # backend. @@ -328,7 +329,6 @@ class OONIBReporter(OReporter): bodyProducer=bodyProducer) except ConnectionRefusedError: log.err("Connection to reporting backend failed (ConnectionRefusedError)") - #yield defer.fail(OONIBReportCreationError()) raise errors.OONIBReportCreationError

except errors.HostUnreachable: @@ -353,6 +353,12 @@ class OONIBReporter(OReporter): log.err("Failed to parse collector response") log.exception(e) raise errors.OONIBReportCreationError + + if response.code == 400: + # XXX make this more strict + log.err("The specified input or nettests cannot be submitted to this collector.") + log.msg("Try running a different test or try reporting to a different collector.") + raise errors.OONIBReportCreationError

self.reportID = parsed_response['report_id'] self.backendVersion = parsed_response['backend_version'] diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py index c039c65..1a1c4bc 100644 --- a/ooni/tests/test_oonibclient.py +++ b/ooni/tests/test_oonibclient.py @@ -1,10 +1,17 @@ +import os +import shutil import socket

from twisted.trial import unittest from twisted.internet import defer

+from ooni import errors as e +from ooni.utils import log +from ooni.settings import config from ooni.oonibclient import OONIBClient

+data_dir = '/tmp/testooni' +config.advanced.data_dir = data_dir input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'

class TestOONIBClient(unittest.TestCase): @@ -15,6 +22,10 @@ class TestOONIBClient(unittest.TestCase): try: s.connect((host, port)) s.shutdown(2) + try: shutil.rmtree(data_dir) + except: pass + os.mkdir(data_dir) + os.mkdir(os.path.join(data_dir, 'inputs')) except Exception as ex: self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)") self.oonibclient = OONIBClient('http://' + host + ':' + str(port)) @@ -51,6 +62,17 @@ class TestOONIBClient(unittest.TestCase): def test_download_deck(self): pass

+ def test_lookup_invalid_helpers(self): + return self.failUnlessFailure( + self.oonibclient.lookupTestHelpers( + ['dns', 'http-return-json-headers', 'sdadsadsa'] + ), e.CouldNotFindTestHelper) + + @defer.inlineCallbacks + def test_lookup_test_helpers(self): + helpers = yield self.oonibclient.lookupTestHelpers(['dns', 'http-return-json-headers']) + self.assertTrue(len(helpers) == 1) + @defer.inlineCallbacks def test_get_nettest_list(self): input_list = yield self.oonibclient.getInputList() diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 1ec9608..845cf4c 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -77,9 +77,12 @@ class BodyReceiver(protocol.Protocol): self.bytes_remaining -= len(b)

def connectionLost(self, reason): - if self.body_processor: - self.data = self.body_processor(self.data) - self.finished.callback(self.data) + try: + if self.body_processor: + self.data = self.body_processor(self.data) + self.finished.callback(self.data) + except Exception as exc: + self.finished.errback(exc)

class Downloader(protocol.Protocol): def __init__(self, download_path,