August 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge branch 'master' into fix/link_task_schedulers
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 25c87329e6a4d349709d890928fabad707c29bf3 Merge: 403d60e 71552b0 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 11:36:47 2013 +0200 Merge branch 'master' into fix/link_task_schedulers * master: (36 commits) Clean up error messages Handle more failure types Properly handle the case when we don't get a collector by asking a bouncer Track the number of failed reporters Fix report write failure bug Track the number of failures, not the instances. Reschedule failures immediately Link TaskManagers by Least Available Slots Add CouldNotFindTestCollector error Fix some issues with ooni.deck Simplify the bouncer API Use error-code 406 to indicate policy rejection Add support for looking up test helpers via a bouncer. Add component.json file to ng-upload Update LICENSE file Start outlining the classes for policy handling Only run unittests when there is something on port 8888 Add a basic deck that includes inputs as URLs Fix bug that leads decks not being started properly Implement downloading of input files based on URL in deck file ... Conflicts: ooni/templates/httpt.py .travis.yml | 1 - LICENSE | 2 +- data/nettests/blocking/dnsconsistency.py | 4 +- data/nettests/blocking/http_requests.py | 1 - data/nettests/manipulation/dnsspoof.py | 1 + .../manipulation/http_header_field_manipulation.py | 1 + data/nettests/manipulation/http_host.py | 1 + .../manipulation/http_invalid_request_line.py | 4 +- data/nettests/manipulation/traceroute.py | 3 +- data/ui/app/libs/ng-upload/component.json | 23 +++ decks/basic.deck | 24 +++ ooni/deck.py | 87 ++++++++ ooni/director.py | 1 + ooni/errors.py | 7 + ooni/nettest.py | 81 +++++++- ooni/oonibclient.py | 212 ++++++++++++++++++++ ooni/oonicli.py | 133 ++++++++---- ooni/reporter.py | 49 ++++- ooni/tasks.py | 7 +- ooni/templates/httpt.py | 8 +- ooni/tests/test_nettest.py | 3 + ooni/tests/test_oonibclient.py | 85 ++++++++ ooni/utils/net.py | 30 ++- 23 files changed, 701 insertions(+), 67 deletions(-)

1 0

[ooni-probe/master] Merge branch 'fix/link_task_schedulers' of https://github.com/aagbsn/ooni-probe into fix/link_task_schedulers
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 403d60ef1545c70162d24b4126524da494de6e0e Merge: f8f2890 23251cf Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 11:23:38 2013 +0200 Merge branch 'fix/link_task_schedulers' of https://github.com/aagbsn/ooni-probe into fix/link_task_schedulers * 'fix/link_task_schedulers' of https://github.com/aagbsn/ooni-probe: Track the number of failures, not the instances. Reschedule failures immediately ooni/managers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)

1 0

[ooni-probe/master] Fix unittests for managers.
by art＠torproject.org 27 Aug '13

27 Aug '13

commit e1364925415fbb1946cf11f0d58c01551a8ea1cc Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 11:59:34 2013 +0200 Fix unittests for managers. failures is now an int and not a list --- ooni/tests/test_managers.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/ooni/tests/test_managers.py b/ooni/tests/test_managers.py index c290155..8265d8f 100644 --- a/ooni/tests/test_managers.py +++ b/ooni/tests/test_managers.py @@ -52,16 +52,20 @@ class TestTaskManager(unittest.TestCase): for x in range(number): mock_task = task_type() all_done.append(mock_task.done) + mock_task.done.addErrback(lambda x: None) self.measurementManager.schedule(mock_task) d = defer.DeferredList(all_done) @d.addCallback def done(res): # 10*2 because 2 is the number of retries - self.assertEqual(len(self.measurementManager.failures), number*3) - for task_result, task_instance in self.measurementManager.failures: - self.assertEqual(task_result, mockFailure) - self.assertIsInstance(task_instance, task_type) + self.assertEqual(self.measurementManager.failures, number*3) + # XXX @aagbsn is there a reason why you switched to using an int + # over a using a list? + # self.assertEqual(len(self.measurementManager.failures), number*3) + # for task_result, task_instance in self.measurementManager.failures: + # self.assertEqual(task_result, mockFailure) + # self.assertIsInstance(task_instance, task_type) return d @@ -93,9 +97,10 @@ class TestTaskManager(unittest.TestCase): @mock_task.done.addBoth def done(res): - self.assertEqual(len(self.measurementManager.failures), 1) - for task_result, task_instance in self.measurementManager.failures: - self.assertIsInstance(task_instance, task_type) + self.assertEqual(self.measurementManager.failures, 1) + # self.assertEqual(len(self.measurementManager.failures), 1) + # for task_result, task_instance in self.measurementManager.failures: + # self.assertIsInstance(task_instance, task_type) return mock_task.done @@ -113,9 +118,10 @@ class TestTaskManager(unittest.TestCase): @mock_task.done.addBoth def done(res): - self.assertEqual(len(self.measurementManager.failures), 1) - for task_result, task_instance in self.measurementManager.failures: - self.assertIsInstance(task_instance, task_type) + self.assertEqual(self.measurementManager.failures, 1) + #self.assertEqual(len(self.measurementManager.failures), 1) + # for task_result, task_instance in self.measurementManager.failures: + # self.assertIsInstance(task_instance, task_type) for task_result, task_instance in self.measurementManager.successes: self.assertEqual(task_result, 42) @@ -148,10 +154,10 @@ class TestTaskManager(unittest.TestCase): @mock_task.done.addCallback def done(res): - self.assertEqual(len(self.measurementManager.failures), 1) - - self.assertEqual(self.measurementManager.failures, - [(mockFailure, mock_task)]) + self.assertEqual(self.measurementManager.failures, 1) + #self.assertEqual(len(self.measurementManager.failures), 1) + # self.assertEqual(self.measurementManager.failures, + # [(mockFailure, mock_task)]) self.assertEqual(self.measurementManager.successes, [(42, mock_task)]) @@ -172,8 +178,8 @@ class TestTaskManager(unittest.TestCase): @d.addCallback def done(res): - self.assertEqual(len(self.measurementManager.failures), number) - + self.assertEqual(self.measurementManager.failures, number) + #self.assertEqual(len(self.measurementManager.failures), number) for task_result, task_instance in self.measurementManager.successes: self.assertEqual(task_result, 42) self.assertIsInstance(task_instance, MockFailOnceTask) @@ -213,11 +219,10 @@ class TestMeasurementManager(unittest.TestCase): @mock_task.done.addErrback def done(failure): - self.assertEqual(len(self.measurementManager.failures), 3) + self.assertEqual(self.measurementManager.failures, 3) + #self.assertEqual(len(self.measurementManager.failures), 3) self.assertEqual(failure, mockFailure) self.assertEqual(len(self.mockNetTest.successes), 0) return mock_task.done - -

1 0

[translation/https_everywhere] Update translations for https_everywhere
by translation＠torproject.org 27 Aug '13

27 Aug '13

commit 2561c73b8e31670fa92905c6c71b7b1602b034d3 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Aug 27 09:46:13 2013 +0000 Update translations for https_everywhere --- ru/https-everywhere.dtd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ru/https-everywhere.dtd b/ru/https-everywhere.dtd index 0f9bdc4..2f33090 100644 --- a/ru/https-everywhere.dtd +++ b/ru/https-everywhere.dtd @@ -36,7 +36,7 @@ <!ENTITY https-everywhere.popup.title ""> <!ENTITY https-everywhere.popup.paragraph1 ""> -<!ENTITY https-everywhere.popup.paragraph2 ""> -<!ENTITY https-everywhere.popup.paragraph3 ""> -<!ENTITY https-everywhere.popup.keep ""> -<!ENTITY https-everywhere.popup.revert ""> +<!ENTITY https-everywhere.popup.paragraph2 "Хотели бы Вы вернуться к стабильной версии?"> +<!ENTITY https-everywhere.popup.paragraph3 "Мы были бы рады, если бы Вы продолжили использовать разрабатываемую версию, и помогли нам сделать HTTPS Everywhere лучше! Вы могли бы найти несколько ошибок, и прислать их нам на https-everywhere(a)eff.org. Извините за неудобства, и спасибо за использование HTTPS Everywhere."> +<!ENTITY https-everywhere.popup.keep "Остаться на разрабатываемой версии"> +<!ENTITY https-everywhere.popup.revert "Загрузить последнюю стабильную версию">

1 0

[ooni-probe/master] Add UnableToLoadDeckInput Exception
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 657044e5a39bf655b9ec3590a92f72ef9b146fd7 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 28 00:57:07 2013 +0200 Add UnableToLoadDeckInput Exception --- ooni/errors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/errors.py b/ooni/errors.py index c9d5fbe..94a83a4 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -168,4 +168,5 @@ class OONIBReportCreationError(OONIBReportError): class OONIBTestDetailsLookupError(OONIBReportError): pass - +class UnableToLoadDeckInput(Exception): + pass

1 0

[ooni-probe/master] Move input checking out of checkOptions
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 0debe3d2ee452db4f86bd4b2f252cb90feeaa3bd Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 28 00:57:21 2013 +0200 Move input checking out of checkOptions Relocates input checking as late as possible so that inputs to NetTest can be downloaded after instancing a NetTestLoader. --- ooni/nettest.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 10eacd3..92c9864 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -360,11 +360,6 @@ class NetTestLoader(object): test_instance._checkRequiredOptions() test_instance._checkValidOptions() - inputs = test_instance.getInputProcessor() - if not inputs: - inputs = [None] - klass.inputs = inputs - def _get_test_methods(self, item): """ Look for test_ methods in subclasses of NetTestCase @@ -501,7 +496,14 @@ class NetTest(object): This is a generator that yields measurements and registers the callbacks for when a measurement is successful or has failed. """ + for test_class, test_methods in self.testCases: + # load the input processor as late as possible + inputs = test_class().getInputProcessor() + if not inputs: + inputs = [None] + test_class.inputs = inputs + for input in test_class.inputs: for method in test_methods: log.debug("Running %s %s" % (test_class, method))

1 0

[ooni-probe/master] Adds class TestDeck
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 10f2d1c93ee47c0db7713466e4373cee6148294a Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 28 00:58:49 2013 +0200 Adds class TestDeck Provides a container for NetTests and helper functionality to download and verify input arguments if specified by URL. --- ooni/deck.py | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/ooni/deck.py b/ooni/deck.py new file mode 100644 index 0000000..02c8fc1 --- /dev/null +++ b/ooni/deck.py @@ -0,0 +1,116 @@ +#-*- coding: utf-8 -*- + +from hashlib import sha1 +from ooni.nettest import NetTestLoader +from ooni.settings import config +from ooni.utils.txagentwithsocks import Agent +from ooni.errors import UnableToLoadDeckInput +from twisted.internet import reactor, defer +import os +import re +import yaml + +onionInputRegex = re.compile("httpo://[a-z0-9]{16}\.onion/input/([a-z0-9]){40}$") + +(a)defer.inlineCallbacks +def downloadFile(fileURL, filePath): + + def writeFile(response_body, filePath): + f = open(filePath, 'w+') + f.write(response_body) + + finished = defer.Deferred() + finished.addCallback(writeFile, filePath) + + agent = Agent(reactor, sockshost="127.0.0.1", + socksport=int(config.tor.socks_port)) + response = yield agent.request("GET", fileURL) + response.deliverBody(BodyReceiver(finished)) + +def verifyFile(filePath): + # get the filename component of the file path + digest = os.path.basename(filePath) + with open(filePath) as f: + sha1digest = sha1(f.read()) + return sha1digest.hexdigest() == digest + return False + +class TestDeck(object): + def __init__(self, deckFile=None): + self.netTestLoaders = [] + self.inputs = [] + if deckFile: self.loadDeck(deckFile) + + def loadDeck(self, deckFile): + test_deck = yaml.safe_load(open(deckFile)) + for test in test_deck: + net_test_loader = NetTestLoader(test['options']['subargs'], + test_file=test['options']['test_file']) + net_test_loader.checkOptions() + self.netTestLoaders.append(net_test_loader) + + def insert(self, net_test_loader): + """ Add a NetTestLoader to this test deck """ + net_test_loader.checkOptions() + self.fetchAndVerifyNetTestInput(net_test_loader) + self.netTestLoaders.append(net_test_loader) + + def fetchAndVerifyDeckInputs(self): + """ fetch and verify inputs for all NetTests in the deck """ + for net_test_loader in self.netTestLoaders: + self.fetchAndVerifyNetTestInput(net_test_loader) + + @defer.inlineCallbacks + def fetchAndVerifyNetTestInput(self, net_test_loader): + """ fetch and verify a single NetTest's inputs """ + for test_class, test_methods in net_test_loader.testCases: + if test_class.inputFile: + inputArg = test_class.inputFile[0] + inputFileURL = test_class.localOptions[inputArg] + + m = onionInputRegex.match(inputFileURL) + if m: + fileDigest = m.group(1) + else: + fileDigest = os.path.basename(inputFileURL) + + cachedInputDir = os.path.join(config.advanced.data_dir, + 'inputs') + cachedPath = os.path.join(cachedInputDir, fileDigest) + self.inputs.append(cachedPath) + + if os.path.exists(cachedPath) and verifyFile(cachedPath): + test_class.localOptions[inputArg] = cachedPath + continue + if m: + yield downloadFile(inputFileURL, cachedPath) + if verifyFile(cachedPath): + test_class.localOptions[inputArg] = cachedPath + continue + + raise UnableToLoadDeckInput, cachedPath + +def test_verify_file_success(): + f = open('/dev/urandom') + r = f.read(1024*1024) + z = sha1(f).hexdigest() + f.close() + fn = '/tmp/%s' % z + f = open(fn) + f.write(r) + f.close() + verifyFile(fn) + os.unlink(fn) + +def test_verify_file_failure(): + pass +def test_load_deck_with_no_inputs(deck): + pass +def test_load_deck_with_cached_input(deckFile): + pass +def test_load_deck_with_evil_path(deckFile): + pass +def test_load_deck_with_download_success(deckFile): + pass +def test_load_deck_with_download_failure(deckFile): + pass

1 0

[ooni-probe/master] Make oonicli download inputs before starting tests.
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 0fd667f224acb19dbb925ca17aaec20b4e351519 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jun 28 01:00:08 2013 +0200 Make oonicli download inputs before starting tests. --- ooni/oonicli.py | 98 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 59 insertions(+), 39 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 218ec2e..2a141f6 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -14,6 +14,7 @@ from ooni import errors from ooni.settings import config from ooni.director import Director +from ooni.deck import TestDeck from ooni.reporter import YAMLReporter, OONIBReporter from ooni.nettest import NetTestLoader, MissingRequiredOption @@ -109,42 +110,43 @@ def runWithDirector(): config.read_config_file() log.start(global_options['logfile']) - # contains (test_cases, options, cmd_line_options) - test_list = [] - if global_options['no-collector']: - log.msg("Not reporting using a collector") - global_options['collector'] = None - - if global_options['testdeck']: - test_deck = yaml.safe_load(open(global_options['testdeck'])) - for test in test_deck: - test_list.append(NetTestLoader(test['options']['subargs'], - test_file=test['options']['test_file'])) - else: - log.debug("No test deck detected") - test_list.append(NetTestLoader(global_options['subargs'], - test_file=global_options['test_file'])) - - # check each test's usageOptions - for net_test_loader in test_list: - try: - net_test_loader.checkOptions() - except MissingRequiredOption, option_name: - log.err('Missing required option: "%s"' % option_name) - print net_test_loader.usageOptions().getUsage() - sys.exit(2) - except usage.UsageError, e: - log.err(e) - print net_test_loader.usageOptions().getUsage() - sys.exit(2) director = Director() d = director.start() + test_deck = TestDeck() + if global_options['no-collector']: + log.msg("Not reporting using a collector") + collector = global_options['collector'] = None + + try: + if global_options['testdeck']: + test_deck.loadDeck(global_options['testdeck']) + else: + log.debug("No test deck detected") + net_test_loader = NetTestLoader(global_options['subargs'], + test_file=global_options['test_file']) + test_deck.insert(net_test_loader) + except MissingRequiredOption, option_name: + log.err('Missing required option: "%s"' % option_name) + print net_test_loader.usageOptions().getUsage() + sys.exit(2) + except usage.UsageError, e: + log.err(e) + print net_test_loader.usageOptions().getUsage() + sys.exit(2) + + def fetch_nettest_inputs(result): + try: + test_deck.fetchAndVerifyDeckInputs() + except errors.UnableToLoadDeckInput, e: + return defer.failure.Failure(result) + def director_startup_failed(failure): log.err("Failed to start the director") r = failure.trap(errors.TorNotRunning, - errors.InvalidOONIBCollectorAddress) + errors.InvalidOONIBCollectorAddress, + errors.UnableToLoadDeckInput) if r == errors.TorNotRunning: log.err("Tor does not appear to be running") log.err("Reporting with the collector %s is not possible" % @@ -154,25 +156,42 @@ def runWithDirector(): log.err("Invalid format for oonib collector address.") log.msg("Should be in the format http://<collector_address>:<port>") log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") + elif r == errors.UnableToLoadDeckInput: + log.err('Missing required input files: %s' % failure) reactor.stop() - # Wait until director has started up (including bootstrapping Tor) before adding tess + # Wait until director has started up (including bootstrapping Tor) + # before adding tests def post_director_start(_): - for net_test_loader in test_list: - collector = global_options['collector'] - test_details = net_test_loader.testDetails - - yaml_reporter = YAMLReporter(test_details) - reporters = [yaml_reporter] + for net_test_loader in test_deck.netTestLoaders: + # TestDecks can specify different collectors + # for each net test, so that each NetTest + # may be paired with a test_helper and its collector + # However, a user can override this behavior by + # specifying a collector from the command-line (-c). + # If a collector is not specified in the deck, or the + # deck is a singleton, the default collector set in + # ooniprobe.conf will be used + + collector = None + if not global_options['no-collector']: + if global_options['collector']: + collector = global_options['collector'] + elif net_test_loader.options['collector']: + collector = net_test_loader.options['collector'] if collector and collector.startswith('httpo:') \ and (not (config.tor_state or config.tor.socks_port)): raise errors.TorNotRunning - elif collector: + + test_details = net_test_loader.testDetails + yaml_reporter = YAMLReporter(test_details) + reporters = [yaml_reporter] + + if collector: log.msg("Reporting using collector: %s" % collector) try: - oonib_reporter = OONIBReporter(test_details, - collector) + oonib_reporter = OONIBReporter(test_details, collector) reporters.append(oonib_reporter) except errors.InvalidOONIBCollectorAddress, e: raise e @@ -184,6 +203,7 @@ def runWithDirector(): director.allTestsDone.addBoth(shutdown) def start(): + d.addCallback(fetch_nettest_inputs) d.addCallback(post_director_start) d.addErrback(director_startup_failed)

1 0

[ooni-probe/master] Get the input processor as late as possible
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 48822189916c36067bdb86c527b096bb72320424 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jul 25 13:36:29 2013 +0200 Get the input processor as late as possible --- ooni/nettest.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 10eacd3..92c9864 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -360,11 +360,6 @@ class NetTestLoader(object): test_instance._checkRequiredOptions() test_instance._checkValidOptions() - inputs = test_instance.getInputProcessor() - if not inputs: - inputs = [None] - klass.inputs = inputs - def _get_test_methods(self, item): """ Look for test_ methods in subclasses of NetTestCase @@ -501,7 +496,14 @@ class NetTest(object): This is a generator that yields measurements and registers the callbacks for when a measurement is successful or has failed. """ + for test_class, test_methods in self.testCases: + # load the input processor as late as possible + inputs = test_class().getInputProcessor() + if not inputs: + inputs = [None] + test_class.inputs = inputs + for input in test_class.inputs: for method in test_methods: log.debug("Running %s %s" % (test_class, method))

1 0

[ooni-probe/master] Make inputProcessor setup be non-blocking
by art＠torproject.org 27 Aug '13

27 Aug '13

commit e08b3909b47ef451f84d8efc105c26ed4bcc2006 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Jul 25 13:44:43 2013 +0200 Make inputProcessor setup be non-blocking --- ooni/director.py | 1 + ooni/nettest.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 2c48bb0..2ba9d30 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -224,6 +224,7 @@ class Director(object): yield net_test.report.open() + yield net_test.initializeInputProcessor() self.measurementManager.schedule(net_test.generateMeasurements()) self.activeNetTests.append(net_test) diff --git a/ooni/nettest.py b/ooni/nettest.py index f940ae9..709e351 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -491,6 +491,13 @@ class NetTest(object): return measurement + @defer.inlineCallbacks + def initializeInputProcessor(self): + for test_class, _ in self.testCases: + test_class.inputs = yield defer.maybeDeferred(test_class().getInputProcessor) + if not test_class.inputs: + test_class.inputs = [None] + def generateMeasurements(self): """ This is a generator that yields measurements and registers the @@ -499,11 +506,6 @@ class NetTest(object): for test_class, test_methods in self.testCases: # load the input processor as late as possible - inputs = yield defer.maybeDeferred(test_class().getInputProcessor()) - if not inputs: - inputs = [None] - test_class.inputs = inputs - for input in test_class.inputs: for method in test_methods: log.debug("Running %s %s" % (test_class, method))

1 0