August 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Reschedule failures immediately
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 7ea42ac6815d46329d979cf6e93f750a440b4166 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 25 14:50:41 2013 +0200 Reschedule failures immediately Reschedule failures immediately so they do not accrue at the end of the task list and eat memory. --- ooni/managers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/managers.py b/ooni/managers.py index cc2d067..36b791e 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -37,8 +37,8 @@ class TaskManager(object): if task.failures <= self.retries: log.debug("Rescheduling...") - self._tasks = itertools.chain(self._tasks, - makeIterable(task)) + self._tasks = itertools.chain(makeIterable(task), self._tasks) + else: # This fires the errback when the task is done but has failed. log.err('Permanent failure for %s' % task)

1 0

[ooni-probe/master] Track the number of failures, not the instances.
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 0791b2dd50507bdf448c239a4b39b05bd8fddcd4 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 25 14:59:35 2013 +0200 Track the number of failures, not the instances. The number of failures on very large scans (1m hosts) results in a large list. This commit tracks the number of failures instead of the objects. --- ooni/managers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ooni/managers.py b/ooni/managers.py index 36b791e..a229044 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -22,7 +22,7 @@ class TaskManager(object): def __init__(self): self._tasks = iter(()) self._active_tasks = [] - self.failures = [] + self.failures = 0 def _failed(self, failure, task): """ @@ -33,7 +33,7 @@ class TaskManager(object): log.exception(failure) self._active_tasks.remove(task) - self.failures.append((failure, task)) + self.failures = self.failures + 1 if task.failures <= self.retries: log.debug("Rescheduling...") @@ -85,7 +85,7 @@ class TaskManager(object): @property def failedMeasurements(self): - return len(self.failures) + return self.failures @property def availableSlots(self): @@ -110,7 +110,7 @@ class TaskManager(object): """ This is called to start the task manager. """ - self.failures = [] + self.failures = 0 self._fillSlots()

1 0

[ooni-probe/master] Fix some issues with ooni.deck
by art＠torproject.org 27 Aug '13

27 Aug '13

commit d66091573872b8451686d5e61a098145835b2b4b Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 25 20:12:37 2013 +0200 Fix some issues with ooni.deck 1. Do not require a bouncer to use a deck 2. If a bouncer is used, also fetch collectors for tests that do not list helpers or already have helpers specified in the deck. --- ooni/deck.py | 25 ++++++++++++++----------- ooni/oonicli.py | 6 +++++- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 81b2afa..a8aee39 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -14,8 +14,8 @@ import re import yaml class Deck(object): - def __init__(self, bouncer, deckFile=None): - self.bouncer = bouncer + def __init__(self, deckFile=None): + self.bouncer = None self.netTestLoaders = [] self.inputs = [] self.testHelpers = {} @@ -40,27 +40,30 @@ class Deck(object): @defer.inlineCallbacks def setup(self): """ fetch and verify inputs for all NetTests in the deck """ - for net_test_loader in self.netTestLoaders: - yield self.fetchAndVerifyNetTestInput(net_test_loader) - yield self.lookupTestHelper(net_test_loader) - yield self.lookupTestCollector(net_test_loader) + if self.bouncer: + for net_test_loader in self.netTestLoaders: + yield self.fetchAndVerifyNetTestInput(net_test_loader) + yield self.lookupTestHelper(net_test_loader) + yield self.lookupTestCollector(net_test_loader) @defer.inlineCallbacks def lookupTestHelper(self, net_test_loader): oonibclient = OONIBClient(self.bouncer) for th in net_test_loader.requiredTestHelpers: # {'name':'', 'option':'', 'test_class':''} - helper = yield oonibclient.lookupTestHelper(th['name']) - th['test_class'].localOptions[th['option']] = helper['test-helper'] + response = yield oonibclient.lookupTestHelper(th['name']) + th['test_class'].localOptions[th['option']] = response['test-helper'] #XXX: collector is only set once! - net_test_loader.collector = helper['collector'] - + if 'collector' in response.keys(): + net_test_loader.collector = response['collector'].encode('utf-8') @defer.inlineCallbacks def lookupTestCollector(self, net_test_loader): oonibclient = OONIBClient(self.bouncer) if net_test_loader.collector is None: - net_test_loader.collector = oonibclient.lookupTestCollector(th['test_class'].testName) + response = yield oonibclient.lookupTestCollector(net_test_loader.testName) + if 'collector' in response.keys(): + net_test_loader.collector = response['collector'].encode('utf-8') @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): diff --git a/ooni/oonicli.py b/ooni/oonicli.py index cf7d34d..fb333c7 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -127,10 +127,14 @@ def runWithDirector(): director = Director() d = director.start() - deck = Deck(global_options['bouncer']) + #XXX: This should mean no bouncer either! if global_options['no-collector']: log.msg("Not reporting using a collector") collector = global_options['collector'] = None + global_options['bouncer'] = None + + deck = Deck() + deck.bouncer = global_options['bouncer'] try: if global_options['testdeck']:

1 0

[ooni-probe/master] Simplify the bouncer API
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 25d94151864bea2406469103491ec77b2279fcea Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 25 19:14:17 2013 +0200 Simplify the bouncer API Simplified bouncer API requires one helper lookup per NetTest. --- ooni/deck.py | 52 ++++++++++++++++++++++----------------------------- ooni/oonibclient.py | 19 +++++++++++++++---- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 1ad8377..81b2afa 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -19,8 +19,6 @@ class Deck(object): self.netTestLoaders = [] self.inputs = [] self.testHelpers = {} - self.collector = None - if deckFile: self.loadDeck(deckFile) def loadDeck(self, deckFile): @@ -28,6 +26,9 @@ class Deck(object): for test in test_deck: net_test_loader = NetTestLoader(test['options']['subargs'], test_file=test['options']['test_file']) + #XXX: If the deck specifies the collector, we use the specified collector + # And it should also specify the test helper address to use + # net_test_loader.collector = test['options']['collector'] net_test_loader.checkOptions() self.netTestLoaders.append(net_test_loader) @@ -36,18 +37,30 @@ class Deck(object): net_test_loader.checkOptions() self.netTestLoaders.append(net_test_loader) - def getRequiredTestHelpers(self): - for net_test_loader in self.netTestLoaders: - for test_helper in net_test_loader.requiredTestHelpers: - self.testHelpers[test_helper['name']] = None - @defer.inlineCallbacks def setup(self): """ fetch and verify inputs for all NetTests in the deck """ for net_test_loader in self.netTestLoaders: yield self.fetchAndVerifyNetTestInput(net_test_loader) - self.getRequiredTestHelpers() - yield self.lookupTestHelpers() + yield self.lookupTestHelper(net_test_loader) + yield self.lookupTestCollector(net_test_loader) + + @defer.inlineCallbacks + def lookupTestHelper(self, net_test_loader): + oonibclient = OONIBClient(self.bouncer) + for th in net_test_loader.requiredTestHelpers: + # {'name':'', 'option':'', 'test_class':''} + helper = yield oonibclient.lookupTestHelper(th['name']) + th['test_class'].localOptions[th['option']] = helper['test-helper'] + #XXX: collector is only set once! + net_test_loader.collector = helper['collector'] + + + @defer.inlineCallbacks + def lookupTestCollector(self, net_test_loader): + oonibclient = OONIBClient(self.bouncer) + if net_test_loader.collector is None: + net_test_loader.collector = oonibclient.lookupTestCollector(th['test_class'].testName) @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): @@ -69,24 +82,3 @@ class Deck(object): raise e.UnableToLoadDeckInput, cached_path i['test_class'].localOptions[i['key']] = input_file.cached_file - - def setNettestOptions(self): - for net_test_loader in self.netTestLoaders: - for th in net_test_loader.requiredTestHelpers: - test_helper_address = self.testHelpers[th['name']] - th['test_class'].localOptions[th['option']] = test_helper_address - net_test_loader.collector = self.collector - log.debug("Using %s: %s" % (test_helper_address, self.collector)) - - @defer.inlineCallbacks - def lookupTestHelpers(self): - log.msg("Looking up test helpers: %s" % self.testHelpers.keys()) - - required_test_helpers = self.testHelpers.keys() - if required_test_helpers: - oonibclient = OONIBClient(self.bouncer) - test_helpers = yield oonibclient.lookupTestHelpers(required_test_helpers) - self.collector = test_helpers['collector'] - for name in self.testHelpers.keys(): - self.testHelpers[name] = test_helpers[name] - self.setNettestOptions() diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index 9bdce95..227c775 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -187,15 +187,26 @@ class OONIBClient(object): return self.queryBackend('GET', '/policy/nettest') @defer.inlineCallbacks - def lookupTestHelpers(self, test_helper_names): + def lookupTestCollector(self, test_name): try: - test_helpers = yield self.queryBackend('POST', '/bouncer', - query={'test-helpers': test_helper_names}) + test_collector = yield self.queryBackend('POST', '/bouncer', + query={'test-collector': test_name}) + except Exception: + raise e.CouldNotFindTestCollector + + defer.returnValue(test_collector) + + @defer.inlineCallbacks + def lookupTestHelper(self, test_helper_name): + try: + + test_helper = yield self.queryBackend('POST', '/bouncer', + query={'test-helper': test_helper_name}) except Exception: raise e.CouldNotFindTestHelper if not test_helpers: raise e.CouldNotFindTestHelper - defer.returnValue(test_helpers) + defer.returnValue(test_helper)

1 0

[ooni-probe/master] Track the number of failed reporters
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 0d4243c1ff29d0e6392413f05bc2c01a29f6b60b Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 25 13:23:20 2013 +0200 Track the number of failed reporters --- ooni/reporter.py | 1 + ooni/tasks.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index aee96e5..b5fcce8 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -456,6 +456,7 @@ class Report(object): def report_failed(failure): log.debug("Report Write Failure") try: + report_tracker.failedReporters.append(reporter) self.failedWritingReport(failure, reporter) except errors.NoMoreReporters, e: log.err("No More Reporters!") diff --git a/ooni/tasks.py b/ooni/tasks.py index ba7ea4e..baefb51 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -133,6 +133,7 @@ class ReportTracker(object): def __init__(self, reporters): self.report_completed = 0 self.reporters = reporters + self.failedReporters = [] def finished(self): """ @@ -142,10 +143,7 @@ class ReportTracker(object): # is considered completed but failed, but the number # of reporters is now decreased by the number of failed # reporters. - # XXX: should we track to see if, for example: - # self.report_completed == len(self.reporters) + \ - # len(self.failed_reporters) - if self.report_completed >= len(self.reporters): + if self.report_completed == (len(self.reporters) + len(self.failedReporters)): return True return False

1 0

[ooni-probe/master] Link TaskManagers by Least Available Slots
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 1a0e6103abc78a6e10e9da2f2e8a94aa9074277d Author: aagbsn <aagbsn(a)extc.org> Date: Thu Aug 22 13:00:45 2013 +0200 Link TaskManagers by Least Available Slots A LinkedTaskManager only has availableSlots if its child TaskManager also has availableSlots. Children LinkedTaskManagers must notify the parent LinkedTaskManager when a task is complete because the task queue is event-driven. --- ooni/director.py | 4 ++++ ooni/managers.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 2ba9d30..50323fd 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -71,6 +71,10 @@ class Director(object): self.reportEntryManager = ReportEntryManager() self.reportEntryManager.director = self + # Link the TaskManager's by least available slots. + self.measurementManager.child = self.reportEntryManager + # Notify the parent when tasks complete # XXX deadlock!? + self.reportEntryManager.parent = self.measurementManager self.successfulMeasurements = 0 self.failedMeasurements = 0 diff --git a/ooni/managers.py b/ooni/managers.py index ff7c2f2..cc2d067 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -129,7 +129,31 @@ class TaskManager(object): """ raise NotImplemented -class MeasurementManager(TaskManager): +class LinkedTaskManager(TaskManager): + def __init__(self): + super(LinkedTaskManager, self).__init__() + self.child = None + self.parent = None + + @property + def availableSlots(self): + mySlots = self.concurrency - len(self._active_tasks) + if self.child: + s = self.child.availableSlots + return min(s, mySlots) + return mySlots + + def _succeeded(self, result, task): + super(LinkedTaskManager, self)._succeeded(result, task) + if self.parent: + self.parent._fillSlots() + + def _failed(self, result, task): + super(LinkedTaskManager, self)._failed(result, task) + if self.parent: + self.parent._fillSlots() + +class MeasurementManager(LinkedTaskManager): """ This is the Measurement Tracker. In here we keep track of active measurements and issue new measurements once the active ones have been completed. @@ -155,7 +179,7 @@ class MeasurementManager(TaskManager): def failed(self, failure, measurement): pass -class ReportEntryManager(TaskManager): +class ReportEntryManager(LinkedTaskManager): def __init__(self): if config.advanced.reporting_retries: self.retries = config.advanced.reporting_retries

1 0

[ooni-probe/master] Add CouldNotFindTestCollector error
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 4920850e60c25bd2ece3cd05160087b0f19cfc85 Author: aagbsn <aagbsn(a)extc.org> Date: Sun Aug 25 20:30:29 2013 +0200 Add CouldNotFindTestCollector error --- ooni/errors.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ooni/errors.py b/ooni/errors.py index 5ad1940..58d894a 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -170,5 +170,9 @@ class OONIBTestDetailsLookupError(OONIBReportError): class UnableToLoadDeckInput(Exception): pass + class CouldNotFindTestHelper(Exception): pass + +class CouldNotFindTestCollector(Exception): + pass

1 0

[ooni-probe/master] Handle more failure types
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 27065e664834a536ecba39798ea98fb2f1a39342 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Aug 22 15:20:16 2013 +0200 Handle more failure types * Make sure tests will still work properly even if Tor is not running. (cherry picked from commit 783fc4673a674bec54b958d23f3356bcc0c27286) --- ooni/errors.py | 17 +++++++++++++---- ooni/templates/httpt.py | 7 +++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 58d894a..200491e 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -2,7 +2,7 @@ from twisted.internet.defer import TimeoutError as DeferTimeoutError from twisted.web._newclient import ResponseNeverReceived from twisted.internet.error import ConnectionRefusedError, TCPTimedOutError -from twisted.internet.error import DNSLookupError +from twisted.internet.error import DNSLookupError, ConnectError, ConnectionLost from twisted.internet.error import TimeoutError as GenericTimeoutError from txsocksx.errors import SOCKSError @@ -26,7 +26,8 @@ def handleAllFailures(failure): SOCKSError, MethodsNotAcceptedError, AddressNotSupported, ConnectionError, NetworkUnreachable, ConnectionLostEarly, ConnectionNotAllowed, NoAcceptableMethods, ServerFailure, - HostUnreachable, ConnectionRefused, TTLExpired, CommandNotSupported) + HostUnreachable, ConnectionRefused, TTLExpired, CommandNotSupported, + ConnectError, ConnectionLost) return failureToString(failure) @@ -47,9 +48,17 @@ def failureToString(failure): string = None if isinstance(failure.value, ConnectionRefusedError): - log.err("Connection refused. The backend may be down") + log.err("Connection refused.") string = 'connection_refused_error' + elif isinstance(failure.value, ConnectionLost): + log.err("Connection lost.") + string = 'connection_lost_error' + + elif isinstance(failure.value, ConnectError): + log.err("Connect error.") + string = 'connect_error' + elif isinstance(failure.value, gaierror): log.err("Address family for hostname not supported") string = 'address_family_not_supported_error' @@ -110,7 +119,7 @@ def failureToString(failure): string = 'socks_error' else: - log.err("Unknown failure type: %s" % type(failure)) + log.err("Unknown failure type: %s" % type(failure.value)) string = 'unknown_failure %s' % str(failure.value) return string diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 0bca5df..472cc77 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -286,6 +286,9 @@ class HTTPTest(NetTestCase): request['url'] = url request['headers'] = headers request['body'] = body + request['tor'] = False + if use_tor: + request['tor'] = True if self.randomizeUA: log.debug("Randomizing user agent") @@ -304,9 +307,9 @@ class HTTPTest(NetTestCase): headers = TrueHeaders(request['headers']) def errback(failure, request): - failure_string = handleAllFailures(failure) log.err("Error performing %s" % request) - HTTPTest.addToReport(self, request, failure_string=failure_string) + failure_string = handleAllFailures(failure) + self.addToReport(request, failure_string=failure_string) return failure d = agent.request(request['method'], request['url'], headers,

1 0

[ooni-probe/master] Properly handle the case when we don't get a collector by asking a bouncer
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 1a087d7760c317675c3c21cfe26d197803304da1 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Aug 27 11:00:19 2013 +0200 Properly handle the case when we don't get a collector by asking a bouncer --- ooni/oonicli.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index fb333c7..c511aa0 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -156,14 +156,15 @@ def runWithDirector(): def setup_nettest(_): try: return deck.setup() - except errors.UnableToLoadDeckInput, e: - return defer.failure.Failure(result) + except errors.UnableToLoadDeckInput as error: + return defer.failure.Failure(error) def director_startup_failed(failure): log.err("Failed to start the director") r = failure.trap(errors.TorNotRunning, errors.InvalidOONIBCollectorAddress, - errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper) + errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper, + errors.CouldNotFindTestCollector) if isinstance(failure.value, errors.TorNotRunning): log.err("Tor does not appear to be running") @@ -184,6 +185,11 @@ def runWithDirector(): log.err("Unable to obtain the required test helpers.") log.msg("Try with a different bouncer or check that Tor is running properly.") + elif isinstance(failure.value, errors.CouldNotFindTestCollector): + log.err("Could not find a valid collector.") + log.msg("Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.") + + if config.advanced.debug: log.exception(failure)

1 0

[ooni-probe/master] Fix report write failure bug
by art＠torproject.org 27 Aug '13

27 Aug '13

commit 3c166e576f74326a5d2c2f3cab0ed1672047c1df Author: aagbsn <aagbsn(a)extc.org> Date: Thu Aug 22 13:05:17 2013 +0200 Fix report write failure bug When a report has opened, and then a reporter has subsequently failed, it is now removed from the list of valid reporters. --- ooni/reporter.py | 40 ++++++++++++++++++++++++++++++++++++++-- ooni/tasks.py | 9 ++++++++- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 67678a6..aee96e5 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -453,13 +453,48 @@ class Report(object): if report_tracker.finished(): all_written.callback(report_tracker) + def report_failed(failure): + log.debug("Report Write Failure") + try: + self.failedWritingReport(failure, reporter) + except errors.NoMoreReporters, e: + log.err("No More Reporters!") + all_written.errback(defer.fail(e)) + else: + report_tracker.completed() + if report_tracker.finished(): + all_written.callback(report_tracker) + return + report_entry_task = ReportEntry(reporter, measurement) self.reportEntryManager.schedule(report_entry_task) - report_entry_task.done.addBoth(report_completed) + report_entry_task.done.addCallback(report_completed) + report_entry_task.done.addErrback(report_failed) return all_written + def failedWritingReport(self, failure, reporter): + """ + This errback gets called every time we fail to write a report. + By fail we mean that the number of retries has exceeded. + Once a report has failed to be written with a reporter we give up and + remove the reporter from the list of reporters to write to. + """ + + # XXX: may have been removed already by another failure. + if reporter in self.reporters: + log.err("Failed to write to %s reporter, giving up..." % reporter) + self.reporters.remove(reporter) + else: + log.err("Failed to write to (already) removed reporter %s" % reporter) + + # Don't forward the exception unless there are no more reporters + if len(self.reporters) == 0: + log.err("Removed last reporter %s" % reporter) + raise errors.NoMoreReporters + return + def failedOpeningReport(self, failure, reporter): """ This errback get's called every time we fail to create a report. @@ -470,7 +505,8 @@ class Report(object): log.err("Failed to open %s reporter, giving up..." % reporter) log.err("Reporter %s failed, removing from report..." % reporter) #log.exception(failure) - self.reporters.remove(reporter) + if reporter in self.reporters: + self.reporters.remove(reporter) # Don't forward the exception unless there are no more reporters if len(self.reporters) == 0: log.err("Removed last reporter %s" % reporter) diff --git a/ooni/tasks.py b/ooni/tasks.py index cf13da4..ba7ea4e 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -138,7 +138,14 @@ class ReportTracker(object): """ Returns true if all the tasks are done. False if not. """ - if self.report_completed == len(self.reporters): + # If a reporter fails and is removed, the report + # is considered completed but failed, but the number + # of reporters is now decreased by the number of failed + # reporters. + # XXX: should we track to see if, for example: + # self.report_completed == len(self.reporters) + \ + # len(self.failed_reporters) + if self.report_completed >= len(self.reporters): return True return False

1 0