March 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Move the instancing of the t.i.w.Agent into createReport
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit 5bf37ab039a64937cdd97959069145148d5d36d2 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 18 00:24:20 2013 +0100 Move the instancing of the t.i.w.Agent into createReport Add XXX about possible cleaner solutions to the problem --- ooni/reporter.py | 22 ++++++++++++++-------- 1 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index bfd2330..c9fb61d 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -242,14 +242,6 @@ class OONIBReporter(OReporter): self.reportID = None - from ooni.utils.txagentwithsocks import Agent - from twisted.internet import reactor - try: - self.agent = Agent(reactor, sockshost="127.0.0.1", - socksport=int(config.tor.socks_port)) - except Exception, e: - log.exception(e) - OReporter.__init__(self, test_details) def validateCollectorAddress(self): @@ -293,6 +285,20 @@ class OONIBReporter(OReporter): """ Creates a report on the oonib collector. """ + # XXX we should probably be setting this inside of the constructor, + # however config.tor.socks_port is not set until Tor is started and the + # reporter is instantiated before Tor is started. We probably want to + # do this with some deferred kung foo or instantiate the reporter after + # tor is started. + + from ooni.utils.txagentwithsocks import Agent + from twisted.internet import reactor + try: + self.agent = Agent(reactor, sockshost="127.0.0.1", + socksport=int(config.tor.socks_port)) + except Exception, e: + log.exception(e) + url = self.collectorAddress + '/report' content = '---\n'

1 0

[ooni-probe/master] Handle errors when writing to the list of Reporters
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit 07d5d569bf4a5746da9495edfedc31b42459d01b Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Jan 16 23:15:18 2013 +0100 Handle errors when writing to the list of Reporters We will try to write to a reporter the number of times the ReportManager is configured to do so. If no more reporters are available an exception will be raised and the test will stop running (XXX this needs to be tested properly) --- ooni/director.py | 14 +++++++++++--- ooni/errors.py | 6 ++++++ ooni/managers.py | 2 -- ooni/nettest.py | 27 ++++++++++++++++++++++----- ooni/reporter.py | 15 +++++++++++++++ 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index b3dfba3..9a20491 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -139,9 +139,17 @@ class Director(object): self.failures.append((failure, measurement)) return failure - def reportEntryFailed(self, failure): - # XXX add failure handling logic - return + def reporterFailed(self, failure, net_test): + """ + This gets called every time a reporter is failing and has been removed + from the reporters of a NetTest. + Once a report has failed to be created that net_test will never use the + reporter again. + + XXX hook some logic here. + note: failure contains an extra attribute called failure.reporter + """ + pass def netTestDone(self, result, net_test): self.activeNetTests.remove(net_test) diff --git a/ooni/errors.py b/ooni/errors.py index 9281c6c..2023dbf 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -118,5 +118,11 @@ class DirectorException(Exception): pass class UnableToStartTor(DirectorException): + pass + class InvalidOONIBCollectorAddress(Exception): pass + +class AllReportersFailed(Exception): + pass + diff --git a/ooni/managers.py b/ooni/managers.py index 533e6b5..011e3b0 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -28,8 +28,6 @@ class TaskManager(object): The has failed to complete, we append it to the end of the task chain to be re-run once all the currently scheduled tasks have run. """ - print "This task has failed" - print failure self._active_tasks.remove(task) self.failures.append((failure, task)) diff --git a/ooni/nettest.py b/ooni/nettest.py index e2c17f1..4aa013c 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -9,6 +9,8 @@ from ooni.utils import log, checkForRoot, NotRootError from ooni import config from ooni import otime +from ooni.errors import AllReportersFailed + from inspect import getmembers from StringIO import StringIO @@ -248,16 +250,32 @@ class NetTest(object): self.state = NetTestState(self.done) - def doneReport(self, result): + def doneReport(self, report_results): """ This will get called every time a measurement is done and therefore a measurement is done. The state for the NetTest is informed of the fact that another task has reached the done state. + + Args: + report_results: + is the list of tuples returned by the self.report.write + :class:twisted.internet.defer.DeferredList + + Returns: + the same deferred list results """ + for report_status, report_result in report_results: + if report_status == False: + self.director.reporterFailed(report_result, self) + self.state.taskDone() - return result + + if len(self.report.reporters) == 0: + raise NoMoreReporters + + return report_results def makeMeasurement(self, test_class, test_method, test_input=None): """ @@ -282,9 +300,8 @@ class NetTest(object): measurement.done.addCallback(self.director.measurementSucceeded) measurement.done.addErrback(self.director.measurementFailed, measurement) - measurement.done.addCallback(self.report.write) - measurement.done.addErrback(self.director.reportEntryFailed) - + measurement.done.addBoth(self.report.write) + # here we are dealing with a deferred list measurement.done.addBoth(self.doneReport) return measurement diff --git a/ooni/reporter.py b/ooni/reporter.py index 237bae7..bfd2330 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -380,6 +380,19 @@ class Report(object): for reporter in self.reporters: d = defer.maybeDeferred(reporter.createReport) d.addCallback(reporter.created.callback) + d.addErrback(reporter.created.callback) + + def failedOpeningReport(self, failure, reporter): + """ + This errback get's called every time we fail to create a report. + By fail we mean that the number of retries has exceeded. + Once a report has failed to be created with a reporter we give up and + remove the reporter from the list of reporters to write to. + """ + log.err("Failed to open %s reporter, giving up..." % reporter) + self.reporters.remove(reporter) + failure.reporter = reporter + return failure def write(self, measurement): """ @@ -406,6 +419,8 @@ class Report(object): return report_write_task.done d = reporter.created.addBoth(writeReportEntry) + # Give up on writing to the reporter if the task fails X times + d.addErrback(self.failedOpeningReport, reporter) l.append(d) dl = defer.DeferredList(l)

1 0

[ooni-probe/master] Add report.close to nettest.done deferred
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit aacec6d509ab112b9466e46f1540b0744a1c20ef Author: aagbsn <aagbsn(a)extc.org> Date: Mon Feb 4 13:07:49 2013 -0500 Add report.close to nettest.done deferred --- ooni/director.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 7bc19f8..6717d70 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -189,6 +189,7 @@ class Director(object): self.activeNetTests.append(net_test) net_test.done.addBoth(self.netTestDone, net_test) + net_test.done.addBoth(report.close) return net_test.done def startSniffing(self):

1 0

[ooni-probe/master] Use proper name for AllReportersFailed
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit fc2a8803c82d836cf4684e9b5f19158c510b1efa Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 18 01:19:36 2013 +0100 Use proper name for AllReportersFailed --- ooni/nettest.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 464d91a..ad216ba 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -295,7 +295,7 @@ class NetTest(object): self.state.taskDone() if len(self.report.reporters) == 0: - raise NoMoreReporters + raise AllReportersFailed return report_results

1 0

[ooni-probe/master] Initialize reporters in startNetTest, not Director __init__
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit f6bae38ccd45ab4360148e617e497d451f6128c3 Author: aagbsn <aagbsn(a)extc.org> Date: Sat Jan 19 15:20:50 2013 +0000 Initialize reporters in startNetTest, not Director __init__ --- ooni/director.py | 8 +++----- 1 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 93e39bd..7bc19f8 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -58,9 +58,7 @@ class Director(object): """ _scheduledTests = 0 - def __init__(self, reporters): - self.reporters = reporters - + def __init__(self): self.netTests = [] self.activeNetTests = [] @@ -172,7 +170,7 @@ class Director(object): def netTestDone(self, result, net_test): self.activeNetTests.remove(net_test) - def startNetTest(self, _, net_test_loader): + def startNetTest(self, _, net_test_loader, reporters): """ Create the Report for the NetTest and start the report NetTest. @@ -182,7 +180,7 @@ class Director(object): _: #XXX very dirty hack """ - report = Report(self.reporters, self.reportEntryManager) + report = Report(reporters, self.reportEntryManager) net_test = NetTest(net_test_loader, report) net_test.director = self

1 0

[ooni-probe/master] Fix typo in doneReport docstring
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit 69a802fafe646fb7bb7aa52c69610587b02476d1 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Feb 4 13:07:58 2013 -0500 Fix typo in doneReport docstring --- ooni/nettest.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index ad216ba..0868ab8 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -274,7 +274,7 @@ class NetTest(object): def doneReport(self, report_results): """ - This will get called every time a measurement is done and therefore a + This will get called every time a report is done and therefore a measurement is done. The state for the NetTest is informed of the fact that another task has

1 0

[ooni-probe/master] Add test deck support to runWithDirector
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit 6ec391c39893dddb56dfd9447c522005425b01c7 Author: aagbsn <aagbsn(a)extc.org> Date: Sat Jan 19 15:22:37 2013 +0000 Add test deck support to runWithDirector --- ooni/oonicli.py | 118 ++++++++++++++++++++----------------------------------- 1 files changed, 43 insertions(+), 75 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 78389f4..e9c6b01 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -168,88 +168,56 @@ def runWithDirector(): test! """ global_options = parseOptions() - config.cmd_line_options = global_options - log.start(global_options['logfile']) - - net_test_args = global_options.pop('subargs') - net_test_file = global_options['test'] - net_test_loader = NetTestLoader(net_test_file, net_test_args) - - try: - net_test_loader.checkOptions() - except MissingRequiredOption, option_name: - log.err('Missing required option: "%s"' % option_name) - print net_test_loader.usageOptions().getUsage() - sys.exit(2) - except usage.UsageError, e: - log.err(e) - print net_test_loader.usageOptions().getUsage() - sys.exit(2) - - # reporters = [YAMLReporter, OONIBReporter] - - yaml_reporter = YAMLReporter(net_test_loader.testDetails) - reporters = [yaml_reporter] - - if global_options['collector']: - try: - oonib_reporter = OONIBReporter(net_test_loader.testDetails, - global_options['collector']) - reporters.append(oonib_reporter) - except InvalidOONIBCollectorAddress: - log.err("Invalid format for oonib collector address.") - log.msg("Should be in the format http://<collector_address>:<port>") - log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") - sys.exit(1) - - director = Director(reporters) - d = director.start() - - d.addCallback(director.startNetTest, net_test_loader) - d.addBoth(shutdown) - reactor.run() - -def run(): - """ - Parses command line arguments of test. - """ - - if config.privacy.includepcap: - log.msg("Starting") - if not config.reports.pcap: - config.generatePcapFilename() - runner.startSniffing() - - resume = cmd_line_options['resume'] + net_test_args = global_options.get('subargs') # contains (test_cases, options, cmd_line_options) test_list = [] - if cmd_line_options['testdeck']: - test_deck = yaml.safe_load(open(cmd_line_options['testdeck'])) + if global_options['testdeck']: + test_deck = yaml.safe_load(open(global_options['testdeck'])) for test in test_deck: - del cmd_line_options - cmd_line_options = test['options'] - if resume: - cmd_line_options['resume'] = True - else: - cmd_line_options['resume'] = False - test_list.append(runner.loadTest(cmd_line_options)) + test_options = test['options'] + test_file = test_options['test'] + test_subargs = test_options['subargs'] + test_list.append(NetTestLoader(test_file, test_subargs)) else: log.debug("No test deck detected") - del cmd_line_options['testdeck'] - test_list.append(runner.loadTest(cmd_line_options)) - - if config.advanced.start_tor: - log.msg("Starting Tor...") - d = runner.startTor() - d.addCallback(runTestList, test_list) - d.addErrback(errorRunningTests) - else: - # We need to pass None as first argument because when the callback is - # fired it will pass it's result to runTestCase. - d = runTestList(None, test_list) - d.addErrback(errorRunningTests) + test_list.append(NetTestLoader(global_options['test'], net_test_args)) + + + director = Director() + d = director.start() + + for net_test_loader in test_list: + + try: + net_test_loader.checkOptions() + except MissingRequiredOption, option_name: + log.err('Missing required option: "%s"' % option_name) + print net_test_loader.usageOptions().getUsage() + sys.exit(2) + except usage.UsageError, e: + log.err(e) + print net_test_loader.usageOptions().getUsage() + sys.exit(2) + + yaml_reporter = YAMLReporter(net_test_loader.testDetails) + reporters = [yaml_reporter] + + if global_options['collector']: + try: + oonib_reporter = OONIBReporter(net_test_loader.testDetails, + global_options['collector']) + reporters.append(oonib_reporter) + except InvalidOONIBCollectorAddress: + log.err("Invalid format for oonib collector address.") + log.msg("Should be in the format http://<collector_address>:<port>") + log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") + sys.exit(1) + + #XXX add all the tests to be run sequentially + d.addCallback(director.startNetTest, net_test_loader, reporters) + d.addBoth(shutdown) reactor.run()

1 0

[ooni-probe/master] Set testName from class
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit b6f9c2284dee35981863652f6886c64b9b6b2885 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jan 25 19:44:45 2013 +0000 Set testName from class --- ooni/nettest.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 0868ab8..bd019cc 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -145,7 +145,7 @@ class NetTestLoader(object): test_class, _ = test_cases[0] self.testVersion = test_class.version - self.testName = os.path.basename(net_test_file).strip('.py') + self.testName = test_class.name.lower().replace(' ','_') return test_cases def checkOptions(self):

1 0

[ooni-probe/master] Add failedOpeningReport to reporter created errback
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit fd5d08666f52031ed3341f6b23775dcfaeaaf71c Author: aagbsn <aagbsn(a)extc.org> Date: Mon Feb 4 13:08:02 2013 -0500 Add failedOpeningReport to reporter created errback --- ooni/reporter.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index b01c99e..23cca69 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -33,7 +33,7 @@ from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni import config -from ooni.tasks import ReportEntry +from ooni.tasks import ReportEntry, TaskTimedOut class ReporterException(Exception): pass @@ -384,10 +384,12 @@ class Report(object): created callback of the reporter whose report got created. """ for reporter in self.reporters: + reporter.created.addErrback(self.failedOpeningReport, reporter) d = defer.maybeDeferred(reporter.createReport) d.addCallback(reporter.created.callback) d.addErrback(reporter.created.callback) + def failedOpeningReport(self, failure, reporter): """ This errback get's called every time we fail to create a report.

1 0

[ooni-probe/master] Shutdown reactor after running all tests, and remove unused code.
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit 59706ca850462d7e6393938ebd0ab623ab9a6523 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Jan 25 20:04:53 2013 +0000 Shutdown reactor after running all tests, and remove unused code. --- ooni/oonicli.py | 82 ++++--------------------------------------------------- 1 files changed, 6 insertions(+), 76 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index e9c6b01..77d4549 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -5,7 +5,7 @@ import os import time import yaml -from twisted.internet import defer, reactor, task +from twisted.internet import reactor from twisted.python import usage from twisted.python.util import spewer @@ -72,78 +72,6 @@ class Options(usage.Options): except: raise usage.UsageError("No test filename specified!") -def parseNetTestOptions(obj, argv): - if not hasattr(obj.usageOptions, 'optParameters'): - obj.usageOptions.optParameters = [] - - if obj.inputFile: - obj.usageOptions.optParameters.append(obj.inputFile) - - if obj.baseParameters: - for parameter in obj.baseParameters: - obj.usageOptions.optParameters.append(parameter) - - if obj.baseFlags: - if not hasattr(obj.usageOptions, 'optFlags'): - obj.usageOptions.optFlags = [] - for flag in obj.baseFlags: - obj.usageOptions.optFlags.append(flag) - - options = obj.usageOptions() - options.parseOptions(argv) - - return options - -def updateStatusBar(): - for test_filename in config.state.keys(): - # The ETA is not updated so we we will not print it out for the - # moment. - eta = config.state[test_filename].eta() - progress = config.state[test_filename].progress() - progress_bar_frmt = "[%s] %s%%" % (test_filename, progress) - log.debug(progress_bar_frmt) - -def testsEnded(*arg, **kw): - """ - You can place here all the post shutdown tasks. - """ - log.debug("testsEnded: Finished running all tests") - config.start_reactor = False - try: reactor.stop() - except: pass - -def testFailed(failure): - log.err("Failed in running a test inside a test list") - failure.printTraceback() - -def runTestList(none, test_list): - """ - none: is always None. - - test_list (list): a list of tuples containing (test_cases, options, - cmd_line_options) - """ - deck_dl = [] - - for test in test_list: - test_cases, options, cmd_line_options = test - d1 = runner.runTestCases(test_cases, options, cmd_line_options) - deck_dl.append(d1) - - d2 = defer.DeferredList(deck_dl) - d2.addCallback(testsEnded) - d2.addErrback(testFailed) - - # Print every 5 second the list of current tests running - l = task.LoopingCall(updateStatusBar) - l.start(5.0) - return d2 - -def errorRunningTests(failure): - log.err("There was an error in running a test") - failure.printTraceback() - - def parseOptions(): cmd_line_options = Options() if len(sys.argv) == 1: @@ -160,7 +88,9 @@ def shutdown(result): This will get called once all the operations that need to be done in the current oonicli session have been completed. """ - reactor.stop() + log.debug("Halting reactor") + try: reactor.stop() + except: pass def runWithDirector(): """ @@ -191,7 +121,6 @@ def runWithDirector(): d = director.start() for net_test_loader in test_list: - try: net_test_loader.checkOptions() except MissingRequiredOption, option_name: @@ -219,5 +148,6 @@ def runWithDirector(): #XXX add all the tests to be run sequentially d.addCallback(director.startNetTest, net_test_loader, reporters) - d.addBoth(shutdown) + d.addCallback(shutdown) + #XXX: if errback is called they do not propagate reactor.run()

1 0