November 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Fix report header in order to fix issue https://github.com/TheTorProject/ooni-probe/issues/223
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 72728414754c6df8defc133992788ba43af53615 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 6 12:42:35 2013 +0100 Fix report header in order to fix issue https://github.com/TheTorProject/ooni-probe/issues/223 --- ooni/deck.py | 5 ++++- ooni/nettest.py | 5 +++-- ooni/oonicli.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 7b8a8c4..9054f25 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -108,6 +108,9 @@ class Deck(InputFile): self.deckHash = sha256(f.read()) f.seek(0) test_deck = yaml.safe_load(f) + + if isinstance(test_deck, dict): + test_deck = [test_deck] for test in test_deck: try: @@ -117,7 +120,7 @@ class Deck(InputFile): log.msg("Skipping...") continue net_test_loader = NetTestLoader(test['options']['subargs'], - test_file=nettest_path) + test_file=nettest_path, global_options=test['options']) #XXX: If the deck specifies the collector, we use the specified collector # And it should also specify the test helper address to use # net_test_loader.collector = test['options']['collector'] diff --git a/ooni/nettest.py b/ooni/nettest.py index 9304f2b..f4ce3be 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -174,9 +174,10 @@ class NetTestLoader(object): method_prefix = 'test' collector = None - def __init__(self, options, test_file=None, test_string=None): + def __init__(self, options, test_file=None, test_string=None, global_options={}): self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]{64})$") self.options = options + self.globalOptions = global_options self.testCases, test_cases = None, None if test_file: @@ -287,7 +288,7 @@ class NetTestLoader(object): 'test_version': self.testVersion, 'software_name': 'ooniprobe', 'software_version': software_version, - 'options': self.options, + 'options': self.globalOptions, 'input_hashes': input_file_hashes } return test_details diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 8622e09..a19c4ef 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -143,7 +143,7 @@ def runWithDirector(): log.debug("No test deck detected") test_file = nettest_to_path(global_options['test_file']) net_test_loader = NetTestLoader(global_options['subargs'], - test_file=test_file) + test_file=test_file, global_options=global_options) deck.insert(net_test_loader) except errors.MissingRequiredOption, option_name: log.err('Missing required option: "%s"' % option_name)

1 0

[ooni-probe/master] Merge pull request #238 from TheTorProject/fix/report_header
by art＠torproject.org 23 Nov '13

23 Nov '13

commit a3d404a56a42f6bcbc02e99a54008a32bd1c6611 Merge: 12731e8 7272841 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Nov 13 10:12:55 2013 -0800 Merge pull request #238 from TheTorProject/fix/report_header Fix report header in order to fix issue https://github.com/TheTorProject... ooni/deck.py | 5 ++++- ooni/nettest.py | 5 +++-- ooni/oonicli.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-)

1 0

[ooni-probe/master] Merge pull request #240 from TheTorProject/feature/include-decks-n-inputs
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 12731e8869b7ab1d1528d6122fa72b12dded83c2 Merge: 9db041e 8612e5d Author: aagbsn <aagbsn(a)extc.org> Date: Wed Nov 13 09:54:57 2013 -0800 Merge pull request #240 from TheTorProject/feature/include-decks-n-inputs Feature/include decks n inputs MANIFEST.in | 2 ++ data/decks/basic.deck | 24 ++++++++++++++ data/decks/before_i_commit.testdeck | 36 +++++++++++++++++++++ data/decks/short_no_root.deck | 60 +++++++++++++++++++++++++++++++++++ data/inputs/Makefile | 13 ++++++++ data/inputs/README | 47 +++++++++++++++++++++++++++ decks/basic.deck | 24 -------------- decks/before_i_commit.testdeck | 36 --------------------- decks/short_no_root.deck | 60 ----------------------------------- inputs/Makefile | 13 -------- inputs/README | 47 --------------------------- 11 files changed, 182 insertions(+), 180 deletions(-)

1 0

[ooni-probe/master] Add decks and inputs to MANIFEST.in
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 8612e5dd8b8df15be3b77b071ac1bde31f369adc Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 13 10:08:07 2013 +0100 Add decks and inputs to MANIFEST.in --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 2447812..adcc182 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ include README.md ChangeLog.md requirements.txt recursive-include data/ui/app * +recursive-include data/decks * +recursive-include data/inputs * include data/Makefile include data/ooniprobe.conf.sample

1 0

[ooni-probe/master] Revert "Fix report header in order to fix issue https://github.com/TheTorProject/ooni-probe/issues/223"
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 40146616bcca5404b1812afc581319b9ec3fa75a Author: aagbsn <aagbsn(a)extc.org> Date: Fri Nov 22 12:30:28 2013 +0000 Revert "Fix report header in order to fix issue https://github.com/TheTorProject/ooni-probe/issues/223" This reverts commit 72728414754c6df8defc133992788ba43af53615. Reason: Breaks report header parsing in ooni-backend --- ooni/deck.py | 5 +---- ooni/nettest.py | 5 ++--- ooni/oonicli.py | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 9054f25..7b8a8c4 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -108,9 +108,6 @@ class Deck(InputFile): self.deckHash = sha256(f.read()) f.seek(0) test_deck = yaml.safe_load(f) - - if isinstance(test_deck, dict): - test_deck = [test_deck] for test in test_deck: try: @@ -120,7 +117,7 @@ class Deck(InputFile): log.msg("Skipping...") continue net_test_loader = NetTestLoader(test['options']['subargs'], - test_file=nettest_path, global_options=test['options']) + test_file=nettest_path) #XXX: If the deck specifies the collector, we use the specified collector # And it should also specify the test helper address to use # net_test_loader.collector = test['options']['collector'] diff --git a/ooni/nettest.py b/ooni/nettest.py index f4ce3be..9304f2b 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -174,10 +174,9 @@ class NetTestLoader(object): method_prefix = 'test' collector = None - def __init__(self, options, test_file=None, test_string=None, global_options={}): + def __init__(self, options, test_file=None, test_string=None): self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]{64})$") self.options = options - self.globalOptions = global_options self.testCases, test_cases = None, None if test_file: @@ -288,7 +287,7 @@ class NetTestLoader(object): 'test_version': self.testVersion, 'software_name': 'ooniprobe', 'software_version': software_version, - 'options': self.globalOptions, + 'options': self.options, 'input_hashes': input_file_hashes } return test_details diff --git a/ooni/oonicli.py b/ooni/oonicli.py index a19c4ef..8622e09 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -143,7 +143,7 @@ def runWithDirector(): log.debug("No test deck detected") test_file = nettest_to_path(global_options['test_file']) net_test_loader = NetTestLoader(global_options['subargs'], - test_file=test_file, global_options=global_options) + test_file=test_file) deck.insert(net_test_loader) except errors.MissingRequiredOption, option_name: log.err('Missing required option: "%s"' % option_name)

1 0

[ooni-probe/master] Revert "Do not raise an exception on no input processor."
by art＠torproject.org 23 Nov '13

23 Nov '13

commit f32a9073607420591c6aabd395589ddf2e1a1835 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Nov 22 12:53:03 2013 +0000 Revert "Do not raise an exception on no input processor." This reverts commit 77cfa98526f43919619dacf27a0f5c2644313985. Reason: this commit breaks writing reports. --- ooni/nettest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index f008654..2982f4d 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -677,7 +677,7 @@ class NetTestCase(object): anything that gets written to the object self.report[] will be added to the final test report. """ - pass + raise NoPostProcessor def inputProcessor(self, filename): """

1 0

[ooni-probe/master] Add measurement result to measurement object.
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 359b99d1399761d2a6cf24cd66b21041cd51e4ab Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 18 14:41:36 2013 +0000 Add measurement result to measurement object. Simplify callback chain by adding the measurement result to the measurement object. --- ooni/director.py | 4 +++- ooni/managers.py | 2 +- ooni/nettest.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 628b812..7a81815 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -174,10 +174,11 @@ class Director(object): def measurementStarted(self, measurement): self.totalMeasurements += 1 - def measurementSucceeded(self, measurement): + def measurementSucceeded(self, result, measurement): log.msg("Successfully completed measurement: %s" % measurement) self.totalMeasurementRuntime += measurement.runtime self.successfulMeasurements += 1 + measurement.result = result return measurement def measurementFailed(self, failure, measurement): @@ -186,6 +187,7 @@ class Director(object): self.failedMeasurements += 1 self.failures.append((failure, measurement)) + measurement.result = failure return measurement def reporterFailed(self, failure, net_test): diff --git a/ooni/managers.py b/ooni/managers.py index a229044..b6e87e4 100644 --- a/ooni/managers.py +++ b/ooni/managers.py @@ -80,7 +80,7 @@ class TaskManager(object): self._fillSlots() # Fires the done deferred when the task has completed - task.done.callback(task) + task.done.callback(result) self.succeeded(result, task) @property diff --git a/ooni/nettest.py b/ooni/nettest.py index 2982f4d..b6dac29 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -535,9 +535,10 @@ class NetTest(object): measurement.netTest = self if self.director: - measurement.done.addCallback(self.director.measurementSucceeded) + measurement.done.addCallback(self.director.measurementSucceeded, + measurement) measurement.done.addErrback(self.director.measurementFailed, - measurement) + measurement) return measurement @defer.inlineCallbacks

1 0

[ooni-probe/master] Revert "Refactor the code of the post processor to work properly."
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 458ee9fc3b7efa542a9c343b078c65c5698f7021 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Nov 22 12:49:14 2013 +0000 Revert "Refactor the code of the post processor to work properly." This reverts commit e89163c608f27cd34cd871b6bcb814a24b65c762. Reason: this commit breaks writing reports. --- ooni/nettest.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 9304f2b..f008654 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -563,16 +563,25 @@ class NetTest(object): measurement = self.makeMeasurement(klass, method, input) measurements.append(measurement.done) self.state.taskCreated() - measurement.done.addCallback(klass.postProcessor) - if self.report: - @measurement.done.addCallback - def cb(*a): - self.report.write(measurement) - measurement.done.addBoth(self.doneReport) yield measurement # When the measurement.done callbacks have all fired # call the postProcessor before writing the report + if self.report: + post = defer.DeferredList(measurements) + + # Call the postProcessor, which must return a single report + # or a deferred + post.addCallback(klass.postProcessor) + def noPostProcessor(failure, report): + failure.trap(NoPostProcessor) + return report + post.addErrback(noPostProcessor, klass.report) + post.addCallback(self.report.write) + + if self.report and self.director: + #ghetto hax to keep NetTestState counts are accurate + [post.addBoth(self.doneReport) for _ in measurements] self.state.allTasksScheduled()

1 0

[ooni-probe/master] Use inlineCallbacks
by art＠torproject.org 23 Nov '13

23 Nov '13

commit e59174fe7d37840eec745bb851ec844e1948ab34 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Sep 18 20:46:59 2013 +0200 Use inlineCallbacks For some reason, using inlineCallbacks seems to produce a proper result from the doRequest method -- however, in the previous code the returned deferred had been called but the result was not set. --- ooni/nettests/blocking/http_requests.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index 48f8f94..20fae42 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -91,27 +91,27 @@ class HTTPRequestsTest(httpt.HTTPTest): self.report['headers_diff'] = diff self.report['headers_match'] = True + @defer.inlineCallbacks def test_get_experiment(self): log.msg("Performing GET request to %s" % self.url) - self.experiment = self.doRequest(self.url, method="GET", + self.experiment = yield self.doRequest(self.url, method="GET", use_tor=False, headers=self.headers) - return self.experiment + @defer.inlineCallbacks def test_get_control(self): log.msg("Performing GET request to %s over Tor" % self.url) - self.control = self.doRequest(self.url, method="GET", + self.control = yield self.doRequest(self.url, method="GET", use_tor=True, headers=self.headers) - return self.control def postProcessor(self, measurements): - if self.experiment.result and self.control.result: - self.compare_body_lengths(len(self.control.result.body), - len(self.experiment.result.body)) - self.compare_headers(self.control.result.headers, - self.experiment.result.headers) - - if not self.control.result: - self.report['control_failure'] = failureToString(self.control.result) - if not self.experiment.result: - self.report['experiment_failure'] = failureToString(self.experiment.result) + if self.experiment and self.control: + self.compare_body_lengths(len(self.control.body), + len(self.experiment.body)) + self.compare_headers(self.control.headers, + self.experiment.headers) + + if not self.control: + self.report['control_failure'] = failureToString(self.control) + if not self.experiment: + self.report['experiment_failure'] = failureToString(self.experiment) return self.report

1 0

[ooni-probe/master] Rewrite http_requests to use postProcessor API
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 0ae4e33dffd7650649df6b150b8bb6af414bbd60 Author: aagbsn <aagbsn(a)extc.org> Date: Wed Sep 18 19:14:04 2013 +0200 Rewrite http_requests to use postProcessor API Both the control and experiment tests can be rescheduled independently if we use the postProcessor API. --- ooni/nettests/blocking/http_requests.py | 67 +++++++++++-------------------- 1 file changed, 24 insertions(+), 43 deletions(-) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index 4e7d582..48f8f94 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -55,6 +55,7 @@ class HTTPRequestsTest(httpt.HTTPTest): self.factor = self.localOptions['factor'] self.report['control_failure'] = None self.report['experiment_failure'] = None + self.headers = {'User-Agent': [random.choice(userAgents)]} def compare_body_lengths(self, body_length_a, body_length_b): @@ -90,47 +91,27 @@ class HTTPRequestsTest(httpt.HTTPTest): self.report['headers_diff'] = diff self.report['headers_match'] = True - def test_get(self): - def callback(res): - experiment, control = res - experiment_succeeded, experiment_result = experiment - control_succeeded, control_result = control - - if control_succeeded and experiment_succeeded: - self.compare_body_lengths(len(experiment_result.body), - len(control_result.body)) - - self.compare_headers(control_result.headers, - experiment_result.headers) - else: - if not control_succeeded: - self.report['control_failure'] = failureToString(control_result) - - if not experiment_succeeded: - self.report['experiment_failure'] = failureToString(experiment_result) - # Now return some kind of failure so we can retry - # However, it would be ideal to split this test into two methods - # and compare the results in the postProcessor - # Sadly the postProcessor API is currently not implemented - if control_succeeded: - return experiment_result - return control_result - - headers = {'User-Agent': [random.choice(userAgents)]} - - l = [] + def test_get_experiment(self): log.msg("Performing GET request to %s" % self.url) - experiment_request = self.doRequest(self.url, method="GET", - headers=headers) - - control_request = self.doRequest(self.url, method="GET", - use_tor=True, headers=headers) - - l.append(experiment_request) - l.append(control_request) - - dl = defer.DeferredList(l, consumeErrors=True) - dl.addCallback(callback) - - return dl - + self.experiment = self.doRequest(self.url, method="GET", + use_tor=False, headers=self.headers) + return self.experiment + + def test_get_control(self): + log.msg("Performing GET request to %s over Tor" % self.url) + self.control = self.doRequest(self.url, method="GET", + use_tor=True, headers=self.headers) + return self.control + + def postProcessor(self, measurements): + if self.experiment.result and self.control.result: + self.compare_body_lengths(len(self.control.result.body), + len(self.experiment.result.body)) + self.compare_headers(self.control.result.headers, + self.experiment.result.headers) + + if not self.control.result: + self.report['control_failure'] = failureToString(self.control.result) + if not self.experiment.result: + self.report['experiment_failure'] = failureToString(self.experiment.result) + return self.report

1 0