November 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Indicate whether Tor was used in a request or not in the report.
by art＠torproject.org 23 Nov '13

23 Nov '13

commit d766cf68fcc720b6d0bce900883fc3610b41026b Author: aagbsn <aagbsn(a)extc.org> Date: Fri Nov 22 13:44:32 2013 +0000 Indicate whether Tor was used in a request or not in the report. --- ooni/templates/httpt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index b6272e9..04c0708 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -126,7 +126,8 @@ class HTTPTest(NetTestCase): 'headers': list(request_headers.getAllRawHeaders()), 'body': request['body'], 'url': request['url'], - 'method': request['method'] + 'method': request['method'], + 'tor': request['tor'] } } if response:

1 0

[ooni-probe/master] Refactor postProcessor to accommodate result callback refactoring.
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 2f232b9fac40f949bbad248c4861e7a1b403c493 Author: aagbsn <aagbsn(a)extc.org> Date: Mon Nov 18 14:45:10 2013 +0000 Refactor postProcessor to accommodate result callback refactoring. --- ooni/nettests/blocking/http_requests.py | 36 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index 20fae42..c3c72af 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -5,7 +5,7 @@ import random from twisted.internet import defer -from twisted.python import usage +from twisted.python import usage, failure from ooni.utils import log from ooni.utils.net import userAgents @@ -91,27 +91,33 @@ class HTTPRequestsTest(httpt.HTTPTest): self.report['headers_diff'] = diff self.report['headers_match'] = True - @defer.inlineCallbacks def test_get_experiment(self): log.msg("Performing GET request to %s" % self.url) - self.experiment = yield self.doRequest(self.url, method="GET", + return self.doRequest(self.url, method="GET", use_tor=False, headers=self.headers) - @defer.inlineCallbacks def test_get_control(self): log.msg("Performing GET request to %s over Tor" % self.url) - self.control = yield self.doRequest(self.url, method="GET", + return self.doRequest(self.url, method="GET", use_tor=True, headers=self.headers) def postProcessor(self, measurements): - if self.experiment and self.control: - self.compare_body_lengths(len(self.control.body), - len(self.experiment.body)) - self.compare_headers(self.control.headers, - self.experiment.headers) - - if not self.control: - self.report['control_failure'] = failureToString(self.control) - if not self.experiment: - self.report['experiment_failure'] = failureToString(self.experiment) + experiment = control = None + for status, measurement in measurements: + if 'experiment' in str(measurement.netTestMethod): + if isinstance(measurement.result, failure.Failure): + self.report['experiment_failure'] = failureToString(measurement.result) + else: + experiment = measurement.result + elif 'control' in str(measurement.netTestMethod): + if isinstance(measurement.result, failure.Failure): + self.report['control_failure'] = failureToString(measurement.result) + else: + control = measurement.result + + if experiment and control: + self.compare_body_lengths(len(control.body), + len(experiment.body)) + self.compare_headers(control.headers, + experiment.headers) return self.report

1 0

[ooni-probe/master] Merge pull request #245 from TheTorProject/fix/logfile_option
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 0808489fcadd30adc4fecb09c53efb9107259f6c Merge: f32a907 172df77 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Fri Nov 22 08:10:25 2013 -0800 Merge pull request #245 from TheTorProject/fix/logfile_option Fix/logfile option ooni/utils/log.py | 3 --- 1 file changed, 3 deletions(-)

1 0

[ooni-probe/master] Remove test for W_OK on logfile
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 172df77d428100ca823c374c99edc75534b7d0b6 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Nov 22 14:17:32 2013 +0000 Remove test for W_OK on logfile Removes the test for W_OK on logfile because it fails when the logfile does not exist, and we do not do a similar test for the default ooniprobe.log filename. --- ooni/utils/log.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 3aedc17..036a4b4 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -30,9 +30,6 @@ def start(logfile=None, application_name="ooniprobe"): if not logfile: logfile = config.basic.logfile - if not (os.access(logfile, os.W_OK)): - logfile = "ooniprobe.log" - log_folder = os.path.dirname(logfile) log_filename = os.path.basename(logfile)

1 0

[ooni-probe/master] Properly encode the helper address
by art＠torproject.org 23 Nov '13

23 Nov '13

commit b95e35380e143859b1eecc493cb5bb0071d6eafe Author: aagbsn <aagbsn(a)extc.org> Date: Fri Nov 22 16:50:16 2013 +0000 Properly encode the helper address --- ooni/deck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/deck.py b/ooni/deck.py index 7b8a8c4..d7b2d7e 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -177,7 +177,7 @@ class Deck(InputFile): continue test_helper = response[th['name']] log.msg("Using this helper: %s" % test_helper) - th['test_class'].localOptions[th['option']] = test_helper['address'] + th['test_class'].localOptions[th['option']] = test_helper['address'].encode('utf-8') if net_test_loader in requires_collector: net_test_loader.collector = test_helper['collector'].encode('utf-8')

1 0

[ooni-probe/master] Merge pull request #246 from TheTorProject/fix/encode_helper_address
by art＠torproject.org 23 Nov '13

23 Nov '13

commit db7d028b630f07b2f831679233e16faddce549a7 Merge: 0808489 b95e353 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Fri Nov 22 08:53:38 2013 -0800 Merge pull request #246 from TheTorProject/fix/encode_helper_address Properly encode the helper address ooni/deck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[ooni-probe/master] Merge branch 'feature/use_postprocessor_with_http_requests'
by art＠torproject.org 23 Nov '13

23 Nov '13

commit ff4a709f5371528ead0fafee07c782cdc62c874c Merge: db7d028 d766cf6 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Nov 22 18:01:08 2013 +0100 Merge branch 'feature/use_postprocessor_with_http_requests' * feature/use_postprocessor_with_http_requests: Indicate whether Tor was used in a request or not in the report. Refactor postProcessor to accommodate result callback refactoring. Add measurement result to measurement object. Use inlineCallbacks Rewrite http_requests to use postProcessor API ooni/director.py | 4 +- ooni/managers.py | 2 +- ooni/nettest.py | 5 ++- ooni/nettests/blocking/http_requests.py | 75 +++++++++++++------------------ ooni/templates/httpt.py | 3 +- 5 files changed, 40 insertions(+), 49 deletions(-)

1 0

[ooni-probe/master] Gotta catch em all
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 4ab0914ef209680512b5b0ae827f7d06866a5858 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Nov 22 18:54:50 2013 +0000 Gotta catch em all --- ooni/nettest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index c813c37..7049ad1 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -575,7 +575,7 @@ class NetTest(object): # or a deferred post.addCallback(klass.postProcessor) def noPostProcessor(failure, report): - failure.trap(NoPostProcessor) + failure.trap(e.NoPostProcessor) return report post.addErrback(noPostProcessor, klass.report) post.addCallback(self.report.write)

1 0

[ooni-probe/master] Check to see if a required option is a test helper we can look up
by art＠torproject.org 23 Nov '13

23 Nov '13

commit ea00095a440cbf84664dfec8880715d4615ead14 Author: aagbsn <aagbsn(a)extc.org> Date: Fri Nov 22 18:10:06 2013 +0000 Check to see if a required option is a test helper we can look up --- ooni/deck.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index d7b2d7e..23bb9a1 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -118,15 +118,23 @@ class Deck(InputFile): continue net_test_loader = NetTestLoader(test['options']['subargs'], test_file=nettest_path) + self.insert(net_test_loader) #XXX: If the deck specifies the collector, we use the specified collector # And it should also specify the test helper address to use # net_test_loader.collector = test['options']['collector'] - net_test_loader.checkOptions() - self.netTestLoaders.append(net_test_loader) def insert(self, net_test_loader): """ Add a NetTestLoader to this test deck """ - net_test_loader.checkOptions() + try: + net_test_loader.checkOptions() + except e.MissingRequiredOption, option_name: + if not self.bouncer: + raise + for rth in net_test_loader.requiredTestHelpers: + if option_name.message == rth['option']: + break + else: + raise self.netTestLoaders.append(net_test_loader) @defer.inlineCallbacks

1 0

[ooni-probe/master] Move exceptions into ooni/errors.py
by art＠torproject.org 23 Nov '13

23 Nov '13

commit 50b7b702be664795fb1c5699c52f1f0d9322e58e Author: aagbsn <aagbsn(a)extc.org> Date: Fri Nov 22 17:39:37 2013 +0000 Move exceptions into ooni/errors.py --- ooni/errors.py | 9 +++++++++ ooni/nettest.py | 23 +++++++---------------- ooni/oonicli.py | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index a124487..c167bbe 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -195,6 +195,15 @@ class NetTestNotFound(Exception): class MissingRequiredOption(Exception): pass +class FailureToLoadNetTest(Exception): + pass + +class NoPostProcessor(Exception): + pass + +class InvalidOption(Exception): + pass + def get_error(error_key): if error_key == 'test-helpers-key-missing': return CouldNotFindTestHelper diff --git a/ooni/nettest.py b/ooni/nettest.py index b6dac29..c813c37 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -58,7 +58,7 @@ def loadNetTestString(net_test_string): test_cases.extend(get_test_methods(item)) if not test_cases: - raise NoTestCasesFound + raise e.NoTestCasesFound return test_cases @@ -72,7 +72,7 @@ def loadNetTestFile(net_test_file): test_cases.extend(get_test_methods(item)) if not test_cases: - raise NoTestCasesFound + raise e.NoTestCasesFound return test_cases @@ -351,7 +351,7 @@ class NetTestLoader(object): test_cases.extend(self._get_test_methods(item)) if not test_cases: - raise NoTestCasesFound + raise e.NoTestCasesFound self.setupTestCases(test_cases) @@ -365,7 +365,7 @@ class NetTestLoader(object): test_cases.extend(self._get_test_methods(item)) if not test_cases: - raise NoTestCasesFound + raise e.NoTestCasesFound self.setupTestCases(test_cases) @@ -678,7 +678,7 @@ class NetTestCase(object): anything that gets written to the object self.report[] will be added to the final test report. """ - raise NoPostProcessor + raise e.NoPostProcessor def inputProcessor(self, filename): """ @@ -758,23 +758,14 @@ class NetTestCase(object): for option in self.localOptions: if option not in self.usageOptions(): if not self.inputFile or option not in self.inputFile: - raise InvalidOption + raise e.InvalidOption def _checkRequiredOptions(self): for required_option in self.requiredOptions: log.debug("Checking if %s is present" % required_option) if required_option not in self.localOptions or \ self.localOptions[required_option] == None: - raise MissingRequiredOption(required_option) + raise e.MissingRequiredOption(required_option) def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) - -class FailureToLoadNetTest(Exception): - pass -class NoPostProcessor(Exception): - pass -class InvalidOption(Exception): - pass -class MissingRequiredOption(Exception): - pass diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 8622e09..c722c52 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -16,7 +16,7 @@ from ooni.settings import config from ooni.director import Director from ooni.deck import Deck, nettest_to_path from ooni.reporter import YAMLReporter, OONIBReporter -from ooni.nettest import NetTestLoader, MissingRequiredOption +from ooni.nettest import NetTestLoader from ooni.utils import log, checkForRoot

1 0