February 2015 - tor-commits - lists.torproject.org

[ooni-probe/master] Make the DNS Consistency test handle errors better
by art＠torproject.org 04 Feb '15

04 Feb '15

commit 0cc9e06f7fc037c648a6a6ddca56e43e2b34a687 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 31 13:01:24 2015 +0100 Make the DNS Consistency test handle errors better --- ooni/nettests/blocking/dns_consistency.py | 31 +++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/ooni/nettests/blocking/dns_consistency.py b/ooni/nettests/blocking/dns_consistency.py index c4cae56..88e5d5e 100644 --- a/ooni/nettests/blocking/dns_consistency.py +++ b/ooni/nettests/blocking/dns_consistency.py @@ -112,17 +112,23 @@ class DNSConsistencyTest(dnst.DNSTest): self.report['tampering'] = {} - control_answers = yield self.performALookup(hostname, - self.control_dns_server) - if not control_answers: - log.err( - "Got no response from control DNS server %s:%d, " - "perhaps the DNS resolver is down?" % - self.control_dns_server) + try: + control_answers = yield self.performALookup(hostname, + self.control_dns_server) + + if not control_answers: + log.err( + "Got no response from control DNS server %s:%d, " + "perhaps the DNS resolver is down?" % + self.control_dns_server) + self.report['tampering'][ + "%s:%d" % + self.control_dns_server] = 'no_answer' + except: self.report['tampering'][ "%s:%d" % - self.control_dns_server] = 'no_answer' - return + self.control_dns_server] = 'error' + control_answers = None for test_resolver in self.test_resolvers: log.msg("Testing resolver: %s" % test_resolver) @@ -156,7 +162,12 @@ class DNSConsistencyTest(dnst.DNSTest): log.debug( "Comparing %s with %s" % (experiment_answers, control_answers)) - if set(experiment_answers) & set(control_answers): + + if not control_answers: + log.msg("Skipping control resolver comparison") + self.report['tampering'][test_resolver] = None + + elif set(experiment_answers) & set(control_answers): lookup_details() log.msg("tampering: false") self.report['tampering'][test_resolver] = False

1 0

[ooni-probe/master] Implement a hotfix for https://github.com/habnabit/txsocksx/issues/9
by art＠torproject.org 04 Feb '15

04 Feb '15

commit 96391035de250a2d5842c5c0e68879f1838eb874 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 4 11:18:05 2015 +0100 Implement a hotfix for https://github.com/habnabit/txsocksx/issues/9 --- ooni/oonibclient.py | 3 ++- ooni/reporter.py | 2 +- ooni/utils/hacks.py | 29 +++++++++++++++++++++++++++++ ooni/utils/trueheaders.py | 2 +- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py index a26b771..126eeb7 100644 --- a/ooni/oonibclient.py +++ b/ooni/oonibclient.py @@ -203,7 +203,8 @@ class OONIBClient(object): try: test_collector = yield self.queryBackend('POST', '/bouncer/net-tests', query={'net-tests': net_tests}) - except Exception: + except Exception as exc: + log.exception(exc) raise e.CouldNotFindTestCollector defer.returnValue(test_collector) diff --git a/ooni/reporter.py b/ooni/reporter.py index 34ff333..bf4a645 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -287,7 +287,7 @@ class OONIBReporter(OReporter): # do this with some deferred kung foo or instantiate the reporter after # tor is started. - from txsocksx.http import SOCKS5Agent + from ooni.utils.hacks import SOCKS5Agent from twisted.internet import reactor if self.collectorAddress.startswith('httpo://'): diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py index 2c84a23..e001d3f 100644 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@ -2,6 +2,9 @@ # hackish way, we put it in here. This one day will be empty. import copy_reg +from twisted.web.client import SchemeNotSupported + +from txsocksx.http import SOCKS5Agent as SOCKS5AgentOriginal def patched_reduce_ex(self, proto): @@ -56,3 +59,29 @@ def patched_reduce_ex(self, proto): else: return copy_reg._reconstructor, args + +class SOCKS5Agent(SOCKS5AgentOriginal): + """ + This is a quick hack to fix: + https://github.com/habnabit/txsocksx/issues/9 + """ + def _getEndpoint(self, scheme_or_uri, host=None, port=None): + if host is not None: + scheme = scheme_or_uri + else: + scheme = scheme_or_uri.scheme + host = scheme_or_uri.host + port = scheme_or_uri.port + if scheme not in ('http', 'https'): + raise SchemeNotSupported('unsupported scheme', scheme) + endpoint = self.endpointFactory( + host, port, self.proxyEndpoint, **self.endpointArgs) + if scheme == 'https': + if hasattr(self, '_wrapContextFactory'): + tlsPolicy = self._wrapContextFactory(host, port) + elif hasattr(self, '_policyForHTTPS'): + tlsPolicy = self._policyForHTTPS.creatorForNetloc(host, port) + else: + raise NotImplementedError("can't figure out how to make a context factory") + endpoint = self._tlsWrapper(tlsPolicy, endpoint) + return endpoint diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index e5dcd0a..f015b62 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -12,7 +12,7 @@ from twisted.web._newclient import RequestNotSent, RequestGenerationFailed, Tran from twisted.internet import reactor from twisted.internet.defer import Deferred, fail, maybeDeferred, failure -from txsocksx.http import SOCKS5Agent +from ooni.utils.hacks import SOCKS5Agent from txsocksx.client import SOCKS5ClientFactory SOCKS5ClientFactory.noisy = False

1 0

[ooni-probe/master] Tag release candiate
by art＠torproject.org 04 Feb '15

04 Feb '15

commit 5ad2ba0e790753b702e518ec2bd3c659d1989f59 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 4 12:39:47 2015 +0100 Tag release candiate --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 305222c..a83105a 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.2.2" +__version__ = "1.2.3-rc1" __all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'report', 'reporter',

1 0

[ooni-probe/master] Set a default bouncer
by art＠torproject.org 04 Feb '15

04 Feb '15

commit bc638e8c477b87933cf31bd6f5623a5dfeeb7630 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 4 14:19:00 2015 +0100 Set a default bouncer --- ooni/report/cli.py | 2 +- ooni/report/tool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/report/cli.py b/ooni/report/cli.py index e52b022..c1b4a22 100644 --- a/ooni/report/cli.py +++ b/ooni/report/cli.py @@ -20,7 +20,7 @@ class Options(usage.Options): "Specify the configuration file to use."], ["collector", "c", None, "Specify the collector to upload the result to."], - ["bouncer", "b", None, + ["bouncer", "b", "httpo://nkvphnp3p6agi5qq.onion", "Specify the bouncer to query for a collector."] ] diff --git a/ooni/report/tool.py b/ooni/report/tool.py index 9eaad5e..a16794a 100644 --- a/ooni/report/tool.py +++ b/ooni/report/tool.py @@ -20,7 +20,7 @@ def upload(report_file, collector=None, bouncer=None): report_log = yaml.safe_load(f) report = parser.ReportLoader(report_file) - if bouncer: + if bouncer and not collector: oonib_client = OONIBClient(bouncer) net_tests = [{ 'test-helpers': [],

1 0

[ooni-probe/master] Update changelog for ooniprobe 1.2.3-rc1
by art＠torproject.org 04 Feb '15

04 Feb '15

commit e358243b21fe8e88b8894cdd94aa4dab3d4f92ce Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 4 14:24:46 2015 +0100 Update changelog for ooniprobe 1.2.3-rc1 --- ChangeLog.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index cddf18f..92f5109 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,17 @@ Changelog ========= +v1.2.3-rc1 (Wed, 4 Feb 2015) +------------------------ +* Restructure directories where ooni software writes/reads from + https://trac.torproject.org/projects/tor/ticket/14086 + +* Properly set exit codes of oonideckgen + +* Exit cleanly if we can't find the probes IP address + +* Make the DNS Consistency test handle errors better + v1.2.2 (Fri, 17 Oct 2014) -------------------------

1 0

[ooni-probe/master] Fix some bugs in oonireport
by art＠torproject.org 04 Feb '15

04 Feb '15

commit eb1daf87240830d8c875455ad92a708ce8443d65 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 4 14:17:55 2015 +0100 Fix some bugs in oonireport --- ooni/report/tool.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ooni/report/tool.py b/ooni/report/tool.py index 11dbeda..9eaad5e 100644 --- a/ooni/report/tool.py +++ b/ooni/report/tool.py @@ -22,9 +22,16 @@ def upload(report_file, collector=None, bouncer=None): report = parser.ReportLoader(report_file) if bouncer: oonib_client = OONIBClient(bouncer) - collector = yield oonib_client.lookupTestCollector( - report.header['test_name'] + net_tests = [{ + 'test-helpers': [], + 'input-hashes': report.header['input_hashes'], + 'name': report.header['test_name'], + 'version': report.header['test_version'], + }] + result = yield oonib_client.lookupTestCollector( + net_tests ) + collector = str(result['net-tests'][0]['collector']) if collector is None: try: @@ -57,7 +64,7 @@ def upload_all(collector=None, bouncer=None): try: yield upload(report_file, collector, bouncer) except Exception as exc: - print exc + log.exception(exc) def print_report(report_file, value):

1 0

[ooni-probe/master] Merge pull request #376 from TheTorProject/fix/deckgen-stall
by art＠torproject.org 04 Feb '15

04 Feb '15

commit cd8618aee6ed78c0d223ff6448b339ef11b9c696 Merge: d3981af 9639103 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Feb 4 11:30:03 2015 +0100 Merge pull request #376 from TheTorProject/fix/deckgen-stall Fix/deckgen stall bin/oonideckgen | 10 ++++++++++ bin/oonireport | 19 ++++++++++++++++-- data/ooniprobe.conf.sample | 1 + ooni/deckgen/cli.py | 8 +++++++- ooni/nettests/blocking/dns_consistency.py | 31 +++++++++++++++++++---------- ooni/oonibclient.py | 3 ++- ooni/report/cli.py | 6 +++++- ooni/report/tool.py | 12 +++++++---- ooni/reporter.py | 6 ++++-- ooni/settings.py | 9 +++++++-- ooni/utils/hacks.py | 29 +++++++++++++++++++++++++++ ooni/utils/trueheaders.py | 2 +- 12 files changed, 112 insertions(+), 24 deletions(-)

1 0

[ooni-probe/master] Run ooniresources only in post install
by art＠torproject.org 04 Feb '15

04 Feb '15

commit f5ee5e39c337c31e75e011fc7dcec4e3cc8e342c Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Feb 4 12:39:21 2015 +0100 Run ooniresources only in post install Fix inclusion of inputs in MANIFEST --- MANIFEST.in | 2 +- setup.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 71c4a58..920f933 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ include README.rst ChangeLog.rst requirements.txt LICENSE recursive-include data/decks * -include data/inputs/ +recursive-include data/inputs * include data/oonideckgen.1 include data/ooniprobe.1 include data/oonireport.1 diff --git a/setup.py b/setup.py index 62335d9..54ec4ee 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ from ConfigParser import SafeConfigParser from os.path import join as pj from setuptools import setup from setuptools.command.install import install as _st_install - +from distutils.spawn import find_executable class install(_st_install): def gen_config(self, share_path): @@ -67,11 +67,18 @@ class install(_st_install): except OSError: pass + def ooniresources(self): + ooniresources = find_executable("ooniresources") + from subprocess import Popen, PIPE + process = Popen([ooniresources, '--update-inputs', '--update-geoip'], + stdout=sys.stdout.fileno(), stderr=sys.stderr.fileno()) + process.wait() + def run(self): prefix = os.path.abspath(self.prefix) self.set_data_files(prefix) self.do_egg_install() - + self.ooniresources() install_requires = [] dependency_links = [] @@ -163,8 +170,3 @@ setup( "Topic :: System :: Networking :: Monitoring", ) ) - -from subprocess import Popen, PIPE -process = Popen(['ooniresources', '--update-inputs', '--update-geoip'], - stdout=sys.stdout.fileno(), stderr=sys.stderr.fileno()) -process.wait()

1 0

[ooni-probe/master] Use the new function for determining the geoip data files path
by art＠torproject.org 04 Feb '15

04 Feb '15

commit 6d2d12aee6513b69795777505a51685d3e5e06de Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 3 18:04:07 2015 +0100 Use the new function for determining the geoip data files path --- ooni/geoip.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ooni/geoip.py b/ooni/geoip.py index 9c3ee31..1ff90a7 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -81,12 +81,8 @@ def database_version(): } } - geoip_data_dir = config.advanced.get("geoip_data_dir") - if not geoip_data_dir: - return version - for key in version.keys(): - geoip_file = os.path.join(geoip_data_dir, key + ".dat") + geoip_file = config.get_data_file_path("GeoIP/" + key + ".dat") if not os.path.isfile(geoip_file): continue timestamp = os.stat(geoip_file).st_mtime

1 0

[ooni-probe/master] Re-add manpages to manifest
by art＠torproject.org 04 Feb '15

04 Feb '15

commit 8d7a40ce66777b5ff49569c9ed2c51ed79b74159 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Jan 4 12:18:24 2015 +0100 Re-add manpages to manifest --- MANIFEST.in | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 9b70d09..71c4a58 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,9 @@ include README.rst ChangeLog.rst requirements.txt LICENSE recursive-include data/decks * include data/inputs/ +include data/oonideckgen.1 +include data/ooniprobe.1 +include data/oonireport.1 +include data/ooniresources.1 include data/ooniprobe.conf.sample include ooni/settings.ini

1 0