September 2017 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge branch 'master' into fix/tor-binary
by art＠torproject.org 22 Sep '17

22 Sep '17

commit eec13dff2c463cb9b185544b9cae88f081ca9acd Merge: 92ce2cb0 b63e9c5b Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jun 30 18:11:23 2017 +0200 Merge branch 'master' into fix/tor-binary LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[ooni-probe/master] comment about errback placement
by art＠torproject.org 22 Sep '17

22 Sep '17

commit 33daa2edf4e9406f4e81b97029a9ce9ba3d564c1 Author: meejah <meejah(a)meejah.ca> Date: Wed Apr 26 14:54:56 2017 -0600 comment about errback placement --- ooni/utils/onion.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index c4d5d4c3..283579cd 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -379,6 +379,9 @@ class TorLauncherWithRetries(object): log.debug("Building a TorState") config.tor.protocol = proto state = TorState(proto.tor_protocol) + # note that errors from _state_complete will only get handled + # if we add the errback *after* -- which is why this isn't + # using .addCallbacks(..) state.post_bootstrap.addCallback(self._state_complete) state.post_bootstrap.addErrback(self._setup_failed)

1 0

[ooni-probe/master] [ci skip] Bump copyright year in the license (#774)
by art＠torproject.org 22 Sep '17

22 Sep '17

commit b63e9c5b85a7a65f4369b1f30a89c35143619467 Author: Simone Basso <bassosimone(a)gmail.com> Date: Fri Jun 30 18:11:09 2017 +0200 [ci skip] Bump copyright year in the license (#774) Straight from the bikeshedding dept. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index acf08098..ba919300 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012-2016, Jacob Appelbaum, Aaron Gibson, Arturo Filastò, +Copyright (c) 2012-2017, Jacob Appelbaum, Aaron Gibson, Arturo Filastò, Isis Lovecruft, The Tor Project. All rights reserved.

1 0

[ooni-probe/master] Update test_onion unittests
by art＠torproject.org 22 Sep '17

22 Sep '17

commit 92ce2cb0351a0f19f37c8f7b822dcb8cc3965f8f Author: Arturo Filastò <arturo(a)filasto.net> Date: Sun Jun 25 19:01:07 2017 +0200 Update test_onion unittests --- ooni/tests/test_onion.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ooni/tests/test_onion.py b/ooni/tests/test_onion.py index 0807ca02..20023211 100644 --- a/ooni/tests/test_onion.py +++ b/ooni/tests/test_onion.py @@ -49,21 +49,26 @@ class TestOnion(unittest.TestCase): self.assertEqual(onion.bridge_line(transport, '/log.txt'), exp_line) - with patch.dict(onion.obfsproxy_details, - {'version': onion.OBFSProxyVersion('0.1.12')}): + def mock_get_details(value): + def mocked_get_details(): + return value + return mocked_get_details + + with patch('ooni.utils.onion.get_obfsproxy_details', + mock_get_details({'version': onion.OBFSProxyVersion('0.1.12')})): self.assertRaises(onion.OutdatedObfsproxy, onion.bridge_line, 'obfs2', '/log.txt') - with patch.dict(onion.tor_details, - {'version': onion.TorVersion('0.2.4.20')}): + with patch('ooni.utils.onion.get_tor_details', + mock_get_details({'version': onion.TorVersion('0.2.4.20')})): onion.bridge_line('fte', '/log.txt') self.assertRaises(onion.OutdatedTor, onion.bridge_line, 'scramblesuit', '/log.txt') self.assertRaises(onion.OutdatedTor, onion.bridge_line, 'obfs4', '/log.txt') - with patch.dict(onion.tor_details, - {'version': onion.TorVersion('0.2.3.20')}): + with patch('ooni.utils.onion.get_tor_details', + mock_get_details({'version': onion.TorVersion('0.2.3.20')})): self.assertRaises(onion.OutdatedTor, onion.bridge_line, 'fte', '/log.txt')

1 0

[ooni-probe/master] Feature/http n https (#770)
by art＠torproject.org 22 Sep '17

22 Sep '17

commit ff1880092e8cf8777b769e152675edf7e2d7641b Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jun 15 12:05:11 2017 +0200 Feature/http n https (#770) * Add support for testing via http when the site is https * Clarify scope of `--no-http` settings --- ooni/nettests/blocking/web_connectivity.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 3807a0de..a4db157e 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -39,6 +39,7 @@ class UsageOptions(usage.Options): ] optFlags = [ ['no-shuffle', '', 'Disable shuffling of URLs'], + ['no-http', '', 'Disable testing also http for https sites specified in the test list (i.e. if you specify `-u <URL>` _only_ that <URL> will be tested)'], ] @@ -52,7 +53,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): "connect to the resolved IPs and then fetching the page " "and comparing all these results with those of a control.") author = "Arturo Filastò" - version = "0.3.0" + version = "0.3.2" contentDecoders = [('gzip', GzipDecoder)] @@ -136,6 +137,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): if (not i.startswith("http://") and not i.startswith("https://")): i = "http://{}/".format(i) + if i.startswith('https://') and self.localOptions['no-http'] != True: + yield 'http'+i[5:] yield i finally: fh.close()

1 0

[ooni-probe/master] Use txtorcon 0.18.0 also in Twisted 15.2
by art＠torproject.org 22 Sep '17

22 Sep '17

commit 123ad18cfe141b50aeb4f3a95ec16b704b9466f6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 15 10:37:26 2017 +0000 Use txtorcon 0.18.0 also in Twisted 15.2 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 38becd03..c9dc7d9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ env: # version in debian jessie - TWISTED=Twisted==14.0.2 TXTORCON=txtorcon==0.18.0 # version in ubuntu wily - - TWISTED=Twisted==15.2.1 TXTORCON=txtorcon + - TWISTED=Twisted==15.2.1 TXTORCON=txtorcon==0.18.0 # version in ubuntu xenial - TWISTED=Twisted==16.0.0 TXTORCON=txtorcon # version in debian jessie-backports

1 0

[ooni-probe/master] add errback after the callback
by art＠torproject.org 22 Sep '17

22 Sep '17

commit c491f79f03e24d118fcbbde0a003584002a054a2 Author: meejah <meejah(a)meejah.ca> Date: Wed Apr 26 14:17:49 2017 -0600 add errback after the callback this is a bit subtle, but if any error happens in _setup_complete itself, the errback only sees it if it is added *after* the callback --- ooni/utils/onion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 9d41b29a..691eda05 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -378,8 +378,8 @@ class TorLauncherWithRetries(object): log.debug("Building a TorState") config.tor.protocol = proto state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallbacks(self._state_complete, - self._setup_failed) + state.post_bootstrap.addCallback(self._state_complete) + state.post_bootstrap.addErrback(self._setup_failed) def _launch_tor(self): return launch_tor(self.tor_config, reactor,

1 0

[ooni-probe/master] Add icon support to decks
by art＠torproject.org 22 Sep '17

22 Sep '17

commit f14488ae768d468e4f4cea19ca3d3df793219313 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Jan 2 13:34:23 2017 +0000 Add icon support to decks --- data/decks/http-invalid.yaml | 1 + data/decks/im.yaml | 1 + data/decks/tor.yaml | 1 + data/decks/web.yaml | 1 + ooni/__init__.py | 2 +- ooni/deck/deck.py | 2 ++ 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/data/decks/http-invalid.yaml b/data/decks/http-invalid.yaml index 9690376c..405ec5da 100644 --- a/data/decks/http-invalid.yaml +++ b/data/decks/http-invalid.yaml @@ -2,6 +2,7 @@ name: HTTP Invalid Request Line description: This deck runs HTTP Invalid Request Line test schedule: "@daily" +icon: "fa-gears" tasks: - name: Runs the HTTP Invalid Request Line test ooni: diff --git a/data/decks/im.yaml b/data/decks/im.yaml index 98dc27ae..31c88d3d 100644 --- a/data/decks/im.yaml +++ b/data/decks/im.yaml @@ -2,6 +2,7 @@ name: Instant Messaging deck description: This test deck will check to see if instant messaging applications are working schedule: "@daily" +icon: "fa-comment-o" tasks: - name: Checks to see if WhatsApp is working ooni: diff --git a/data/decks/tor.yaml b/data/decks/tor.yaml index ea6e1a3f..d732f345 100644 --- a/data/decks/tor.yaml +++ b/data/decks/tor.yaml @@ -2,6 +2,7 @@ name: Tor test deck description: This deck runs test related to testing the reachability of the Tor network schedule: "@daily" +icon: "fa-wrench" tasks: - name: Checks if the Tor client works ooni: diff --git a/data/decks/web.yaml b/data/decks/web.yaml index 9232cec1..7e9b9c8f 100644 --- a/data/decks/web.yaml +++ b/data/decks/web.yaml @@ -2,6 +2,7 @@ name: Web test deck description: This deck runs HTTP Header Field Manipulation and the Web Connectivity test schedule: "@daily" +icon: "fa-unlink" tasks: - name: Runs the HTTP Header Field Manipulation test ooni: diff --git a/ooni/__init__.py b/ooni/__init__.py index a0887e07..4b4a1bcb 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "2.1.0" +__version__ = "2.2.0.dev0" __all__ = [ 'agent', diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index 50887b55..07f0799e 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -81,6 +81,7 @@ class NGDeck(object): self.no_collector = no_collector self.name = "" self.description = "" + self.icon = "" self.schedule = None self.metadata = {} @@ -113,6 +114,7 @@ class NGDeck(object): self.name = deck_data.pop("name", "Un-named Deck") self.description = deck_data.pop("description", "No description") + self.icon = deck_data.pop("icon", "fa-gears") bouncer_address = self.global_options.get('bouncer', deck_data.pop("bouncer", None))

1 0

[ooni-probe/master] Update settings.py (#721)
by art＠torproject.org 22 Sep '17

22 Sep '17

commit dece0974e1d7d1affa04c12cd575fa04d2866da9 Author: E.Iosifidis <iosifidise(a)gmail.com> Date: Fri Jan 20 17:30:08 2017 +0200 Update settings.py (#721) --- ooni/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/settings.py b/ooni/settings.py index a59664cd..51009fb2 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -532,8 +532,8 @@ class OConfig(object): except Exception: incoherent.append('tor:control_port') else: - conf_unix_socket_path = self.tor.control_port - if conf_unix_socket_path.lstrip.startswith("unix:"): + conf_unix_socket_path = self.tor.control_port.lstrip() + if conf_unix_socket_path.startswith("unix:"): if os.path.exists(conf_unix_socket_path.lstrip("unix:")): unix_socket_path = conf_unix_socket_path.lstrip("unix:") else:

1 0

[ooni-probe/master] Include the deck_id in the returned value
by art＠torproject.org 22 Sep '17

22 Sep '17

commit bcb93b889d55a22347f2d1f2d01a63c707fb7ef1 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jan 12 12:54:17 2017 +0000 Include the deck_id in the returned value --- ooni/deck/deck.py | 8 +++++--- ooni/measurements.py | 13 ++++++++----- ooni/ui/web/server.py | 8 +++++++- ooni/utils/__init__.py | 6 ++++-- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index fbbb8c3d..c57ccb63 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -234,7 +234,7 @@ class NGDeck(object): # XXX-REFACTOR we do this so late to avoid the collision between the # same id and hence generating the same filename. test_details = net_test_loader.getTestDetails() - task.id = generate_filename(test_details) + task.id = generate_filename(test_details, deck_id=self.id) measurement_id = None report_filename = task.output_path @@ -330,7 +330,8 @@ class DeckTask(object): self.ooni = { 'bouncer_client': None, - 'test_details': {} + 'test_details': {}, + 'test_name': None } self.output_path = None @@ -353,8 +354,9 @@ class DeckTask(object): if required_key not in task_data: raise MissingTaskDataKey(required_key) + self.ooni['test_name'] = task_data.pop('test_name') # This raises e.NetTestNotFound, we let it go onto the caller - nettest_path = nettest_to_path(task_data.pop("test_name"), + nettest_path = nettest_to_path(self.ooni['test_name'], self._arbitrary_paths) annotations = self._pop_option('annotations', task_data, {}) diff --git a/ooni/measurements.py b/ooni/measurements.py index 9197ee75..68b5c555 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -106,8 +106,11 @@ def get_measurement(measurement_id, compute_size=False): if compute_size is True: size = directory_usage(measurement.path) - test_start_time, country_code, asn, test_name = \ - measurement_id.split("-")[:4] + measurement_metadata = measurement_id.split("-") + test_start_time, country_code, asn, test_name = measurement_metadata[:4] + deck_id = "none" + if len(measurement_metadata) > 4: + deck_id = '-'.join(measurement_metadata[4:]) return { "test_name": test_name, "country_code": country_code, @@ -119,8 +122,7 @@ def get_measurement(measurement_id, compute_size=False): "running": running, "stale": stale, "size": size, - # XXX we need the deck ID in here - "deck_id": "none" + "deck_id": deck_id } @@ -156,8 +158,9 @@ def list_measurements(compute_size=False, order=None): for measurement_id in measurement_path.listdir(): try: measurements.append(get_measurement(measurement_id, compute_size)) - except: + except Exception as exc: log.err("Failed to get metadata for measurement {0}".format(measurement_id)) + log.exception(exc) if order is None: return measurements diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index c8f191c7..e009cb74 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -370,6 +370,12 @@ class WebUIAPI(object): def api_deck_list(self, request): deck_list = {'decks': []} for deck_id, deck in self.director.deck_store.list(): + nettests = [] + for task in deck.tasks: + if task.type == 'ooni': + assert task.ooni['test_name'] is not None + nettests.append(task.ooni['test_name']) + deck_list['decks'].append({ 'id': deck_id, 'name': deck.name, @@ -378,7 +384,7 @@ class WebUIAPI(object): deck_id, from_schedule=False), 'running_scheduled': self.director.isDeckRunning( deck_id, from_schedule=True), - 'nettests': [], #XXX + 'nettests': nettests, 'description': deck.description, 'schedule': deck.schedule, 'enabled': self.director.deck_store.is_enabled(deck_id) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 6abd04fd..34b9fbba 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -1,7 +1,6 @@ import shutil import string import random -import signal import errno import gzip import os @@ -103,7 +102,7 @@ def randomDate(start, end): LONG_DATE = "%Y-%m-%d %H:%M:%S" SHORT_DATE = "%Y%m%dT%H%M%SZ" -def generate_filename(test_details, prefix=None, extension=None): +def generate_filename(test_details, prefix=None, extension=None, deck_id=None): """ Returns a filename for every test execution. @@ -116,6 +115,9 @@ def generate_filename(test_details, prefix=None, extension=None): kwargs["prefix"] = prefix filename_format += "{prefix}-" filename_format += "{timestamp}-{probe_cc}-{probe_asn}-{test_name}" + if deck_id is not None: + kwargs["deck_id"] = deck_id + filename_format += "-{deck_id}" if extension is not None: kwargs["extension"] = extension filename_format += ".{extension}"

1 0