September 2017 - tor-commits - lists.torproject.org

[ooni-probe/master] Add support for deck status
by art＠torproject.org 22 Sep '17

22 Sep '17

commit c937439d960ba3cfc77d173615fc029c44e417a7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Jan 11 14:35:03 2017 +0000 Add support for deck status --- ooni/agent/scheduler.py | 2 +- ooni/deck/deck.py | 16 +++++++++++++--- ooni/deck/store.py | 3 +-- ooni/director.py | 27 +++++++++++++++++++++++++++ ooni/measurements.py | 16 +++++++++++----- ooni/ui/cli.py | 2 +- ooni/ui/web/server.py | 38 ++++++++++++++++++-------------------- 7 files changed, 72 insertions(+), 32 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 4bf058ab..8de3b303 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -294,7 +294,7 @@ class RunDeck(ScheduledTask): def task(self): deck = deck_store.get(self.deck_id) yield deck.setup() - yield deck.run(self.director) + yield deck.run(self.director, from_schedule=True) class RefreshDeckList(ScheduledTask): diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index 07f0799e..fbbb8c3d 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -1,6 +1,7 @@ import os import uuid import errno +import hashlib from copy import deepcopy from string import Template @@ -82,6 +83,7 @@ class NGDeck(object): self.name = "" self.description = "" self.icon = "" + self.id = None self.schedule = None self.metadata = {} @@ -102,10 +104,15 @@ class NGDeck(object): def open(self, deck_path, global_options=None): with open(deck_path) as fh: deck_data = yaml.safe_load(fh) + self.id = os.path.basename(deck_path[:-1*len('.yaml')]) self.deck_directory = os.path.abspath(os.path.dirname(deck_path)) self.load(deck_data, global_options) def load(self, deck_data, global_options=None): + if self.id is None: + # This happens when you load a deck not from a filepath so we + # use the first 16 characters of the SHA256 hexdigest as an ID + self.id = hashlib.sha256(deck_data).hexdigest()[:16] if global_options is not None: self.global_options = normalize_options(global_options) @@ -209,7 +216,8 @@ class NGDeck(object): ) generate_summary( measurement_dir.child("measurements.njson").path, - measurement_dir.child("summary.json").path + measurement_dir.child("summary.json").path, + deck_id=self.id ) measurement_dir.child("running.pid").remove() @@ -274,18 +282,20 @@ class NGDeck(object): self._is_setup = True @defer.inlineCallbacks - def run(self, director): + def run(self, director, from_schedule=False): assert self._is_setup, "You must call setup() before you can run a " \ "deck" if self.requires_tor: yield director.start_tor() yield self.query_bouncer() + director.deckStarted(self.id, from_schedule) for task in self._tasks: if task.skip is True: - log.msg("Skipping running {0}".format(task.id)) + log.debug("Skipping running {0}".format(task.id)) continue if task.type == "ooni": yield self._run_ooni_task(task, director) + director.deckFinished(self.id, from_schedule) self._is_setup = False diff --git a/ooni/deck/store.py b/ooni/deck/store.py index 6e3e6de4..204d9357 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -227,11 +227,10 @@ class DeckStore(object): for deck_path in self.available_directory.listdir(): if not deck_path.endswith('.yaml'): continue - deck_id = deck_path[:-1*len('.yaml')] deck = NGDeck( deck_path=self.available_directory.child(deck_path).path ) - new_cache[deck_id] = deck + new_cache[deck.id] = deck self._cache = new_cache self._cache_stale = False diff --git a/ooni/director.py b/ooni/director.py index 1e55bb67..f7566f5f 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -70,6 +70,7 @@ class Director(object): def __init__(self): self.activeNetTests = [] + self.activeDecks = [] self.measurementManager = MeasurementManager() self.measurementManager.director = self @@ -290,6 +291,32 @@ class Director(object): measurement.result = failure return measurement + def deckStarted(self, deck_id, from_schedule): + log.debug("Starting {0} ({1})".format(deck_id, + 'scheduled' if from_schedule + else 'user-run')) + self.activeDecks.append((deck_id, from_schedule)) + + def deckFinished(self, deck_id, from_schedule): + log.debug("Finished {0} ({1})".format(deck_id, + 'scheduled' if from_schedule + else 'user-run')) + try: + self.activeDecks.remove((deck_id, from_schedule)) + except ValueError: + log.error("Completed deck {0} is not actually running".format( + deck_id)) + + + def isDeckRunning(self, deck_id, from_schedule): + """ + :param deck_id: the ID of the deck to check if it's running + :param from_schedule: True if we want to know the status of a + scheduled deck run False for user initiated runs. + :return: True if the deck is running False otherwise + """ + return (deck_id, from_schedule) in self.activeDecks + def netTestDone(self, net_test): self.notify(DirectorEvent("success", "Successfully ran test {0}".format( diff --git a/ooni/measurements.py b/ooni/measurements.py index b3b3e0fc..9197ee75 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -11,7 +11,7 @@ from ooni.settings import config class MeasurementInProgress(Exception): pass -class Process(): +class MeasurementTypes(): supported_tests = [ "web_connectivity", "http_requests", @@ -51,14 +51,15 @@ class Process(): result['url'] = entry['input'] return result -def generate_summary(input_file, output_file): + +def generate_summary(input_file, output_file, deck_id='none'): results = {} with open(input_file) as in_file: for idx, line in enumerate(in_file): entry = json.loads(line.strip()) result = {} - if entry['test_name'] in Process.supported_tests: - result = getattr(Process, entry['test_name'])(entry) + if entry['test_name'] in MeasurementTypes.supported_tests: + result = getattr(MeasurementTypes, entry['test_name'])(entry) result['idx'] = idx if not result.get('url', None): result['url'] = entry['input'] @@ -66,6 +67,7 @@ def generate_summary(input_file, output_file): results['test_start_time'] = entry['test_start_time'] results['country_code'] = entry['probe_cc'] results['asn'] = entry['probe_asn'] + results['deck_id'] = deck_id results['results'] = results.get('results', []) results['results'].append(result) @@ -73,9 +75,11 @@ def generate_summary(input_file, output_file): json.dump(results, fw) return results + class MeasurementNotFound(Exception): pass + def get_measurement(measurement_id, compute_size=False): size = -1 measurement_path = FilePath(config.measurements_directory) @@ -114,7 +118,9 @@ def get_measurement(measurement_id, compute_size=False): "keep": keep, "running": running, "stale": stale, - "size": size + "size": size, + # XXX we need the deck ID in here + "deck_id": "none" } diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 162354c1..cf889a3c 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -377,7 +377,7 @@ def runTestWithDirector(director, global_options, url=None, def post_director_start(_): try: yield deck.setup() - yield deck.run(director) + yield deck.run(director, from_schedule=False) except errors.UnableToLoadDeckInput as error: raise defer.failure.Failure(error) except errors.NoReachableTestHelpers as error: diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index efcad146..c8f191c7 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -147,7 +147,7 @@ class WebUIAPI(object): app = Klein() # Maximum number in seconds after which to return a result even if no # change happened. - _long_polling_timeout = 5 + _long_polling_timeout = 30 _reactor = reactor _enable_xsrf_protection = True @@ -171,8 +171,8 @@ class WebUIAPI(object): # We use exponential backoff to trigger retries of the startup of # the director. self._director_startup_retries = 0 - # Maximum delay should be 6 hours - self._director_max_retry_delay = 6*60*60 + # Maximum delay should be 30 minutes + self._director_max_retry_delay = 30*60 self.status_poller = LongPoller( self._long_polling_timeout, _reactor) @@ -368,26 +368,21 @@ class WebUIAPI(object): @xsrf_protect(check=False) @requires_true(attrs=['_director_started', '_is_initialized']) def api_deck_list(self, request): - deck_list = { - 'available': {}, - 'enabled': {} - } + deck_list = {'decks': []} for deck_id, deck in self.director.deck_store.list(): - deck_list['available'][deck_id] = { + deck_list['decks'].append({ + 'id': deck_id, 'name': deck.name, + 'icon': deck.icon, + 'running': self.director.isDeckRunning( + deck_id, from_schedule=False), + 'running_scheduled': self.director.isDeckRunning( + deck_id, from_schedule=True), + 'nettests': [], #XXX 'description': deck.description, 'schedule': deck.schedule, 'enabled': self.director.deck_store.is_enabled(deck_id) - } - - for deck_id, deck in self.director.deck_store.list_enabled(): - deck_list['enabled'][deck_id] = { - 'name': deck.name, - 'description': deck.description, - 'schedule': deck.schedule, - 'enabled': True - } - + }) return self.render_json(deck_list, request) @app.route('/api/deck/<string:deck_id>/run', methods=["POST"]) @@ -433,9 +428,12 @@ class WebUIAPI(object): # These are dangling deferreds try: yield deck.setup() - yield deck.run(self.director) + yield deck.run(self.director, from_schedule=False) + self.director_event_poller.notify(DirectorEvent("success", + "Started Deck " + + deck.id)) except: - self.director_event_poller.notify(DirectorEvent("error", + self.director_event_poller.notify(DirectorEvent("error", "Failed to start deck")) @app.route('/api/nettest/<string:test_name>/start', methods=["POST"])

1 0

[ooni-probe/master] Tweak README.md wording (#726)
by art＠torproject.org 22 Sep '17

22 Sep '17

commit ede2bc6eec8839329003b1b5eb70d62b1d5ebc92 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon Jan 30 20:48:41 2017 +0100 Tweak README.md wording (#726) * Tweak README.md wording * Use macOS and change brew link * wrap url in angular brackets * Clarify what happens when you run commands While there, also explain how to run ooniprobe-agent, which is part of what you want to know as a tester / developer. * Tweak previous * Tweak previous again --- README.rst | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 3de9489b..d2ecac39 100644 --- a/README.rst +++ b/README.rst @@ -86,15 +86,15 @@ Note: You'll need to swap out ``$RELEASE`` for either ``yakkety``, ``xenial`` or ``trusty``. This will not happen automatically. You will also need to ensure that you have the ``universe`` repository enabled. The ``universe`` repository is enabled by default in a standard Ubuntu installation but may not be on some -minimal on not standard installations. +minimal, or not standard, installations. Installation ============ -Mac OS X --------- +macOS +----- -You can install ooniprobe on OSX if you have installed homebrew (http://mxcl.github.io/homebrew) with:: +You can install ooniprobe on macOS if you have installed homebrew (http://brew.sh/) with:: brew install ooniprobe @@ -147,7 +147,7 @@ Configuring ooniprobe --------------------- After successfully installing ooniprobe you should be able to access the web UI -on your host machine at http://localhost:8842/ . +on your host machine at <http://localhost:8842/>. You should now be presented with the web UI setup wizard where you can read the risks involved with running ooniprobe. Upon answering the quiz correctly you can @@ -247,8 +247,17 @@ to be activated:: pip install -r requirements.txt pip install -r requirements-dev.txt python setup.py install - ooniprobe -s # if all went well, lists available tests +Then, you can check whether the installation went well with:: + + ooniprobe -s + +This will explain you the risks of running ooniprobe and make sure you have +understood them, afterwards it shows you the available tests. + +To run the ooniprobe agent, instead, type:: + + ooniprobe-agent run Donate -------

1 0

[ooni-probe/master] [bugfix] Fix displaying of nettest options when --help is called (#722)
by art＠torproject.org 22 Sep '17

22 Sep '17

commit e2c3f48cf3a768c6193b4c2e2278883f3ecbaa24 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jan 27 11:00:18 2017 +0000 [bugfix] Fix displaying of nettest options when --help is called (#722) [hotfix] Fix displaying of nettest options when --help is called --- ooni/nettest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ooni/nettest.py b/ooni/nettest.py index edfd9cd7..edba448c 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -146,6 +146,10 @@ def usageOptionsFactory(test_name, test_version): test_name ) + def opt_help(self): + map(log.msg, self.__str__().split("\n")) + sys.exit(0) + def opt_version(self): """ Display the net_test version and exit.

1 0

[ooni-probe/master] Merge pull request #713 from TheTorProject/feature/deck-status
by art＠torproject.org 22 Sep '17

22 Sep '17

commit 73d43153e9bb99c2c64946cacef1c22c28d31bf0 Merge: f14488ae 83303ffd Author: Simone Basso <bassosimone(a)gmail.com> Date: Wed Jan 25 20:55:45 2017 +0100 Merge pull request #713 from TheTorProject/feature/deck-status Feature/deck status ooni/agent/scheduler.py | 2 +- ooni/deck/deck.py | 29 +++++++++++++++++++-------- ooni/deck/legacy.py | 12 +++++++++++ ooni/deck/store.py | 3 +-- ooni/director.py | 27 +++++++++++++++++++++++++ ooni/measurements.py | 38 +++++++++++++++++++++++------------ ooni/tests/test_deck.py | 6 ++++-- ooni/tests/test_scheduler.py | 2 +- ooni/ui/cli.py | 2 +- ooni/ui/web/server.py | 47 ++++++++++++++++++++++++-------------------- ooni/utils/__init__.py | 6 ++++-- 11 files changed, 123 insertions(+), 51 deletions(-)

1 0

[ooni-probe/master] Fix bug that leads to "Failed to get metadata for measurement"
by art＠torproject.org 22 Sep '17

22 Sep '17

commit f7b8225f006162d358aee7f58803f0bc6c7d3b26 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jan 12 13:28:35 2017 +0000 Fix bug that leads to "Failed to get metadata for measurement" This happens when the test process dies before it's able to write the running.pid file. --- ooni/deck/deck.py | 7 +++---- ooni/measurements.py | 13 ++++++++----- ooni/ui/web/server.py | 3 ++- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index c57ccb63..7e4faa4e 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -1,4 +1,5 @@ import os +import json import uuid import errno import hashlib @@ -36,14 +37,12 @@ def options_to_args(options): for k, v in options.items(): if v is None: continue - if v == False or v == 0: + if isinstance(v, bool): continue if (len(k)) == 1: args.append('-'+k) else: args.append('--'+k) - if isinstance(v, bool) or isinstance(v, int): - continue args.append(v) return args @@ -112,7 +111,7 @@ class NGDeck(object): if self.id is None: # This happens when you load a deck not from a filepath so we # use the first 16 characters of the SHA256 hexdigest as an ID - self.id = hashlib.sha256(deck_data).hexdigest()[:16] + self.id = hashlib.sha256(json.dumps(deck_data)).hexdigest()[:16] if global_options is not None: self.global_options = normalize_options(global_options) diff --git a/ooni/measurements.py b/ooni/measurements.py index 68b5c555..2a4440ad 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -93,11 +93,14 @@ def get_measurement(measurement_id, compute_size=False): stale = False if measurement.child("measurements.njson.progress").exists(): completed = False - pid = measurement.child("running.pid").open("r").read() - pid = int(pid) - if is_process_running(pid): - running = True - else: + try: + pid = measurement.child("running.pid").open("r").read() + pid = int(pid) + if is_process_running(pid): + running = True + else: + stale = True + except IOError: stale = True if measurement.child("keep").exists(): diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index e009cb74..a79c1e19 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -481,7 +481,8 @@ class WebUIAPI(object): raise WebUIError( 400, 'Insufficient privileges' ) - except: + except Exception as exc: + log.exception(exc) raise WebUIError( 500, 'Failed to start nettest' )

1 0

[ooni-probe/master] Edited the test descriptions for OONI's web UI
by art＠torproject.org 22 Sep '17

22 Sep '17

commit e498bbbeb750b545fa1be3771f8212d6f2a984a4 Author: Maria <maria.x(a)espiv.net> Date: Wed Jan 25 18:07:16 2017 +0100 Edited the test descriptions for OONI's web UI --- ooni/nettests/blocking/bridge_reachability.py | 4 ++-- ooni/nettests/blocking/facebook_messenger.py | 4 ++-- ooni/nettests/blocking/meek_fronted_requests.py | 4 ++-- ooni/nettests/blocking/vanilla_tor.py | 2 +- ooni/nettests/blocking/whatsapp.py | 4 ++-- ooni/nettests/manipulation/http_invalid_request_line.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ooni/nettests/blocking/bridge_reachability.py b/ooni/nettests/blocking/bridge_reachability.py index 2efbc422..e5cc512e 100644 --- a/ooni/nettests/blocking/bridge_reachability.py +++ b/ooni/nettests/blocking/bridge_reachability.py @@ -26,8 +26,8 @@ class UsageOptions(usage.Options): class BridgeReachability(nettest.NetTestCase): name = "Bridge Reachability" - description = "A test for checking if bridges are reachable " \ - "from a given location." + description = "This test examines the reachability of Tor bridges " \ + "in your network." author = "Arturo Filastò" version = "0.1.2" diff --git a/ooni/nettests/blocking/facebook_messenger.py b/ooni/nettests/blocking/facebook_messenger.py index 3f20683f..651fca0d 100644 --- a/ooni/nettests/blocking/facebook_messenger.py +++ b/ooni/nettests/blocking/facebook_messenger.py @@ -43,8 +43,8 @@ def is_facebook_ip(ip_address): class FacebookMessengerTest(httpt.HTTPTest, dnst.DNSTest): name = "Facebook Messenger" - description = ("This test checks to see if the servers used by Facebook " - "messenger are reachable") + description = ("This test examines the reachability of Facebook " + "Messenger in your network.") author = "Arturo Filastò" version = "0.4.0" diff --git a/ooni/nettests/blocking/meek_fronted_requests.py b/ooni/nettests/blocking/meek_fronted_requests.py index a0088f58..5918cd4b 100644 --- a/ooni/nettests/blocking/meek_fronted_requests.py +++ b/ooni/nettests/blocking/meek_fronted_requests.py @@ -28,8 +28,8 @@ class meekTest(httpt.HTTPTest): """ name = "Meek fronted requests test" - description = "This tests for the Meek Tor pluggable transport "\ - "frontend reachability." + description = "This test examines whether the domains used by Meek "\ + "(a type of Tor bridge) work in your network." version = "0.0.1" usageOptions = UsageOptions diff --git a/ooni/nettests/blocking/vanilla_tor.py b/ooni/nettests/blocking/vanilla_tor.py index 7ed050fb..717fa733 100644 --- a/ooni/nettests/blocking/vanilla_tor.py +++ b/ooni/nettests/blocking/vanilla_tor.py @@ -25,7 +25,7 @@ class UsageOptions(usage.Options): class VanillaTor(nettest.NetTestCase): name = "Vanilla Tor" - description = "A test for checking if vanilla Tor connections work." + description = "This test examines the reachability of the Tor network." author = "Arturo Filastò" version = "0.1.0" diff --git a/ooni/nettests/blocking/whatsapp.py b/ooni/nettests/blocking/whatsapp.py index 74480e42..6957e620 100644 --- a/ooni/nettests/blocking/whatsapp.py +++ b/ooni/nettests/blocking/whatsapp.py @@ -261,8 +261,8 @@ class UsageOptions(usage.Options): class WhatsappTest(httpt.HTTPTest, dnst.DNSTest): name = "Whatsapp" - description = ("This test checks to see if the servers used by whatsapp " - "messenger are reachable") + description = ("This test examines the reachability of WhatsApp " + " and WhatsApp's web interface (web.whatsapp.com) in your network.") author = "Arturo Filastò" version = "0.5.0" diff --git a/ooni/nettests/manipulation/http_invalid_request_line.py b/ooni/nettests/manipulation/http_invalid_request_line.py index be0497c2..8e204d92 100644 --- a/ooni/nettests/manipulation/http_invalid_request_line.py +++ b/ooni/nettests/manipulation/http_invalid_request_line.py @@ -25,7 +25,7 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): ascii letters or numbers ('XxXx' will be 4). """ name = "HTTP Invalid Request Line" - description = "Performs out of spec HTTP requests in the attempt to "\ + description = "Performs out of spec HTTP requests in an attempt to "\ "trigger a proxy error message." version = "0.2" authors = "Arturo Filastò"

1 0

[ooni-probe/master] Improve test_deck unittest.
by art＠torproject.org 22 Sep '17

22 Sep '17

commit f9101d96b2d8b8295e79eaeb11dd14eb901d2915 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jan 13 16:16:28 2017 +0000 Improve test_deck unittest. * Fix bug in creating options based on deck options --- ooni/deck/deck.py | 4 +++- ooni/deck/legacy.py | 5 ++++- ooni/tests/test_deck.py | 6 ++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index 7e4faa4e..cb52bb22 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -37,12 +37,14 @@ def options_to_args(options): for k, v in options.items(): if v is None: continue - if isinstance(v, bool): + if v == False: continue if (len(k)) == 1: args.append('-'+k) else: args.append('--'+k) + if v == True: + continue args.append(v) return args diff --git a/ooni/deck/legacy.py b/ooni/deck/legacy.py index 83e16a3a..4e9702e3 100644 --- a/ooni/deck/legacy.py +++ b/ooni/deck/legacy.py @@ -30,7 +30,10 @@ boolean_options = [ 'no-collector', 'no-geoip', 'no-yamloo', - 'verbose' + 'verbose', + 'help', + 'no-default-reporter', + 'resume' ] def convert_legacy_deck(deck_data): diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index b2d05a65..1b8a3087 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -336,5 +336,7 @@ class TestNGDeck(ConfigTestCase): '/path/to/citizenlab-urls-global.txt') def test_options_to_args(self): - args = options_to_args({"f": "foobar.txt", "bar": None, "help": 0}) - print(args) + args = options_to_args({"f": "foobar.txt", "bar": None, + "disabled-flag": False, "enabled-flag": True}) + self.assertEqual(set(args), set(['-f', 'foobar.txt', + '--enabled-flag']))

1 0

[ooni-probe/master] Fix scheduler unittest to check for the from_schedule keyword arg
by art＠torproject.org 22 Sep '17

22 Sep '17

commit 83303ffd18127b5ec53fed903cad1ed4b48835a7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jan 13 19:36:07 2017 +0000 Fix scheduler unittest to check for the from_schedule keyword arg --- ooni/tests/test_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/tests/test_scheduler.py b/ooni/tests/test_scheduler.py index c0ab5ebd..56a4bb40 100644 --- a/ooni/tests/test_scheduler.py +++ b/ooni/tests/test_scheduler.py @@ -312,7 +312,7 @@ class TestSchedulerService(ConfigTestCase): # We check that the run method of the deck was called twice self.mock_deck.run.assert_has_calls([ - mock.call(mock_director), mock.call(mock_director) + mock.call(mock_director, from_schedule=True), mock.call(mock_director, from_schedule=True) ]) d.callback(None)

1 0

[ooni-probe/master] Fix legacy deck converter
by art＠torproject.org 22 Sep '17

22 Sep '17

commit 7a3913cbb5cf0d2f86f19ab848dcaf4189c6ec61 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jan 13 15:58:47 2017 +0000 Fix legacy deck converter --- ooni/deck/legacy.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ooni/deck/legacy.py b/ooni/deck/legacy.py index cf0a30ba..83e16a3a 100644 --- a/ooni/deck/legacy.py +++ b/ooni/deck/legacy.py @@ -26,6 +26,13 @@ def subargs_to_options(subargs): return options +boolean_options = [ + 'no-collector', + 'no-geoip', + 'no-yamloo', + 'verbose' +] + def convert_legacy_deck(deck_data): """ I take a legacy deck list and convert it to the new deck format. @@ -58,6 +65,8 @@ def convert_legacy_deck(deck_data): deck_task["ooni"][name] = value for name, value in options.items(): + if name in boolean_options: + value = False if value == 0 else True deck_task["ooni"][name] = value new_deck_data["tasks"].append(deck_task)

1 0

[ooni-probe/master] support unix-socket tor control ports #479 (#715)
by art＠torproject.org 22 Sep '17

22 Sep '17

commit 19a301c6a1b9bd66b237c3c5a231f75b1f2bded7 Author: E.Iosifidis <iosifidise(a)gmail.com> Date: Thu Jan 19 16:49:26 2017 +0200 support unix-socket tor control ports #479 (#715) * Added a space when printing the -g opt * support unix-socket tor control ports --- data/ooniprobe.conf.sample | 3 +++ ooni/settings.py | 27 ++++++++++++++++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index e324c8a3..8a663824 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -67,6 +67,9 @@ advanced: tor: #socks_port: 8801 #control_port: 8802 + # Alternatively set control_port to the unix domain socket. Useful for Debian based systems + #control_port: unix:/var/run/tor/control + # # Specify the absolute path to the Tor bridges to use for testing #bridges: bridges.list # Specify path of the tor datadirectory. diff --git a/ooni/settings.py b/ooni/settings.py index eb4f052d..a59664cd 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -85,6 +85,9 @@ advanced: tor: #socks_port: 8801 #control_port: 8802 + # Alternatively set control_port to the unix domain socket. Useful for Debian based systems + #control_port: unix:/var/run/tor/control + # # Specify the absolute path to the Tor bridges to use for testing #bridges: bridges.list # Specify path of the tor datadirectory. @@ -520,13 +523,23 @@ class OConfig(object): incoherent.append('tor:socks_port') if self.tor.control_port is not None: - control_port_ep = TCP4ClientEndpoint(reactor, - "localhost", - self.tor.control_port) - try: - yield connectProtocol(control_port_ep, ConnectAndCloseProtocol()) - except Exception: - incoherent.append('tor:control_port') + if isinstance(self.tor.control_port, int): + control_port_ep = TCP4ClientEndpoint(reactor, + "localhost", + self.tor.control_port) + try: + yield connectProtocol(control_port_ep, ConnectAndCloseProtocol()) + except Exception: + incoherent.append('tor:control_port') + else: + conf_unix_socket_path = self.tor.control_port + if conf_unix_socket_path.lstrip.startswith("unix:"): + if os.path.exists(conf_unix_socket_path.lstrip("unix:")): + unix_socket_path = conf_unix_socket_path.lstrip("unix:") + else: + incoherent.append('tor:control_port') + else: + incoherent.append('tor:control_port') self.log_incoherences(incoherent)

1 0