tor-commits September 2016

tor-commits@lists.torproject.org

20 participants
1187 discussions

[ooni-probe/master] Use is_process_running function in reporter
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 379533a802272ad34bbd1353493b6d9bf8d8449b Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 16 11:55:57 2016 +0200 Use is_process_running function in reporter --- ooni/reporter.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index b149d49..951602f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -17,7 +17,7 @@ from twisted.python.util import untilConcludes from twisted.internet import defer from twisted.internet.error import ConnectionRefusedError -from ooni.utils import log +from ooni.utils import log, is_process_running from ooni.tasks import Measurement try: from scapy.packet import Packet @@ -458,9 +458,7 @@ class OONIBReportLog(object): if entry['completed'] is False: continue if entry['status'] in ('created',): - try: - os.kill(entry['pid'], 0) - except OSError: + if not is_process_running(entry['pid']): incomplete_reports.append( (entry['measurements_path'], entry) ) @@ -476,13 +474,10 @@ class OONIBReportLog(object): all_entries = yield self.get_report_log_entries() for entry in all_entries[:]: if entry['status'] in ('created',): - try: - os.kill(entry['pid'], 0) + if is_process_running(entry['pid']): in_progress_reports.append( (entry['measurements_path'], entry) ) - except OSError: - pass defer.returnValue(in_progress_reports) @defer.inlineCallbacks

1 0

[ooni-probe/master] Update Changelog and version for 2.0.0-rc.2
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 1fd8ad6526f4d49ad8bcd387a0d912efff568582 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 13 12:38:24 2016 +0200 Update Changelog and version for 2.0.0-rc.2 --- ChangeLog.rst | 18 ++++++++++++++++++ ooni/__init__.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 678983c..3c0ecd8 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,24 @@ Changelog ========= +v2.0.0-rc.2 (Tue, 13 Sep 2016) +------------------------------ + +This is a release candidate for a major ooniprobe release. + +It includes a new web user interface and a system daemon for running ooniprobe +tests. + +Feature list: + +* System daemon for running tests periodically (https://github.com/TheTorProject/ooni-probe/issues/576) + +* Web user interface for viewing measurement results (https://github.com/TheTorProject/ooni-probe/issues/575) + +* New deck format (https://github.com/TheTorProject/ooni-probe/issues/571) + +* Local reports are written in JSON (https://github.com/TheTorProject/ooni-probe/issues/557) + v1.6.1 (Tue, 26 Jul 2016) ------------------------- diff --git a/ooni/__init__.py b/ooni/__init__.py index de2157c..20bdef0 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "2.0.0-rc.1" +__version__ = "2.0.0-rc.2" __all__ = [ 'agent',

1 0

[ooni-probe/master] ooniprobe_agent: fix behavior when no command is specified
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 300660ff9873d45d78e54b4a9c09c7c659afe6f5 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon Sep 19 11:02:17 2016 +0200 ooniprobe_agent: fix behavior when no command is specified --- ooni/scripts/ooniprobe_agent.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index 4c97f5e..49694eb 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -12,6 +12,7 @@ from twisted.python import usage from ooni.utils import log, is_process_running from ooni.settings import config from ooni.agent.agent import AgentService +from ooni import __version__ class StartOoniprobeAgentPlugin: @@ -38,6 +39,10 @@ class RunOptions(usage.Options): pass class AgentOptions(usage.Options): + + synopsis = """%s [options] command + """ % (os.path.basename(sys.argv[0]),) + subCommands = [ ['start', None, StartOptions, "Start the ooniprobe-agent in the " "background"], @@ -45,9 +50,17 @@ class AgentOptions(usage.Options): ['status', None, StatusOptions, "Show status of the ooniprobe-agent"], ['run', None, RunOptions, "Run the ooniprobe-agent in the foreground"] ] + def postOptions(self): self.twistd_args = [] + def opt_version(self): + """ + Display the ooniprobe version and exit. + """ + print("ooniprobe-agent version:", __version__) + sys.exit(0) + def start_agent(options=None): config.set_paths() config.initialize_ooni_home() @@ -188,8 +201,9 @@ def run(): options = AgentOptions() options.parseOptions() - if options.subCommand == "run": - options.twistd_args += ("--nodaemon",) + if options.subCommand == None: + print(options) + return if options.subCommand == "stop": return stop_agent() @@ -197,6 +211,9 @@ def run(): if options.subCommand == "status": return status_agent() + if options.subCommand == "run": + options.twistd_args += ("--nodaemon",) + return start_agent(options) if __name__ == "__main__":

1 0

[ooni-probe/master] Bump to release candidate 3
by art＠torproject.org 19 Sep '16

19 Sep '16

commit b6b03acf87c44e798480deefd86ce2bb5926b7c0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 19 13:38:27 2016 +0200 Bump to release candidate 3 --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 20bdef0..41f5c37 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "2.0.0-rc.2" +__version__ = "2.0.0-rc.3" __all__ = [ 'agent',

1 0

[ooni-probe/master] Fix cache updating logic.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 79ed2a50afc489b925bd6eaa78f1342d60877955 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 16 17:52:06 2016 +0200 Fix cache updating logic. * Fix typo in comment --- ooni/deck/store.py | 11 ++++++++--- ooni/ui/web/server.py | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ooni/deck/store.py b/ooni/deck/store.py index bf1eb7f..f30b0d6 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -92,15 +92,17 @@ class InputStore(object): yield self.create(country_code) def _update_cache(self): + new_cache = {} descs = self.path.child("descriptors") if not descs.exists(): - self._cache = {} + self._cache = new_cache return for fn in descs.listdir(): with descs.child(fn).open("r") as in_fh: input_desc = json.load(in_fh) - self._cache[input_desc.pop("id")] = input_desc + new_cache[input_desc.pop("id")] = input_desc + self._cache = new_cache self._cache_stale = False return @@ -172,6 +174,7 @@ class DeckStore(object): deck_enabled_path.remove() def _update_cache(self): + new_cache = {} for deck_path in self.available_directory.listdir(): if not deck_path.endswith('.yaml'): continue @@ -179,7 +182,9 @@ class DeckStore(object): deck = NGDeck( deck_path=self.available_directory.child(deck_path).path ) - self._cache[deck_id] = deck + new_cache[deck_id] = deck + self._cache = new_cache + self._cache_stale = False def get(self, deck_id): if self._cache_stale: diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 48f21c7..cee42ce 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -144,8 +144,8 @@ class LongPoller(object): class WebUIAPI(object): app = Klein() - # Maximum number in seconds after which to return a result even if not - # change happenned. + # Maximum number in seconds after which to return a result even if no + # change happened. _long_polling_timeout = 5 _reactor = reactor _enable_xsrf_protection = True

1 0

[ooni-probe/master] Revert "ui/web/server.py: pass binary, not unicode cookies"
by art＠torproject.org 19 Sep '16

19 Sep '16

commit baee35fe51e70547e6128885ee0a97e1937d95d0 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon Sep 19 11:37:20 2016 +0200 Revert "ui/web/server.py: pass binary, not unicode cookies" This reverts commit 21548e9f0f53100336ef4f22d423f5f728a117e5 since now Travis is failing for all versions of Twisted. --- ooni/ui/web/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 385bf18..eec82d3 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -56,13 +56,13 @@ def xsrf_protect(check=True): @wraps(f) def wrapper(instance, request, *a, **kw): should_check = check and instance._enable_xsrf_protection - token_cookie = request.getCookie(b'XSRF-TOKEN') + token_cookie = request.getCookie(u'XSRF-TOKEN') token_header = request.getHeader(b"X-XSRF-TOKEN") if (token_cookie != instance._xsrf_token and instance._enable_xsrf_protection): - request.addCookie(b'XSRF-TOKEN', + request.addCookie(u'XSRF-TOKEN', instance._xsrf_token, - path=b'/') + path=u'/') if should_check and token_cookie != token_header: raise WebUIError(404, "Invalid XSRF token") return f(instance, request, *a, **kw) @@ -161,7 +161,7 @@ class WebUIAPI(object): # We use a double submit token to protect against XSRF rng = SystemRandom() token_space = string.letters+string.digits - self._xsrf_token = b''.join([rng.choice(token_space) + self._xsrf_token = ''.join([rng.choice(token_space) for _ in range(30)]) self._director_started = False

1 0

[ooni-probe/master] Refactor the pattern for directory creation into a function
by art＠torproject.org 19 Sep '16

19 Sep '16

commit ba8558514aabc963782688ac694daa196c53dc8d Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 16 12:10:20 2016 +0200 Refactor the pattern for directory creation into a function --- ooni/deck/store.py | 14 ++++---------- ooni/resources.py | 16 +++++----------- ooni/scripts/oonideckgen.py | 8 ++------ ooni/utils/__init__.py | 13 +++++++++++++ ooni/utils/log.py | 9 ++------- ooni/utils/onion.py | 8 +++----- 6 files changed, 29 insertions(+), 39 deletions(-) diff --git a/ooni/deck/store.py b/ooni/deck/store.py index bd852c7..bf1eb7f 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -6,6 +6,7 @@ from copy import deepcopy from twisted.internet import defer from twisted.python.filepath import FilePath +from ooni.utils import mkdir_p from ooni.deck.deck import NGDeck from ooni.otime import timestampNowISO8601UTC from ooni.resources import check_for_update @@ -80,16 +81,9 @@ class InputStore(object): self.path = FilePath(config.inputs_directory) self.resources = FilePath(config.resources_directory) - try: - self.path.child("descriptors").makedirs() - except OSError as e: - if not e.errno == errno.EEXIST: - raise - try: - self.path.child("data").makedirs() - except OSError as e: - if not e.errno == errno.EEXIST: - raise + mkdir_p(self.path.child("descriptors").path) + mkdir_p(self.path.child("data").path) + yield self.update_url_lists(country_code) @defer.inlineCallbacks diff --git a/ooni/resources.py b/ooni/resources.py index edb7781..1830d42 100644 --- a/ooni/resources.py +++ b/ooni/resources.py @@ -5,7 +5,7 @@ from twisted.python.filepath import FilePath from twisted.internet import defer from twisted.web.client import downloadPage, getPage, HTTPClientFactory -from ooni.utils import log, gunzip, rename +from ooni.utils import log, gunzip, rename, mkdir_p from ooni.settings import config # Disable logs of HTTPClientFactory @@ -107,11 +107,7 @@ def check_for_update(country_code=None): latest_version = yield get_latest_version() resources_dir = FilePath(config.resources_directory) - try: - resources_dir.makedirs() - except OSError as e: - if not e.errno == errno.EEXIST: - raise + mkdir_p(resources_dir.path) current_manifest = resources_dir.child("manifest.json") if current_manifest.exists(): @@ -153,11 +149,9 @@ def check_for_update(country_code=None): filename = filename[:-3] gzipped = True dst_file = resources_dir.child(pre_path).child(filename) - try: - dst_file.parent().makedirs() - except OSError as e: - if not e.errno == errno.EEXIST: - raise + + mkdir_p(dst_file.parent().path) + src_file = dst_file.temporarySibling() src_file.alwaysCreate = 0 diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py index ba27ffe..89541d7 100644 --- a/ooni/scripts/oonideckgen.py +++ b/ooni/scripts/oonideckgen.py @@ -1,12 +1,12 @@ from __future__ import print_function -import errno import os import sys from twisted.internet import defer, task from twisted.python import usage +from ooni.utils import mkdir_p from ooni.otime import prettyDateNowUTC from ooni import errors from ooni.geoip import probe_ip @@ -120,11 +120,7 @@ def oonideckgen(reactor): options['country-code'] = options['country-code'].lower() - try: - os.makedirs(os.path.dirname(options['output'])) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise + mkdir_p(os.path.dirname(options['output'])) generate_deck(options) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index a894daf..99f1985 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -175,3 +175,16 @@ def is_process_running(pid): else: raise return running + +def mkdir_p(path): + """ + Like makedirs, but it also ignores EEXIST errors, unless it exists but + isn't a directory. + """ + try: + os.makedirs(path) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise + if not os.path.isdir(path): + raise diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 93e6b31..f20fdce 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -7,6 +7,7 @@ import logging from twisted.python import log as tw_log from twisted.python.logfile import DailyLogFile +from ooni.utils import mkdir_p from ooni import otime # Get rid of the annoying "No route found for @@ -140,13 +141,7 @@ class OONILogger(object): log_folder = config.running_path logfile = os.path.join(log_folder, "ooniprobe.log") - try: - os.makedirs(log_folder) - except OSError as ose: - if ose.errno == errno.EEXIST and os.path.isdir(log_folder): - pass - else: - raise + mkdir_p(log_folder) log_filename = os.path.basename(logfile) file_log_level = levels.get(config.basic.loglevel, diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 10f80a3..7414e2f 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -15,6 +15,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from txtorcon import TorConfig, TorState, launch_tor, build_tor_connection from txtorcon.util import find_tor_binary as tx_find_tor_binary +from ooni.utils import mkdir_p from ooni.utils.net import randomFreePort from ooni import constants from ooni import errors @@ -253,12 +254,9 @@ def get_tor_config(): # 2. We have write permissions to it data_dir_usable = is_tor_data_dir_usable(data_dir) try: - os.makedirs(data_dir) - log.debug("%s does not exist. Creating it." % data_dir) + mkdir_p(data_dir) except OSError as ose: - if ose.errno == errno.EEXIST: - pass - elif ose.errno == errno.EACCESS: + if ose.errno == errno.EACCESS: data_dir_usable = False else: raise

1 0

[ooni-probe/master] Various code style fixes suggested by @bassosimone
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 3483c8005ecf8eb0c856f04913f6132f547f2d84 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 16 11:56:38 2016 +0200 Various code style fixes suggested by @bassosimone --- ooni/reporter.py | 6 +++--- ooni/resources.py | 4 ++++ ooni/scripts/ooniprobe.py | 10 ++++++---- ooni/scripts/ooniprobe_agent.py | 12 +++++------- ooni/scripts/ooniresources.py | 3 ++- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 951602f..ab32c54 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -221,7 +221,8 @@ class NJSONReporter(OReporter): OReporter.__init__(self, test_details) def _writeln(self, line): - self._write("%s\n" % line) + self._write(line) + self._write("\n") def _write(self, data): if not self._stream: @@ -249,8 +250,7 @@ class NJSONReporter(OReporter): 'test_keys': e } report_entry.update(self.testDetails) - self._write(json.dumps(report_entry)) - self._write("\n") + self._writeln(json.dumps(report_entry)) def createReport(self): self._stream = open(self.report_path, 'w+') diff --git a/ooni/resources.py b/ooni/resources.py index ac0f967..edb7781 100644 --- a/ooni/resources.py +++ b/ooni/resources.py @@ -91,6 +91,10 @@ def check_for_update(country_code=None): Checks if we need to update the resources. If the country_code is specified then only the resources for that country will be updated/downloaded. + + XXX we currently don't check the shasum of resources although this is + included inside of the manifest. + This should probably be done once we have signing of resources. :return: the latest version. """ temporary_files = [] diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py index d67dd80..c8ffee2 100644 --- a/ooni/scripts/ooniprobe.py +++ b/ooni/scripts/ooniprobe.py @@ -12,10 +12,12 @@ def ooniprobe(reactor): check_incoherences=True) if global_options['queue']: return runWithDaemonDirector(global_options) - elif global_options['initialize']: + + if global_options['initialize']: initializeOoniprobe(global_options) return defer.succeed(None) - elif global_options['web-ui']: + + if global_options['web-ui']: from ooni.scripts.ooniprobe_agent import WEB_UI_URL from ooni.scripts.ooniprobe_agent import status_agent, start_agent if status_agent() != 0: @@ -25,8 +27,8 @@ def ooniprobe(reactor): print("Started ooniprobe-agent") webbrowser.open_new(WEB_UI_URL) return defer.succeed(None) - else: - return runWithDirector(global_options) + + return runWithDirector(global_options) def run(): task.react(ooniprobe) diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index c455f9b..479d230 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -1,6 +1,7 @@ from __future__ import print_function import os +import sys import time import errno import signal @@ -65,8 +66,9 @@ def start_agent(options=None): twistd_args.append("StartOoniprobeAgent") try: twistd_config.parseOptions(twistd_args) - except usage.error, ue: + except usage.error as ue: print("ooniprobe: usage error from twistd: {}\n".format(ue)) + sys.exit(1) twistd_config.loadedPlugins = { "StartOoniprobeAgent": StartOoniprobeAgentPlugin() } @@ -103,12 +105,8 @@ def get_running_pidfile(): pid = open(pidfile, "r").read() pid = int(pid) if is_process_running(pid): - running_pidfile = pidfile - else: - continue - if running_pidfile is None: - raise NotRunning - return running_pidfile + return pidfile + raise NotRunning def status_agent(): try: diff --git a/ooni/scripts/ooniresources.py b/ooni/scripts/ooniresources.py index 5fdbbf0..8e77431 100644 --- a/ooni/scripts/ooniresources.py +++ b/ooni/scripts/ooniresources.py @@ -30,5 +30,6 @@ def run(): print "%s: Try --help for usage details." % (sys.argv[0]) sys.exit(1) - print("WARNING: Usage of this script is deprecated.") + print("WARNING: Usage of this script is deprecated. We will not do " + "anything.") sys.exit(0)

1 0

[ooni-probe/master] Merge pull request #582 from TheTorProject/v2.0.0-alpha
by art＠torproject.org 19 Sep '16

19 Sep '16

1 0

[ooni-probe/master] Skip tests for http_requests integration tests
by art＠torproject.org 19 Sep '16

19 Sep '16

commit a89d9124caaac15dd54af44975e6029821c41f78 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 19 12:43:55 2016 +0200 Skip tests for http_requests integration tests --- ooni/tests/test_oonicli.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 05bc273..896a4b0 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -111,7 +111,7 @@ class TestRunDirector(ConfigTestCase): @defer.inlineCallbacks def test_http_requests(self): - retries = 3 + self.skipTest("XXX This integration test fails non deterministically") def verify_function(entry): assert 'body_length_match' in entry assert 'body_proportion' in entry @@ -120,17 +120,13 @@ class TestRunDirector(ConfigTestCase): assert 'factor' in entry assert 'headers_diff' in entry assert 'headers_match' in entry - while retries > 0: - try: - yield self.run_helper('blocking/http_requests', - ['-u', 'http://torproject.org/'], - verify_function) - break - except: - retries -= 1 + yield self.run_helper('blocking/http_requests', + ['-u', 'http://torproject.org/'], + verify_function) @defer.inlineCallbacks def test_http_requests_with_file(self): + self.skipTest("XXX This integration test fails non deterministically") def verify_function(entry): assert 'body_length_match' in entry assert 'body_proportion' in entry

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits September 2016