September 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Use is_process_running function in reporter
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 379533a802272ad34bbd1353493b6d9bf8d8449b Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 16 11:55:57 2016 +0200 Use is_process_running function in reporter --- ooni/reporter.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index b149d49..951602f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -17,7 +17,7 @@ from twisted.python.util import untilConcludes from twisted.internet import … [View More]

1 0

[ooni-probe/master] Update Changelog and version for 2.0.0-rc.2
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 1fd8ad6526f4d49ad8bcd387a0d912efff568582 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 13 12:38:24 2016 +0200 Update Changelog and version for 2.0.0-rc.2 --- ChangeLog.rst | 18 ++++++++++++++++++ ooni/__init__.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 678983c..3c0ecd8 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,24 @@ Changelog ========= +v2.0.0-rc.2 (Tue, 13 Sep 2016) +--… [View More]

1 0

[ooni-probe/master] ooniprobe_agent: fix behavior when no command is specified
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 300660ff9873d45d78e54b4a9c09c7c659afe6f5 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon Sep 19 11:02:17 2016 +0200 ooniprobe_agent: fix behavior when no command is specified --- ooni/scripts/ooniprobe_agent.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index 4c97f5e..49694eb 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/… [View More]

1 0

[ooni-probe/master] Bump to release candidate 3
by art＠torproject.org 19 Sep '16

19 Sep '16

commit b6b03acf87c44e798480deefd86ce2bb5926b7c0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 19 13:38:27 2016 +0200 Bump to release candidate 3 --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 20bdef0..41f5c37 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "2.0.0-… [View More]

1 0

[ooni-probe/master] Fix cache updating logic.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 79ed2a50afc489b925bd6eaa78f1342d60877955 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 16 17:52:06 2016 +0200 Fix cache updating logic. * Fix typo in comment --- ooni/deck/store.py | 11 ++++++++--- ooni/ui/web/server.py | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ooni/deck/store.py b/ooni/deck/store.py index bf1eb7f..f30b0d6 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -92,15 +92,17 @@ class InputStore(… [View More]

1 0

[ooni-probe/master] Revert "ui/web/server.py: pass binary, not unicode cookies"
by art＠torproject.org 19 Sep '16

19 Sep '16

commit baee35fe51e70547e6128885ee0a97e1937d95d0 Author: Simone Basso <bassosimone(a)gmail.com> Date: Mon Sep 19 11:37:20 2016 +0200 Revert "ui/web/server.py: pass binary, not unicode cookies" This reverts commit 21548e9f0f53100336ef4f22d423f5f728a117e5 since now Travis is failing for all versions of Twisted. --- ooni/ui/web/server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 385bf18..… [View More]

1 0

[ooni-probe/master] Refactor the pattern for directory creation into a function
by art＠torproject.org 19 Sep '16

19 Sep '16

commit ba8558514aabc963782688ac694daa196c53dc8d Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 16 12:10:20 2016 +0200 Refactor the pattern for directory creation into a function --- ooni/deck/store.py | 14 ++++---------- ooni/resources.py | 16 +++++----------- ooni/scripts/oonideckgen.py | 8 ++------ ooni/utils/__init__.py | 13 +++++++++++++ ooni/utils/log.py | 9 ++------- ooni/utils/onion.py | 8 +++----- 6 files … [View More]changed, 29 insertions(+), 39 deletions(-) diff --git a/ooni/deck/store.py b/ooni/deck/store.py index bd852c7..bf1eb7f 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -6,6 +6,7 @@ from copy import deepcopy from twisted.internet import defer from twisted.python.filepath import FilePath +from ooni.utils import mkdir_p from ooni.deck.deck import NGDeck from ooni.otime import timestampNowISO8601UTC from ooni.resources import check_for_update @@ -80,16 +81,9 @@ class InputStore(object): self.path = FilePath(config.inputs_directory) self.resources = FilePath(config.resources_directory) - try: - self.path.child("descriptors").makedirs() - except OSError as e: - if not e.errno == errno.EEXIST: - raise - try: - self.path.child("data").makedirs() - except OSError as e: - if not e.errno == errno.EEXIST: - raise + mkdir_p(self.path.child("descriptors").path) + mkdir_p(self.path.child("data").path) + yield self.update_url_lists(country_code) @defer.inlineCallbacks diff --git a/ooni/resources.py b/ooni/resources.py index edb7781..1830d42 100644 --- a/ooni/resources.py +++ b/ooni/resources.py @@ -5,7 +5,7 @@ from twisted.python.filepath import FilePath from twisted.internet import defer from twisted.web.client import downloadPage, getPage, HTTPClientFactory -from ooni.utils import log, gunzip, rename +from ooni.utils import log, gunzip, rename, mkdir_p from ooni.settings import config # Disable logs of HTTPClientFactory @@ -107,11 +107,7 @@ def check_for_update(country_code=None): latest_version = yield get_latest_version() resources_dir = FilePath(config.resources_directory) - try: - resources_dir.makedirs() - except OSError as e: - if not e.errno == errno.EEXIST: - raise + mkdir_p(resources_dir.path) current_manifest = resources_dir.child("manifest.json") if current_manifest.exists(): @@ -153,11 +149,9 @@ def check_for_update(country_code=None): filename = filename[:-3] gzipped = True dst_file = resources_dir.child(pre_path).child(filename) - try: - dst_file.parent().makedirs() - except OSError as e: - if not e.errno == errno.EEXIST: - raise + + mkdir_p(dst_file.parent().path) + src_file = dst_file.temporarySibling() src_file.alwaysCreate = 0 diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py index ba27ffe..89541d7 100644 --- a/ooni/scripts/oonideckgen.py +++ b/ooni/scripts/oonideckgen.py @@ -1,12 +1,12 @@ from __future__ import print_function -import errno import os import sys from twisted.internet import defer, task from twisted.python import usage +from ooni.utils import mkdir_p from ooni.otime import prettyDateNowUTC from ooni import errors from ooni.geoip import probe_ip @@ -120,11 +120,7 @@ def oonideckgen(reactor): options['country-code'] = options['country-code'].lower() - try: - os.makedirs(os.path.dirname(options['output'])) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise + mkdir_p(os.path.dirname(options['output'])) generate_deck(options) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index a894daf..99f1985 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -175,3 +175,16 @@ def is_process_running(pid): else: raise return running + +def mkdir_p(path): + """ + Like makedirs, but it also ignores EEXIST errors, unless it exists but + isn't a directory. + """ + try: + os.makedirs(path) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise + if not os.path.isdir(path): + raise diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 93e6b31..f20fdce 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -7,6 +7,7 @@ import logging from twisted.python import log as tw_log from twisted.python.logfile import DailyLogFile +from ooni.utils import mkdir_p from ooni import otime # Get rid of the annoying "No route found for @@ -140,13 +141,7 @@ class OONILogger(object): log_folder = config.running_path logfile = os.path.join(log_folder, "ooniprobe.log") - try: - os.makedirs(log_folder) - except OSError as ose: - if ose.errno == errno.EEXIST and os.path.isdir(log_folder): - pass - else: - raise + mkdir_p(log_folder) log_filename = os.path.basename(logfile) file_log_level = levels.get(config.basic.loglevel, diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index 10f80a3..7414e2f 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -15,6 +15,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from txtorcon import TorConfig, TorState, launch_tor, build_tor_connection from txtorcon.util import find_tor_binary as tx_find_tor_binary +from ooni.utils import mkdir_p from ooni.utils.net import randomFreePort from ooni import constants from ooni import errors @@ -253,12 +254,9 @@ def get_tor_config(): # 2. We have write permissions to it data_dir_usable = is_tor_data_dir_usable(data_dir) try: - os.makedirs(data_dir) - log.debug("%s does not exist. Creating it." % data_dir) + mkdir_p(data_dir) except OSError as ose: - if ose.errno == errno.EEXIST: - pass - elif ose.errno == errno.EACCESS: + if ose.errno == errno.EACCESS: data_dir_usable = False else: raise [View Less]

1 0

[ooni-probe/master] Various code style fixes suggested by @bassosimone
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 3483c8005ecf8eb0c856f04913f6132f547f2d84 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 16 11:56:38 2016 +0200 Various code style fixes suggested by @bassosimone --- ooni/reporter.py | 6 +++--- ooni/resources.py | 4 ++++ ooni/scripts/ooniprobe.py | 10 ++++++---- ooni/scripts/ooniprobe_agent.py | 12 +++++------- ooni/scripts/ooniresources.py | 3 ++- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/ooni/… [View More]reporter.py b/ooni/reporter.py index 951602f..ab32c54 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -221,7 +221,8 @@ class NJSONReporter(OReporter): OReporter.__init__(self, test_details) def _writeln(self, line): - self._write("%s\n" % line) + self._write(line) + self._write("\n") def _write(self, data): if not self._stream: @@ -249,8 +250,7 @@ class NJSONReporter(OReporter): 'test_keys': e } report_entry.update(self.testDetails) - self._write(json.dumps(report_entry)) - self._write("\n") + self._writeln(json.dumps(report_entry)) def createReport(self): self._stream = open(self.report_path, 'w+') diff --git a/ooni/resources.py b/ooni/resources.py index ac0f967..edb7781 100644 --- a/ooni/resources.py +++ b/ooni/resources.py @@ -91,6 +91,10 @@ def check_for_update(country_code=None): Checks if we need to update the resources. If the country_code is specified then only the resources for that country will be updated/downloaded. + + XXX we currently don't check the shasum of resources although this is + included inside of the manifest. + This should probably be done once we have signing of resources. :return: the latest version. """ temporary_files = [] diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py index d67dd80..c8ffee2 100644 --- a/ooni/scripts/ooniprobe.py +++ b/ooni/scripts/ooniprobe.py @@ -12,10 +12,12 @@ def ooniprobe(reactor): check_incoherences=True) if global_options['queue']: return runWithDaemonDirector(global_options) - elif global_options['initialize']: + + if global_options['initialize']: initializeOoniprobe(global_options) return defer.succeed(None) - elif global_options['web-ui']: + + if global_options['web-ui']: from ooni.scripts.ooniprobe_agent import WEB_UI_URL from ooni.scripts.ooniprobe_agent import status_agent, start_agent if status_agent() != 0: @@ -25,8 +27,8 @@ def ooniprobe(reactor): print("Started ooniprobe-agent") webbrowser.open_new(WEB_UI_URL) return defer.succeed(None) - else: - return runWithDirector(global_options) + + return runWithDirector(global_options) def run(): task.react(ooniprobe) diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index c455f9b..479d230 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -1,6 +1,7 @@ from __future__ import print_function import os +import sys import time import errno import signal @@ -65,8 +66,9 @@ def start_agent(options=None): twistd_args.append("StartOoniprobeAgent") try: twistd_config.parseOptions(twistd_args) - except usage.error, ue: + except usage.error as ue: print("ooniprobe: usage error from twistd: {}\n".format(ue)) + sys.exit(1) twistd_config.loadedPlugins = { "StartOoniprobeAgent": StartOoniprobeAgentPlugin() } @@ -103,12 +105,8 @@ def get_running_pidfile(): pid = open(pidfile, "r").read() pid = int(pid) if is_process_running(pid): - running_pidfile = pidfile - else: - continue - if running_pidfile is None: - raise NotRunning - return running_pidfile + return pidfile + raise NotRunning def status_agent(): try: diff --git a/ooni/scripts/ooniresources.py b/ooni/scripts/ooniresources.py index 5fdbbf0..8e77431 100644 --- a/ooni/scripts/ooniresources.py +++ b/ooni/scripts/ooniresources.py @@ -30,5 +30,6 @@ def run(): print "%s: Try --help for usage details." % (sys.argv[0]) sys.exit(1) - print("WARNING: Usage of this script is deprecated.") + print("WARNING: Usage of this script is deprecated. We will not do " + "anything.") sys.exit(0) [View Less]

1 0

[ooni-probe/master] Merge pull request #582 from TheTorProject/v2.0.0-alpha
by art＠torproject.org 19 Sep '16

19 Sep '16

1 0

[ooni-probe/master] Skip tests for http_requests integration tests
by art＠torproject.org 19 Sep '16

19 Sep '16

commit a89d9124caaac15dd54af44975e6029821c41f78 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 19 12:43:55 2016 +0200 Skip tests for http_requests integration tests --- ooni/tests/test_oonicli.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index 05bc273..896a4b0 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -111,7 +111,7 @@ class TestRunDirector(… [View More]

1 0