September 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Fix typo in data deletion schedule
by art＠torproject.org 19 Sep '16

19 Sep '16

commit b41bc4b5c6e9616fd4a0ea50a770d862984379e2 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 5 15:54:40 2016 +0200 Fix typo in data deletion schedule --- ooni/agent/scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 7c2212b..3cbd29d 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -182,7 +182,8 @@ class CheckMeasurementQuota(ScheduledTask): if (float(available_bytes) / float(maximum_bytes)) >= self._warn_when: log.warn("You are about to reach the maximum allowed quota. Be careful") with open(warning_path, "w") as out_file: - out_file.write("{0} {1}".split(available_bytes, maximum_bytes)) + out_file.write("{0} {1}".format(available_bytes, + maximum_bytes)) else: try: os.remove(warning_path)

1 0

[ooni-probe/master] Fix test_runtime key value
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 23669a79bfb6b402a0332b68eb8383bc13251aea Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 5 16:29:29 2016 +0200 Fix test_runtime key value * This closes: https://github.com/TheTorProject/ooni-probe/issues/598 --- ooni/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/tasks.py b/ooni/tasks.py index 925089b..3a7c949 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -110,8 +110,6 @@ class Measurement(TaskWithTimeout): """ self.testInstance = test_instance self.testInstance.input = test_input - if not hasattr(self.testInstance, '_start_time'): - self.testInstance._start_time = time.time() self.testInstance.setUp() if 'input' not in self.testInstance.report.keys(): @@ -139,6 +137,8 @@ class Measurement(TaskWithTimeout): def run(self): if 'measurement_start_time' not in self.testInstance.report.keys(): self.testInstance.report['measurement_start_time'] = otime.timestampNowLongUTC() + if not hasattr(self.testInstance, '_start_time'): + self.testInstance._start_time = time.time() return self.netTestMethod()

1 0

[ooni-probe/master] Update the updater
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 8fa6e1ef7d7e1d8d7140e6171a7debabeffab908 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Sep 7 13:21:27 2016 +0200 Update the updater --- data/updater.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/data/updater.py b/data/updater.py index 0f423ab..10086f5 100755 --- a/data/updater.py +++ b/data/updater.py @@ -20,8 +20,8 @@ from subprocess import check_output, check_call, CalledProcessError UPDATE_BASE_URL = "https://github.com/OpenObservatory/lepidopter-update/releases/download/" CURRENT_VERSION_PATH = "/etc/lepidopter-update/version" -UPDATER_PATH = "/opt/ooni/updater/versions/" -SCRIPT_INSTALL_PATH = "/opt/ooni/updater/updater.py" +UPDATER_PATH = "/opt/ooni/lepidopter-update/versions/" +SCRIPT_INSTALL_PATH = "/opt/ooni/lepidopter-update/updater.py" SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-update.service" SYSTEMD_SCRIPT = """\ @@ -30,14 +30,16 @@ Description=lepidopter-update service [Service] Type=simple -ExecStart={0} --log-file /var/log/lepidopter-update.log update --watch +Environment="UPDATER={0}" +ExecStart=$UPDATER --log-file /var/log/ooni/lepidopter-update.log update --watch TimeoutStartSec=300 Restart=on-failure + [Install] WantedBy=multi-user.target """.format(SCRIPT_INSTALL_PATH) -PUBLIC_KEY_PATH = "/opt/ooni/updater/public.asc" +PUBLIC_KEY_PATH = "/opt/ooni/lepidopter-update/public.asc" PUBLIC_KEY = """\ -----BEGIN PGP PUBLIC KEY BLOCK----- Comment: GPGTools - https://gpgtools.org @@ -306,8 +308,8 @@ def install(args): with open(SYSTEMD_SCRIPT_PATH, "w") as out_file: out_file.write(SYSTEMD_SCRIPT) - check_call(["systemctl", "enable", "lepidopter-updater"]) - check_call(["systemctl", "start", "lepidopter-updater"]) + check_call(["systemctl", "enable", "lepidopter-update"]) + check_call(["systemctl", "start", "lepidopter-update"]) class InvalidLogLevel(Exception): pass

1 0

[ooni-probe/master] Add ooniprobe systemd service
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 8da2312d61c4c1a840d0878e19967629526cd731 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 6 11:24:31 2016 +0200 Add ooniprobe systemd service --- scripts/systemd/ooniprobe.service | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/systemd/ooniprobe.service b/scripts/systemd/ooniprobe.service new file mode 100644 index 0000000..4743fac --- /dev/null +++ b/scripts/systemd/ooniprobe.service @@ -0,0 +1,15 @@ +[Unit] +Description=%n, network interference detection tool +After=network.target nss-lookup.target + +[Service] +Type=forking +PIDFile=/var/lib/ooni/twistd.pid +ExecStart=/usr/local/bin/ooniprobe-agent start +ExecStop=/usr/local/bin/ooniprobe-agent stop +TimeoutStartSec=300 +TimeoutStopSec=60 +Restart=on-failure + +[Install] +WantedBy=multi-user.target

1 0

[ooni-probe/master] Add to setup.py support for installing the updater
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 21cf4b8525e911e89ee5c567ae5d375f88fda767 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 5 16:36:13 2016 +0200 Add to setup.py support for installing the updater --- MANIFEST.in | 5 +- data/configs/lepidopter-ooniprobe.conf | 75 ------- data/configs/lepidopter-oonireport.conf | 69 ------- data/updater.py | 350 ++++++++++++++++++++++++++++++++ setup.py | 26 ++- 5 files changed, 375 insertions(+), 150 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 60d2ef9..0528d4b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,7 +6,8 @@ include data/ooniprobe.1 include data/oonireport.1 include data/ooniresources.1 include data/ooniprobe.conf.sample -include data/configs/lepidopter-ooniprobe.conf -include data/configs/lepidopter-oonireport.conf + +include data/updater.py + include ooni/settings.ini include ooni/ui/consent-form.md diff --git a/data/configs/lepidopter-ooniprobe.conf b/data/configs/lepidopter-ooniprobe.conf deleted file mode 100644 index 5c8ba40..0000000 --- a/data/configs/lepidopter-ooniprobe.conf +++ /dev/null @@ -1,75 +0,0 @@ -# This is the configuration file for OONIProbe -# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html -# Keep in mind that indentation matters. - -basic: - # Where OONIProbe should be writing it's log file - logfile: /var/log/ooni/ooniprobe.log - # in the future we will support loglevels - loglevel: WARNING -privacy: - # Should we include the IP address of the probe in the report? - includeip: false - # Should we include the ASN of the probe in the report? - includeasn: true - # Should we include the country as reported by GeoIP in the report? - includecountry: true - # Should we include the city as reported by GeoIP in the report? - includecity: false - # Should we collect a full packet capture on the client? - includepcap: false -reports: - # Should we place a unique ID inside of every report - unique_id: true - # This is a prefix for each packet capture file (.pcap) per test: - pcap: null - collector: null -advanced: - geoip_data_dir: /usr/share/GeoIP - debug: false - # enable if auto detection fails - #tor_binary: /usr/sbin/tor - #obfsproxy_binary: /usr/bin/obfsproxy - # For auto detection - interface: auto - # Of specify a specific interface - #interface: wlan0 - # If you do not specify start_tor, you will have to have Tor running and - # explicitly set the control port and SOCKS port - start_tor: true - # After how many seconds we should give up on a particular measurement - measurement_timeout: 120 - # After how many retries we should give up on a measurement - measurement_retries: 2 - # How many measurements to perform concurrently - measurement_concurrency: 4 - # After how may seconds we should give up reporting - reporting_timeout: 360 - # After how many retries to give up on reporting - reporting_retries: 5 - # How many reports to perform concurrently - reporting_concurrency: 7 - oonid_api_port: 8042 - report_log_file: null - inputs_dir: null - decks_dir: null -tor: - #socks_port: 8801 - #control_port: 8802 - # Specify the absolute path to the Tor bridges to use for testing - #bridges: bridges.list - # Specify path of the tor datadirectory. - # This should be set to something to avoid having Tor download each time - # the descriptors and consensus data. - data_dir: /opt/ooni/tor_data_dir -torrc: - #HTTPProxy: host:port - #HTTPProxyAuthenticator: user:password - #HTTPSProxy: host:port - #HTTPSProxyAuthenticator: user:password - # Uncomment following 5 lines to connect via meek pluggable transport in tor - #UseBridges: 1 - #Bridge: - #- "meek_lite 0.0.2.0:1 url=https://d2zfqthxsdq309.cloudfront.net/ front=a0.awsstatic.com" - #- "meek_lite 0.0.2.0:2 url=https://az786092.vo.msecnd.net/ front=ajax.aspnetcdn.com" - #ClientTransportPlugin: "meek_lite exec /usr/bin/obfs4proxy" diff --git a/data/configs/lepidopter-oonireport.conf b/data/configs/lepidopter-oonireport.conf deleted file mode 100644 index 8ea3ccc..0000000 --- a/data/configs/lepidopter-oonireport.conf +++ /dev/null @@ -1,69 +0,0 @@ -# This is the configuration file for OONIProbe -# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html -# Keep in mind that indentation matters. - -basic: - # Where OONIProbe should be writing it's log file - logfile: /var/log/ooni/oonireport.log - # in the future we will support loglevels - loglevel: WARNING -privacy: - # Should we include the IP address of the probe in the report? - includeip: false - # Should we include the ASN of the probe in the report? - includeasn: true - # Should we include the country as reported by GeoIP in the report? - includecountry: true - # Should we include the city as reported by GeoIP in the report? - includecity: false - # Should we collect a full packet capture on the client? - includepcap: false -reports: - # Should we place a unique ID inside of every report - unique_id: true - # This is a prefix for each packet capture file (.pcap) per test: - pcap: null - collector: null -advanced: - geoip_data_dir: /usr/share/GeoIP - debug: false - # enable if auto detection fails - #tor_binary: /usr/sbin/tor - #obfsproxy_binary: /usr/bin/obfsproxy - # For auto detection - interface: auto - # Of specify a specific interface - #interface: wlan0 - # If you do not specify start_tor, you will have to have Tor running and - # explicitly set the control port and SOCKS port - start_tor: false - # After how many seconds we should give up on a particular measurement - measurement_timeout: 120 - # After how many retries we should give up on a measurement - measurement_retries: 2 - # How many measurements to perform concurrently - measurement_concurrency: 4 - # After how may seconds we should give up reporting - reporting_timeout: 360 - # After how many retries to give up on reporting - reporting_retries: 5 - # How many reports to perform concurrently - reporting_concurrency: 7 - oonid_api_port: 8042 - report_log_file: null - inputs_dir: null - decks_dir: null -tor: - socks_port: 9050 - #control_port: 8802 - # Specify the absolute path to the Tor bridges to use for testing - #bridges: bridges.list - # Specify path of the tor datadirectory. - # This should be set to something to avoid having Tor download each time - # the descriptors and consensus data. - #data_dir: ~/.tor/ - torrc: - #HTTPProxy: host:port - #HTTPProxyAuthenticator: user:password - #HTTPSProxy: host:port - #HTTPSProxyAuthenticator: user:password diff --git a/data/updater.py b/data/updater.py new file mode 100755 index 0000000..dc2fb2a --- /dev/null +++ b/data/updater.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python2 + +from __future__ import print_function + +import os +import re +import imp # XPY3 this is deprecated in python3 +import time +import errno +import shutil +import logging +import tempfile +import argparse + +from subprocess import check_output, check_call, CalledProcessError + +# UPDATE_BASE_URL/latest/version must return an integer containing the latest version number +# UPDATE_BASE_URL/VERSION/update.py must return the update script for VERSION +# UPDATE_BASE_URL/VERSION/update.py.asc must return a valid GPG signature for update.py +UPDATE_BASE_URL = "https://github.com/OpenObservatory/lepidopter-update/releases/download/" + +CURRENT_VERSION_PATH = "/etc/lepidopter-update/version" +UPDATER_PATH = "/opt/ooni/updater/versions/" +SCRIPT_INSTALL_PATH = "/opt/ooni/updater/updater.py" + +SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-updater.service" +SYSTEMD_SCRIPT = """\ +[Unit] +Description=lepidopter-updater service + +[Service] +Type=simple +ExecStart={0} --log-file /var/log/lepidopter-update.log update --watch +TimeoutStartSec=300 +Restart=on-failure +[Install] +WantedBy=multi-user.target +""".format(SCRIPT_INSTALL_PATH) + +PUBLIC_KEY_PATH = "/opt/ooni/updater/public.asc" +PUBLIC_KEY = """\ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: GPGTools - https://gpgtools.org + +mQINBFfEAKABEADNBPp2nD48xXRhMdKMVXS2qHgDzokSAn3hikA+cb2IL5ssde0o +9HHzMxSNCbQBWo1bpmg84zsHvZTL+yEVGJ+o8DjLfdKKdMUOPsLTc0O1rqD0M6L4 +35n6JjaeJp98HhVIRkmNqBG4pWMKLqvW1crEt5U8m/X7LWtTzsBt2DPi6UB6yDqw +520DLK051/0WKE+s7W8f8hYheHqyaUl35wtU6Qj7kjcDm0Kg57l7pY7gdYEeRizA +TECXy2c2mKJusql3p65FD/jNX6TncfHWiESvS8p31E8xx1hfgsgmh15JqrMTALm/ +7cn3/IDV5vPBzi2pf4IlVHo34QcE26uj7QaXjrlQUkuds5cAFy/4uozN6J2PbH2x +e1+oI9rGxSf9m7UfAbudC+QATAlMDNeH2ngeqA0tm4vrMk/ybj5efeUjGNGNW0c8 +6xfhbyhNJb6Rw2ScwdFUc/niWone3O1J3QkQ6CS6/gT3JCBMRVwLl+CkbeaALBTI +6We0CNQc1FXcWB84LI9F3UAHiR9jrmA3J/ck4R1oqv9STTrClTdWIvCK4sNa0sv7 +ra1fdEV4CK1Z0qKxbKCk/JTlD/9w/OqZQqyJLOrWXomYxR6I6lxNwhoC+3Ysj5EG +Mmagpi+nnqAK0oIBkPytts9e6e1D54hS9sEG4uaEQRm229e0yhmQNQOKNwARAQAB +tDZPT05JIHNvZnR3YXJlIHVwZGF0ZSBrZXkgPGNvbnRhY3RAb3Blbm9ic2VydmF0 +b3J5Lm9yZz6JAj0EEwEKACcFAlfEAKACGwMFCQHhM4AFCwkIBwMFFQoJCAsFFgID +AQACHgECF4AACgkQw+zcBCBPnSm6ug//eVOV7RiG8q5ry64TvgeTNfPlVF0R3y3d +2dUNaWy+4H0ay9UjW/ayxZNnSSreVZY+50pOiqsKWdV5bEgtOZXkDfth8NuCNddo +CYmVkV/x2Mvmpf3eTBXlXtmFn9j2an3GKSSHFscdfdZsPATUUv+YFyX8LK5K6vq+ +BdNEGpqqHxPEM0wyQm2/f2s0dmjkmPFNZpCGWnuBRpQQD5O2YwFKK316VNdBXvVA +i9+MA81vLtn40FsOKZ/kDLt65khEdgYTYj8lRXIEWGuWp1iPUuMmEL8dxtlY8K1R +qU2JbgHHOA7RHnAUqgg0Hjmyg4ZsQ/ZyWi2/3IoLn/7QGeV0HBdiGMFuShSfkFWx +bNNMuei9FVK4nwXRLcVfAMXv1GtqQU9jTeCYXzxgr81rkEivkdqlZ3Iins+KgWEQ +SbEEYAXOWp/oheTBOBQvLSZi+2vjMiUeIQHQUDNfhlp3/Mk6RTVLMml6thIY/NyL +f/vABO5V9oKAdIaFMu/70tYn8PxTqPE0uJ7FwcTa7awp10dkpXXk0tm5ywYsms8l +CA/vizq7VMiZC9G4JvZqa3vXNBT1yFe+4Ri+fLtdZw9IDgECi5ZdQlp7dx2Rei2i +S2XkUwWR4Qv3/WzvPDChr25BMlu0Pkb8MbynrxcMs5ODFxOuOiP2kL4YW2Qppo66 +U3Z92swhAIq5Ag0EV8QAoAEQAOQwsRo+2260kBYKnxRHr6rzTjStXtxsCsMUB08E +XS7eTElwDSE2C+pfeQjFe366f1zNTxY/CN6wCtd7wI4cVXWKLescFfCUrsg+S0Wf +ot85AXqCqrPKFtKwW8khUeVnQfmHwhQl1W+/t+bE2p4X+0OR8qugHsMnvYwl+KpK +sZ094LwkO8GRySB+LKm6KQtJ+WOnsvs3X8v8fSA6GwJjYdtKqNUzPBLpw8RrIH9l +eaT2pe9Ta48GqEwrU8wxwKyRBIfJJP/zq5n1rKcOBpvLZDVcyrVw+pIGa0zfmr/c +qWYG7znx2Xq3i22d36xPkfkZEyVnQcCJJ28hkAfXRYpp+gMnL0Zt4u3GgzSARSBS +VrcMyNlaft/aSOkojyjh3+2zF1PCfW1Nw9Sx50gdN3FfF0yEWjUoA1R/NW9CQZVG +4qh/n2k508PYfZRuJ74T2jABFJIztv2pmq3VpSA7hkHGl3nXrdqpsw3V9bkFqZa/ +ihhY7IpGwUWx4pDHh1gKhjJ0qPUVK5sOx3GZfEvMCCiH9XPk70fn3nuYupRr9WNr +HJwUSeLMhRvi4jTT+z5QLdYloFRZmDRwNg63csGZRkly9vjrAiMVHMpcJI0eCei/ +XgeKSxoiAmzNuc2J47SF2z7WIsDwHhwRj6tj4dOW3Ye0WIkcTIvHd7UTVX02v+oB +d5YhABEBAAGJAiUEGAEKAA8FAlfEAKACGwwFCQHhM4AACgkQw+zcBCBPnSn25BAA +xU7NKi6BokqnloYncvL74fuUpam3LJBwsOkFehuO2D+X9A2blIpbpXtUoWXwRc9V +Jf7nL/yhMKcOB9m1MHVPtxtN5JzV9p8k2BT04/X6fa09umsJ3hwg/zhXrkGFrMVE +hfAk6q8a0Y5oJKUOoJhzxqD9ItibxHPkqb+R/GSMfrDIRE0ecfIltDLIRQMlBF1b +z5WN5Dwv8cikeQrsK6DNvbUUuHAbG8RZYG9QxFdkbehp46bCA8CfINENBzIskckx +xwlTtVxcDD0Irql7EuIM1bpWdFxBZPlmcDyLrZgCYgSqPOe2mOK9V37jM3fsTqR9 +C9fmr3DDmAm8XrUBL9ORwMTFa6LIUpUoSSzG258h9qPiBySj7b51QbrJ9WXiISsQ +5X/V8u6Qbp9PJnEUXXfIE9YVspF3+Zrfn0XqqjEND+bWi52cgDvwROSFf4KQnAto +FdlxHtxlcZgJEFeHLyb370XI4rcNs7zMoD3B06Eyay5oY98w9JFYwrm6bVvNzis7 +CrK2K46QzSUZm1BHTOi0AlnqpYtLaJnMSQCxTmjyRBaJM5DGXs/86y/OYdBcpiYG +aV0q9EcvDLQnSpa0vlsVM0AfMY+To1RTCMv+TGKMZHHWeV2yUABWK52raBpQHtjs +7SonBqg04+2E4w1WmZEx1u9QyDXmlaLxnR+YIqilM7g= +=RbWA +-----END PGP PUBLIC KEY BLOCK----- +""" + + +class RequestFailed(Exception): + pass + +def get_request(url, follow_redirects=True): + cmd = ["curl", "-q"] + if follow_redirects is True: + cmd.append("-L") + cmd.append(url) + + tmp_file = tempfile.TemporaryFile() + + try: + check_call(cmd, stdout=tmp_file) + except CalledProcessError: + raise RequestFailed + + tmp_file.seek(0) + + return tmp_file.read() + +def get_current_version(): + if not os.path.exists(CURRENT_VERSION_PATH): + return 0 + with open(CURRENT_VERSION_PATH) as in_file: + version = in_file.read() + return int(version) + +def get_latest_version(): + version = get_request(UPDATE_BASE_URL + "latest/version") + return int(version) + +class InvalidSignature(Exception): + pass + +class InvalidPublicKey(Exception): + pass + + +def verify_file(signature_path, signer_pk_path): + tmp_dir = tempfile.mkdtemp() + tmp_key = os.path.join(tmp_dir, "signing-key.gpg") + + try: + try: + check_call(["gpg", "--yes", "-o", tmp_key, "--dearmor", signer_pk_path]) + except CalledProcessError: + raise InvalidPublicKey + + try: + output = check_output(["gpg", "--status-fd", "1", + "--no-default-keyring", "--keyring", + tmp_key, "--trust-model", "always", + "--verify", signature_path]) + except CalledProcessError: + raise InvalidSignature + + except Exception as e: + raise e + + finally: + shutil.rmtree(tmp_dir) + + return output + +class UpdateFailed(Exception): + pass + +def perform_update(version, skip_verification=False): + try: + updater = get_request(UPDATE_BASE_URL + "{0}/update.py".format(version)) + updater_path = os.path.join(UPDATER_PATH, "update-{0}.py".format(version)) + except RequestFailed: + logging.error("Failed to download update file") + raise UpdateFailed + + if skip_verification is not True: + try: + updater_sig = get_request(UPDATE_BASE_URL + "{0}/update.py.asc".format(version)) + updater_sig_path = os.path.join(UPDATER_PATH, "update-{0}.py.asc".format(version)) + except RequestFailed: + logging.error("Failed to download update file") + raise UpdateFailed + + with open(updater_path, "w+") as out_file: + out_file.write(updater) + + if skip_verification is not True: + with open(updater_sig_path, "w+") as out_file: + out_file.write(updater_sig) + + if skip_verification is not True: + try: + verify_file(updater_sig_path, PUBLIC_KEY_PATH) + except InvalidSignature: + logging.error("Found an invalid signature. Bailing") + raise UpdateFailed + + updater = imp.load_source('updater_{0}'.format(version), + updater_path) + + try: + logging.info("Running install script") + updater.run() + except Exception: + logging.error("Failed to run the version update script for version {0}".format(version)) + raise UpdateFailed + + current_version_dir = os.path.dirname(CURRENT_VERSION_PATH) + try: + os.makedirs(current_version_dir) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise + + # Update the current version number + with open(CURRENT_VERSION_PATH, "w+") as out_file: + out_file.write(str(version)) + +def update_to_version(from_version, to_version, skip_verification=False): + versions = range(from_version + 1, to_version + 1) + for version in versions: + try: + perform_update(version, skip_verification) + except UpdateFailed: + logging.error("Failed to update to version {0}".format(version)) + return + +def check_for_update(skip_verification=False): + logging.info("Checking for update") + current_version = get_current_version() + try: + latest_version = get_latest_version() + except RequestFailed: + logging.error("Failed to learn the latest version") + return + + if current_version < latest_version: + logging.info("Updating {0}->{1}".format(current_version, latest_version)) + update_to_version(current_version, latest_version, skip_verification) + else: + logging.info("Already up to date") + +class InvalidInterval(Exception): + pass + +def _get_interval(interval): + """ + Returns the interval in seconds. + """ + seconds = 0 + INTERVAL_REGEXP = re.compile("(\d+d)?(\d+h)?(\d+m)?") + m = INTERVAL_REGEXP.match(interval) + days, hours, minutes = m.groups() + + if days is not None: + seconds += int(days[:-1]) * 24 * 60 * 60 + if hours is not None: + seconds += int(hours[:-1]) * 60 * 60 + if minutes is not None: + seconds += int(minutes[:-1]) * 60 + + if seconds == 0: + try: + seconds = int(interval) + except ValueError: + raise InvalidInterval + return seconds + + +def update(args): + """ + This command fires the updater. + """ + if args.watch is True: + seconds = _get_interval(args.interval) + while True: + check_for_update(skip_verification=args.skip_verification) + time.sleep(seconds) + else: + check_for_update(skip_verification=args.skip_verification) + + +def install(args): + """ + This command installs the updater. + """ + directories = [ + UPDATER_PATH, + os.path.dirname(CURRENT_VERSION_PATH) + ] + for path in directories: + try: + os.makedirs(path) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise + + with open(CURRENT_VERSION_PATH, "w") as out_file: + out_file.write("0") + + # Copy myself over to the SCRIPT_INSTALL_PATH + shutil.copyfile(__file__, SCRIPT_INSTALL_PATH) + os.chmod(SCRIPT_INSTALL_PATH, int('744', 8)) + + with open(PUBLIC_KEY_PATH, "w") as out_file: + out_file.write(PUBLIC_KEY) + os.chmod(PUBLIC_KEY_PATH, int('644', 8)) + + with open(SYSTEMD_SCRIPT_PATH, "w") as out_file: + out_file.write(SYSTEMD_SCRIPT) + + check_call(["systemctl", "enable", "lepidopter-updater"]) + check_call(["systemctl", "start", "lepidopter-updater"]) + +class InvalidLogLevel(Exception): + pass + +def _setup_logging(args): + log_file = args.log_file + + try: + log_level = getattr(logging, args.log_level) + except AttributeError: + raise InvalidLogLevel() + + logging.basicConfig(filename=log_file, level=log_level) + +def main(): + parser = argparse.ArgumentParser(description="Auto-update system for lepidopter") + parser.add_argument('--log-file', help="Specify the path to the logfile") + parser.add_argument('--log-level', help="Specify the loglevel (CRITICAL, ERROR, WARNING, INFO, DEBUG)", default="INFO") + + sub_parsers = parser.add_subparsers() + + parser_update = sub_parsers.add_parser('update') + parser_update.add_argument('--watch', + action='store_true', + help="Keep watching for changes in version and automatically update when a new version is available") + parser_update.add_argument('--interval', default='6h') + parser_update.add_argument('--skip-verification', + action='store_true', + help="Skip key verification (DANGER USE ONLY FOR TESTING))") + parser_update.set_defaults(func=update) + + parser_install = sub_parsers.add_parser('install') + parser_install.set_defaults(func=install) + + args = parser.parse_args() + _setup_logging(args) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 4c0bcfe..c465789 100644 --- a/setup.py +++ b/setup.py @@ -95,6 +95,7 @@ from ConfigParser import SafeConfigParser from os.path import join as pj from setuptools import setup from setuptools.command.install import install as InstallCommand +from subprocess import check_call from ooni import __version__, __author__ @@ -134,6 +135,18 @@ Topic :: System :: Networking :: Monitoring """ +def is_lepidopter(): + return os.path.exists('/etc/default/lepidopter') + + +def is_updater_installed(): + return os.path.exists('/etc/lepidopter-update/version') + + +def install_updater(): + check_call(["data/updater.py", "install"]) + + class OoniInstall(InstallCommand): def gen_config(self, share_path): config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample") @@ -183,15 +196,20 @@ class OoniInstall(InstallCommand): prefix = os.path.abspath(self.prefix) self.set_data_files(prefix) - def post_install(self): - pass - def run(self): self.pre_install() self.do_egg_install() - self.post_install() + def setup_package(): + if is_lepidopter() and not is_updater_installed(): + print("Lepidopter now requires that ooniprobe is installed via the " + "updater") + print("Let me install the auto-updater for you and we shall use that " + "for updates in the future.") + install_updater() + return + setup_requires = [] install_requires = [] dependency_links = []

1 0

[ooni-probe/master] Add some basic unittests for the ScheduledTasks
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 5723658b20d834dfd02b04264bf3f9e4cd8c9f50 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 6 16:50:03 2016 +0200 Add some basic unittests for the ScheduledTasks --- ooni/agent/scheduler.py | 6 +++--- ooni/settings.py | 2 -- ooni/tests/test_scheduler.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 21dd1cd..98b395b 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -53,7 +53,8 @@ class ScheduledTask(object): schedule = None identifier = None - def __init__(self, schedule=None, identifier=None): + def __init__(self, schedule=None, identifier=None, + scheduler_directory=config.scheduler_directory): if schedule is not None: self.schedule = schedule if identifier is not None: @@ -61,7 +62,6 @@ class ScheduledTask(object): assert self.identifier is not None, "self.identifier must be set" assert self.schedule is not None, "self.schedule must be set" - scheduler_directory = config.scheduler_directory self._last_run = FilePath(scheduler_directory).child(self.identifier) self._last_run_lock = FileSystemlockAndMutex( @@ -91,7 +91,7 @@ class ScheduledTask(object): out_file.write(current_time.strftime(self._time_format)) def task(self): - raise NotImplemented + raise NotImplementedError def first_run(self): """ diff --git a/ooni/settings.py b/ooni/settings.py index a1c55fc..409e81e 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -3,7 +3,6 @@ import sys import yaml import errno import getpass -from pkg_resources import parse_version from ConfigParser import SafeConfigParser from twisted.internet import defer, reactor @@ -11,7 +10,6 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from os.path import abspath, expanduser -from ooni import __version__ as ooniprobe_version from ooni.utils import Storage, log, get_ooni_root CONFIG_FILE_TEMPLATE = """\ diff --git a/ooni/tests/test_scheduler.py b/ooni/tests/test_scheduler.py new file mode 100644 index 0000000..cb90a30 --- /dev/null +++ b/ooni/tests/test_scheduler.py @@ -0,0 +1,51 @@ +import os +import shutil +import tempfile + +from twisted.internet import defer +from twisted.trial import unittest + +from ooni.agent.scheduler import ScheduledTask, DidNotRun + +class TestScheduler(unittest.TestCase): + def test_scheduled_task(self): + schedule = "@daily" + identifier = "dummy" + scheduler_directory = tempfile.mkdtemp() + scheduled_task = ScheduledTask(schedule=schedule, + identifier=identifier, + scheduler_directory=scheduler_directory + ) + self.assertEqual(scheduled_task.should_run, True) + self.assertFailure(scheduled_task.run(), NotImplementedError) + self.assertEqual(scheduled_task.should_run, True) + shutil.rmtree(scheduler_directory) + + @defer.inlineCallbacks + def test_call_twice_scheduled_task(self): + """ + If we call the scheduled task twice in a row the second time it will not run. + Tests for possible race conditions. + """ + scheduler_directory = tempfile.mkdtemp() + spam_path = os.path.join(scheduler_directory, 'spam.txt') + class DummyST(ScheduledTask): + def task(self): + with open(spam_path, 'w') as out_file: + out_file.write("I ran\n") + + schedule = "@daily" + identifier = "dummy" + dummy_st = DummyST(schedule=schedule, + identifier=identifier, + scheduler_directory=scheduler_directory + ) + + dummy_st.run() + yield self.assertFailure(dummy_st.run(), DidNotRun) + + with open(spam_path) as in_file: + self.assertEqual(len(in_file.readlines()), 1) + + self.assertEqual(dummy_st.should_run, False) + shutil.rmtree(scheduler_directory)

1 0

[ooni-probe/master] Update the updater
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 1046f0ec3692f57d80ee73488bcdabc56d6e315c Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 6 12:50:09 2016 +0200 Update the updater --- data/updater.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/data/updater.py b/data/updater.py index dc2fb2a..0f423ab 100755 --- a/data/updater.py +++ b/data/updater.py @@ -23,10 +23,10 @@ CURRENT_VERSION_PATH = "/etc/lepidopter-update/version" UPDATER_PATH = "/opt/ooni/updater/versions/" SCRIPT_INSTALL_PATH = "/opt/ooni/updater/updater.py" -SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-updater.service" +SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-update.service" SYSTEMD_SCRIPT = """\ [Unit] -Description=lepidopter-updater service +Description=lepidopter-update service [Service] Type=simple @@ -138,12 +138,13 @@ def verify_file(signature_path, signer_pk_path): try: try: - check_call(["gpg", "--yes", "-o", tmp_key, "--dearmor", signer_pk_path]) + check_call(["gpg", "--batch", "--yes", "-o", tmp_key, + "--dearmor", signer_pk_path]) except CalledProcessError: raise InvalidPublicKey try: - output = check_output(["gpg", "--status-fd", "1", + output = check_output(["gpg", "--batch", "--status-fd", "1", "--no-default-keyring", "--keyring", tmp_key, "--trust-model", "always", "--verify", signature_path])

1 0

[ooni-probe/master] Bugfixes to quota management
by art＠torproject.org 19 Sep '16

19 Sep '16

commit e6b6f023681bd4e04efa7eb896f2daf9291451ef Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 5 18:32:45 2016 +0200 Bugfixes to quota management --- ooni/agent/scheduler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 3cbd29d..21dd1cd 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -176,13 +176,13 @@ class CheckMeasurementQuota(ScheduledTask): if config.basic.measurement_quota is None: return maximum_bytes = human_size_to_bytes(config.basic.measurement_quota) - available_bytes = directory_usage(config.measurements_directory) + used_bytes = directory_usage(config.measurements_directory) warning_path = os.path.join(config.running_path, 'quota_warning') - if (float(available_bytes) / float(maximum_bytes)) >= self._warn_when: + if (float(used_bytes) / float(maximum_bytes)) >= self._warn_when: log.warn("You are about to reach the maximum allowed quota. Be careful") with open(warning_path, "w") as out_file: - out_file.write("{0} {1}".format(available_bytes, + out_file.write("{0} {1}".format(used_bytes, maximum_bytes)) else: try: @@ -191,12 +191,12 @@ class CheckMeasurementQuota(ScheduledTask): if ose.errno != errno.ENOENT: raise - if float(available_bytes) < float(maximum_bytes): + if float(used_bytes) < float(maximum_bytes): # We are within the allow quota exit. return # We should begin to delete old reports - amount_to_delete = float(maximum_bytes) - float(available_bytes) + amount_to_delete = float(used_bytes) - float(maximum_bytes) amount_deleted = 0 measurement_path = FilePath(config.measurements_directory)

1 0

[ooni-probe/master] Implement retry of director startup with bounded exponential binary backoff
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 4f80a3bb07df8d742cb79f8ca918734412159422 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 8 16:17:28 2016 +0200 Implement retry of director startup with bounded exponential binary backoff --- ooni/director.py | 7 +++---- ooni/geoip.py | 8 +++++++- ooni/ui/web/server.py | 29 ++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index cb2bdf0..1807e65 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -203,11 +203,9 @@ class Director(object): yield probe_ip.lookup() self.notify(DirectorEvent("success", "Looked up probe IP")) - self.notify(DirectorEvent("success", - "Running system tasks")) + self.notify(DirectorEvent("success", "Running system tasks")) yield run_system_tasks(no_input_store=not create_input_store) - self.notify(DirectorEvent("success", - "Ran system tasks")) + self.notify(DirectorEvent("success", "Ran system tasks")) @defer.inlineCallbacks def start(self, start_tor=False, check_incoherences=True, @@ -218,6 +216,7 @@ class Director(object): self._director_starting.callback(self._director_state) except Exception as exc: self._director_starting.errback(Failure(exc)) + raise @property def measurementSuccessRatio(self): diff --git a/ooni/geoip.py b/ooni/geoip.py index f271790..40fad25 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -181,12 +181,17 @@ class ProbeIP(object): self._state = INITIAL self._looking_up = defer.Deferred() self._looking_up.addCallback(self._looked_up) + self._looking_up.addErrback(self._lookup_failed) def _looked_up(self, result): self._last_lookup = time.time() self._reset_state() return result + def _lookup_failed(self, failure): + self._reset_state() + return failure + def resolveGeodata(self): from ooni.settings import config @@ -227,8 +232,9 @@ class ProbeIP(object): self.resolveGeodata() self._looking_up.callback(self.address) defer.returnValue(self.address) - except Exception: + except Exception as exc: log.msg("Unable to lookup the probe IP via GeoIPService") + self._looking_up.errback(defer.failure.Failure(exc)) raise @defer.inlineCallbacks diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 26bfd47..aed9951 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -4,6 +4,7 @@ import os import json import errno import string +import random from functools import wraps from random import SystemRandom @@ -150,6 +151,7 @@ class WebUIAPI(object): _enable_xsrf_protection = True def __init__(self, config, director, scheduler, _reactor=reactor): + self._reactor = reactor self.director = director self.scheduler = scheduler @@ -165,6 +167,12 @@ class WebUIAPI(object): self._director_started = False self._is_initialized = config.is_initialized() + # We use exponential backoff to trigger retries of the startup of + # the director. + self._director_startup_retries = 0 + # Maximum delay should be 6 hours + self._director_max_retry_delay = 6*60*60 + self.status_poller = LongPoller( self._long_polling_timeout, _reactor) self.director_event_poller = LongPoller( @@ -179,9 +187,11 @@ class WebUIAPI(object): self.start_director() def start_director(self): + log.debug("Starting director") d = self.director.start() d.addCallback(self.director_started) + d.addErrback(self.director_startup_failed) d.addBoth(lambda _: self.status_poller.notify()) @property @@ -208,7 +218,24 @@ class WebUIAPI(object): log.debug("Handling event {0}".format(event.type)) self.director_event_poller.notify(event) + def director_startup_failed(self, failure): + self._director_startup_retries += 1 + + # We delay the startup using binary exponential backoff with an + # upper bound. + startup_delay = random.uniform( + 0, min(2**self._director_startup_retries, + self._director_max_retry_delay) + ) + log.err("Failed to start the director, " + "retrying in {0}s".format(startup_delay)) + self._reactor.callLater( + startup_delay, + self.start_director + ) + def director_started(self, _): + log.debug("Started director") self._director_started = True @app.handle_errors(NotFound) @@ -435,7 +462,7 @@ class WebUIAPI(object): deck.load(deck_data) self.run_deck(deck) - except errors.MissingRequiredOption, option_name: + except errors.MissingRequiredOption as option_name: raise WebUIError( 400, 'Missing required option: "{}"'.format(option_name) )

1 0

[ooni-probe/master] Check permissions on logfile as well
by art＠torproject.org 19 Sep '16

19 Sep '16

commit bfc0a51a0b63ceb1d07812f2a7edc6a63847ea90 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 6 17:09:18 2016 +0200 Check permissions on logfile as well * Handle the case where the measurement directory doesn't exist yet --- ooni/measurements.py | 2 ++ ooni/utils/log.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ooni/measurements.py b/ooni/measurements.py index fd722ee..294b182 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -128,6 +128,8 @@ def get_summary(measurement_id): def list_measurements(compute_size=False): measurements = [] measurement_path = FilePath(config.measurements_directory) + if not measurement_path.exists(): + return measurements for measurement_id in measurement_path.listdir(): try: measurements.append(get_measurement(measurement_id, compute_size)) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 67079e0..93e6b31 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -133,9 +133,10 @@ class OONILogger(object): logfile = os.path.expanduser(config.basic.logfile) log_folder = os.path.dirname(logfile) - if not os.access(log_folder, os.W_OK): - # If we don't have permissions to write to the log_folder, - # write to running dir. + if (not os.access(log_folder, os.W_OK) or + (os.path.exists(logfile) and not os.access(logfile, os.W_OK))): + # If we don't have permissions to write to the log_folder or + # logfile. log_folder = config.running_path logfile = os.path.join(log_folder, "ooniprobe.log")

1 0