commit 21cf4b8525e911e89ee5c567ae5d375f88fda767 Author: Arturo Filastò <arturo@filasto.net> Date: Mon Sep 5 16:36:13 2016 +0200 Add to setup.py support for installing the updater --- MANIFEST.in | 5 +- data/configs/lepidopter-ooniprobe.conf | 75 ------- data/configs/lepidopter-oonireport.conf | 69 ------- data/updater.py | 350 ++++++++++++++++++++++++++++++++ setup.py | 26 ++- 5 files changed, 375 insertions(+), 150 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 60d2ef9..0528d4b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,7 +6,8 @@ include data/ooniprobe.1 include data/oonireport.1 include data/ooniresources.1 include data/ooniprobe.conf.sample -include data/configs/lepidopter-ooniprobe.conf -include data/configs/lepidopter-oonireport.conf + +include data/updater.py + include ooni/settings.ini include ooni/ui/consent-form.md diff --git a/data/configs/lepidopter-ooniprobe.conf b/data/configs/lepidopter-ooniprobe.conf deleted file mode 100644 index 5c8ba40..0000000 --- a/data/configs/lepidopter-ooniprobe.conf +++ /dev/null @@ -1,75 +0,0 @@ -# This is the configuration file for OONIProbe -# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html -# Keep in mind that indentation matters. - -basic: - # Where OONIProbe should be writing it's log file - logfile: /var/log/ooni/ooniprobe.log - # in the future we will support loglevels - loglevel: WARNING -privacy: - # Should we include the IP address of the probe in the report? - includeip: false - # Should we include the ASN of the probe in the report? - includeasn: true - # Should we include the country as reported by GeoIP in the report? - includecountry: true - # Should we include the city as reported by GeoIP in the report? - includecity: false - # Should we collect a full packet capture on the client? - includepcap: false -reports: - # Should we place a unique ID inside of every report - unique_id: true - # This is a prefix for each packet capture file (.pcap) per test: - pcap: null - collector: null -advanced: - geoip_data_dir: /usr/share/GeoIP - debug: false - # enable if auto detection fails - #tor_binary: /usr/sbin/tor - #obfsproxy_binary: /usr/bin/obfsproxy - # For auto detection - interface: auto - # Of specify a specific interface - #interface: wlan0 - # If you do not specify start_tor, you will have to have Tor running and - # explicitly set the control port and SOCKS port - start_tor: true - # After how many seconds we should give up on a particular measurement - measurement_timeout: 120 - # After how many retries we should give up on a measurement - measurement_retries: 2 - # How many measurements to perform concurrently - measurement_concurrency: 4 - # After how may seconds we should give up reporting - reporting_timeout: 360 - # After how many retries to give up on reporting - reporting_retries: 5 - # How many reports to perform concurrently - reporting_concurrency: 7 - oonid_api_port: 8042 - report_log_file: null - inputs_dir: null - decks_dir: null -tor: - #socks_port: 8801 - #control_port: 8802 - # Specify the absolute path to the Tor bridges to use for testing - #bridges: bridges.list - # Specify path of the tor datadirectory. - # This should be set to something to avoid having Tor download each time - # the descriptors and consensus data. - data_dir: /opt/ooni/tor_data_dir -torrc: - #HTTPProxy: host:port - #HTTPProxyAuthenticator: user:password - #HTTPSProxy: host:port - #HTTPSProxyAuthenticator: user:password - # Uncomment following 5 lines to connect via meek pluggable transport in tor - #UseBridges: 1 - #Bridge: - #- "meek_lite 0.0.2.0:1 url=https://d2zfqthxsdq309.cloudfront.net/ front=a0.awsstatic.com" - #- "meek_lite 0.0.2.0:2 url=https://az786092.vo.msecnd.net/ front=ajax.aspnetcdn.com" - #ClientTransportPlugin: "meek_lite exec /usr/bin/obfs4proxy" diff --git a/data/configs/lepidopter-oonireport.conf b/data/configs/lepidopter-oonireport.conf deleted file mode 100644 index 8ea3ccc..0000000 --- a/data/configs/lepidopter-oonireport.conf +++ /dev/null @@ -1,69 +0,0 @@ -# This is the configuration file for OONIProbe -# This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html -# Keep in mind that indentation matters. - -basic: - # Where OONIProbe should be writing it's log file - logfile: /var/log/ooni/oonireport.log - # in the future we will support loglevels - loglevel: WARNING -privacy: - # Should we include the IP address of the probe in the report? - includeip: false - # Should we include the ASN of the probe in the report? - includeasn: true - # Should we include the country as reported by GeoIP in the report? - includecountry: true - # Should we include the city as reported by GeoIP in the report? - includecity: false - # Should we collect a full packet capture on the client? - includepcap: false -reports: - # Should we place a unique ID inside of every report - unique_id: true - # This is a prefix for each packet capture file (.pcap) per test: - pcap: null - collector: null -advanced: - geoip_data_dir: /usr/share/GeoIP - debug: false - # enable if auto detection fails - #tor_binary: /usr/sbin/tor - #obfsproxy_binary: /usr/bin/obfsproxy - # For auto detection - interface: auto - # Of specify a specific interface - #interface: wlan0 - # If you do not specify start_tor, you will have to have Tor running and - # explicitly set the control port and SOCKS port - start_tor: false - # After how many seconds we should give up on a particular measurement - measurement_timeout: 120 - # After how many retries we should give up on a measurement - measurement_retries: 2 - # How many measurements to perform concurrently - measurement_concurrency: 4 - # After how may seconds we should give up reporting - reporting_timeout: 360 - # After how many retries to give up on reporting - reporting_retries: 5 - # How many reports to perform concurrently - reporting_concurrency: 7 - oonid_api_port: 8042 - report_log_file: null - inputs_dir: null - decks_dir: null -tor: - socks_port: 9050 - #control_port: 8802 - # Specify the absolute path to the Tor bridges to use for testing - #bridges: bridges.list - # Specify path of the tor datadirectory. - # This should be set to something to avoid having Tor download each time - # the descriptors and consensus data. - #data_dir: ~/.tor/ - torrc: - #HTTPProxy: host:port - #HTTPProxyAuthenticator: user:password - #HTTPSProxy: host:port - #HTTPSProxyAuthenticator: user:password diff --git a/data/updater.py b/data/updater.py new file mode 100755 index 0000000..dc2fb2a --- /dev/null +++ b/data/updater.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python2 + +from __future__ import print_function + +import os +import re +import imp # XPY3 this is deprecated in python3 +import time +import errno +import shutil +import logging +import tempfile +import argparse + +from subprocess import check_output, check_call, CalledProcessError + +# UPDATE_BASE_URL/latest/version must return an integer containing the latest version number +# UPDATE_BASE_URL/VERSION/update.py must return the update script for VERSION +# UPDATE_BASE_URL/VERSION/update.py.asc must return a valid GPG signature for update.py +UPDATE_BASE_URL = "https://github.com/OpenObservatory/lepidopter-update/releases/download/" + +CURRENT_VERSION_PATH = "/etc/lepidopter-update/version" +UPDATER_PATH = "/opt/ooni/updater/versions/" +SCRIPT_INSTALL_PATH = "/opt/ooni/updater/updater.py" + +SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-updater.service" +SYSTEMD_SCRIPT = """\ +[Unit] +Description=lepidopter-updater service + +[Service] +Type=simple +ExecStart={0} --log-file /var/log/lepidopter-update.log update --watch +TimeoutStartSec=300 +Restart=on-failure +[Install] +WantedBy=multi-user.target +""".format(SCRIPT_INSTALL_PATH) + +PUBLIC_KEY_PATH = "/opt/ooni/updater/public.asc" +PUBLIC_KEY = """\ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: GPGTools - https://gpgtools.org + +mQINBFfEAKABEADNBPp2nD48xXRhMdKMVXS2qHgDzokSAn3hikA+cb2IL5ssde0o +9HHzMxSNCbQBWo1bpmg84zsHvZTL+yEVGJ+o8DjLfdKKdMUOPsLTc0O1rqD0M6L4 +35n6JjaeJp98HhVIRkmNqBG4pWMKLqvW1crEt5U8m/X7LWtTzsBt2DPi6UB6yDqw +520DLK051/0WKE+s7W8f8hYheHqyaUl35wtU6Qj7kjcDm0Kg57l7pY7gdYEeRizA +TECXy2c2mKJusql3p65FD/jNX6TncfHWiESvS8p31E8xx1hfgsgmh15JqrMTALm/ +7cn3/IDV5vPBzi2pf4IlVHo34QcE26uj7QaXjrlQUkuds5cAFy/4uozN6J2PbH2x +e1+oI9rGxSf9m7UfAbudC+QATAlMDNeH2ngeqA0tm4vrMk/ybj5efeUjGNGNW0c8 +6xfhbyhNJb6Rw2ScwdFUc/niWone3O1J3QkQ6CS6/gT3JCBMRVwLl+CkbeaALBTI +6We0CNQc1FXcWB84LI9F3UAHiR9jrmA3J/ck4R1oqv9STTrClTdWIvCK4sNa0sv7 +ra1fdEV4CK1Z0qKxbKCk/JTlD/9w/OqZQqyJLOrWXomYxR6I6lxNwhoC+3Ysj5EG +Mmagpi+nnqAK0oIBkPytts9e6e1D54hS9sEG4uaEQRm229e0yhmQNQOKNwARAQAB +tDZPT05JIHNvZnR3YXJlIHVwZGF0ZSBrZXkgPGNvbnRhY3RAb3Blbm9ic2VydmF0 +b3J5Lm9yZz6JAj0EEwEKACcFAlfEAKACGwMFCQHhM4AFCwkIBwMFFQoJCAsFFgID +AQACHgECF4AACgkQw+zcBCBPnSm6ug//eVOV7RiG8q5ry64TvgeTNfPlVF0R3y3d +2dUNaWy+4H0ay9UjW/ayxZNnSSreVZY+50pOiqsKWdV5bEgtOZXkDfth8NuCNddo +CYmVkV/x2Mvmpf3eTBXlXtmFn9j2an3GKSSHFscdfdZsPATUUv+YFyX8LK5K6vq+ +BdNEGpqqHxPEM0wyQm2/f2s0dmjkmPFNZpCGWnuBRpQQD5O2YwFKK316VNdBXvVA +i9+MA81vLtn40FsOKZ/kDLt65khEdgYTYj8lRXIEWGuWp1iPUuMmEL8dxtlY8K1R +qU2JbgHHOA7RHnAUqgg0Hjmyg4ZsQ/ZyWi2/3IoLn/7QGeV0HBdiGMFuShSfkFWx +bNNMuei9FVK4nwXRLcVfAMXv1GtqQU9jTeCYXzxgr81rkEivkdqlZ3Iins+KgWEQ +SbEEYAXOWp/oheTBOBQvLSZi+2vjMiUeIQHQUDNfhlp3/Mk6RTVLMml6thIY/NyL +f/vABO5V9oKAdIaFMu/70tYn8PxTqPE0uJ7FwcTa7awp10dkpXXk0tm5ywYsms8l +CA/vizq7VMiZC9G4JvZqa3vXNBT1yFe+4Ri+fLtdZw9IDgECi5ZdQlp7dx2Rei2i +S2XkUwWR4Qv3/WzvPDChr25BMlu0Pkb8MbynrxcMs5ODFxOuOiP2kL4YW2Qppo66 +U3Z92swhAIq5Ag0EV8QAoAEQAOQwsRo+2260kBYKnxRHr6rzTjStXtxsCsMUB08E +XS7eTElwDSE2C+pfeQjFe366f1zNTxY/CN6wCtd7wI4cVXWKLescFfCUrsg+S0Wf +ot85AXqCqrPKFtKwW8khUeVnQfmHwhQl1W+/t+bE2p4X+0OR8qugHsMnvYwl+KpK +sZ094LwkO8GRySB+LKm6KQtJ+WOnsvs3X8v8fSA6GwJjYdtKqNUzPBLpw8RrIH9l +eaT2pe9Ta48GqEwrU8wxwKyRBIfJJP/zq5n1rKcOBpvLZDVcyrVw+pIGa0zfmr/c +qWYG7znx2Xq3i22d36xPkfkZEyVnQcCJJ28hkAfXRYpp+gMnL0Zt4u3GgzSARSBS +VrcMyNlaft/aSOkojyjh3+2zF1PCfW1Nw9Sx50gdN3FfF0yEWjUoA1R/NW9CQZVG +4qh/n2k508PYfZRuJ74T2jABFJIztv2pmq3VpSA7hkHGl3nXrdqpsw3V9bkFqZa/ +ihhY7IpGwUWx4pDHh1gKhjJ0qPUVK5sOx3GZfEvMCCiH9XPk70fn3nuYupRr9WNr +HJwUSeLMhRvi4jTT+z5QLdYloFRZmDRwNg63csGZRkly9vjrAiMVHMpcJI0eCei/ +XgeKSxoiAmzNuc2J47SF2z7WIsDwHhwRj6tj4dOW3Ye0WIkcTIvHd7UTVX02v+oB +d5YhABEBAAGJAiUEGAEKAA8FAlfEAKACGwwFCQHhM4AACgkQw+zcBCBPnSn25BAA +xU7NKi6BokqnloYncvL74fuUpam3LJBwsOkFehuO2D+X9A2blIpbpXtUoWXwRc9V +Jf7nL/yhMKcOB9m1MHVPtxtN5JzV9p8k2BT04/X6fa09umsJ3hwg/zhXrkGFrMVE +hfAk6q8a0Y5oJKUOoJhzxqD9ItibxHPkqb+R/GSMfrDIRE0ecfIltDLIRQMlBF1b +z5WN5Dwv8cikeQrsK6DNvbUUuHAbG8RZYG9QxFdkbehp46bCA8CfINENBzIskckx +xwlTtVxcDD0Irql7EuIM1bpWdFxBZPlmcDyLrZgCYgSqPOe2mOK9V37jM3fsTqR9 +C9fmr3DDmAm8XrUBL9ORwMTFa6LIUpUoSSzG258h9qPiBySj7b51QbrJ9WXiISsQ +5X/V8u6Qbp9PJnEUXXfIE9YVspF3+Zrfn0XqqjEND+bWi52cgDvwROSFf4KQnAto +FdlxHtxlcZgJEFeHLyb370XI4rcNs7zMoD3B06Eyay5oY98w9JFYwrm6bVvNzis7 +CrK2K46QzSUZm1BHTOi0AlnqpYtLaJnMSQCxTmjyRBaJM5DGXs/86y/OYdBcpiYG +aV0q9EcvDLQnSpa0vlsVM0AfMY+To1RTCMv+TGKMZHHWeV2yUABWK52raBpQHtjs +7SonBqg04+2E4w1WmZEx1u9QyDXmlaLxnR+YIqilM7g= +=RbWA +-----END PGP PUBLIC KEY BLOCK----- +""" + + +class RequestFailed(Exception): + pass + +def get_request(url, follow_redirects=True): + cmd = ["curl", "-q"] + if follow_redirects is True: + cmd.append("-L") + cmd.append(url) + + tmp_file = tempfile.TemporaryFile() + + try: + check_call(cmd, stdout=tmp_file) + except CalledProcessError: + raise RequestFailed + + tmp_file.seek(0) + + return tmp_file.read() + +def get_current_version(): + if not os.path.exists(CURRENT_VERSION_PATH): + return 0 + with open(CURRENT_VERSION_PATH) as in_file: + version = in_file.read() + return int(version) + +def get_latest_version(): + version = get_request(UPDATE_BASE_URL + "latest/version") + return int(version) + +class InvalidSignature(Exception): + pass + +class InvalidPublicKey(Exception): + pass + + +def verify_file(signature_path, signer_pk_path): + tmp_dir = tempfile.mkdtemp() + tmp_key = os.path.join(tmp_dir, "signing-key.gpg") + + try: + try: + check_call(["gpg", "--yes", "-o", tmp_key, "--dearmor", signer_pk_path]) + except CalledProcessError: + raise InvalidPublicKey + + try: + output = check_output(["gpg", "--status-fd", "1", + "--no-default-keyring", "--keyring", + tmp_key, "--trust-model", "always", + "--verify", signature_path]) + except CalledProcessError: + raise InvalidSignature + + except Exception as e: + raise e + + finally: + shutil.rmtree(tmp_dir) + + return output + +class UpdateFailed(Exception): + pass + +def perform_update(version, skip_verification=False): + try: + updater = get_request(UPDATE_BASE_URL + "{0}/update.py".format(version)) + updater_path = os.path.join(UPDATER_PATH, "update-{0}.py".format(version)) + except RequestFailed: + logging.error("Failed to download update file") + raise UpdateFailed + + if skip_verification is not True: + try: + updater_sig = get_request(UPDATE_BASE_URL + "{0}/update.py.asc".format(version)) + updater_sig_path = os.path.join(UPDATER_PATH, "update-{0}.py.asc".format(version)) + except RequestFailed: + logging.error("Failed to download update file") + raise UpdateFailed + + with open(updater_path, "w+") as out_file: + out_file.write(updater) + + if skip_verification is not True: + with open(updater_sig_path, "w+") as out_file: + out_file.write(updater_sig) + + if skip_verification is not True: + try: + verify_file(updater_sig_path, PUBLIC_KEY_PATH) + except InvalidSignature: + logging.error("Found an invalid signature. Bailing") + raise UpdateFailed + + updater = imp.load_source('updater_{0}'.format(version), + updater_path) + + try: + logging.info("Running install script") + updater.run() + except Exception: + logging.error("Failed to run the version update script for version {0}".format(version)) + raise UpdateFailed + + current_version_dir = os.path.dirname(CURRENT_VERSION_PATH) + try: + os.makedirs(current_version_dir) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise + + # Update the current version number + with open(CURRENT_VERSION_PATH, "w+") as out_file: + out_file.write(str(version)) + +def update_to_version(from_version, to_version, skip_verification=False): + versions = range(from_version + 1, to_version + 1) + for version in versions: + try: + perform_update(version, skip_verification) + except UpdateFailed: + logging.error("Failed to update to version {0}".format(version)) + return + +def check_for_update(skip_verification=False): + logging.info("Checking for update") + current_version = get_current_version() + try: + latest_version = get_latest_version() + except RequestFailed: + logging.error("Failed to learn the latest version") + return + + if current_version < latest_version: + logging.info("Updating {0}->{1}".format(current_version, latest_version)) + update_to_version(current_version, latest_version, skip_verification) + else: + logging.info("Already up to date") + +class InvalidInterval(Exception): + pass + +def _get_interval(interval): + """ + Returns the interval in seconds. + """ + seconds = 0 + INTERVAL_REGEXP = re.compile("(\d+d)?(\d+h)?(\d+m)?") + m = INTERVAL_REGEXP.match(interval) + days, hours, minutes = m.groups() + + if days is not None: + seconds += int(days[:-1]) * 24 * 60 * 60 + if hours is not None: + seconds += int(hours[:-1]) * 60 * 60 + if minutes is not None: + seconds += int(minutes[:-1]) * 60 + + if seconds == 0: + try: + seconds = int(interval) + except ValueError: + raise InvalidInterval + return seconds + + +def update(args): + """ + This command fires the updater. + """ + if args.watch is True: + seconds = _get_interval(args.interval) + while True: + check_for_update(skip_verification=args.skip_verification) + time.sleep(seconds) + else: + check_for_update(skip_verification=args.skip_verification) + + +def install(args): + """ + This command installs the updater. + """ + directories = [ + UPDATER_PATH, + os.path.dirname(CURRENT_VERSION_PATH) + ] + for path in directories: + try: + os.makedirs(path) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise + + with open(CURRENT_VERSION_PATH, "w") as out_file: + out_file.write("0") + + # Copy myself over to the SCRIPT_INSTALL_PATH + shutil.copyfile(__file__, SCRIPT_INSTALL_PATH) + os.chmod(SCRIPT_INSTALL_PATH, int('744', 8)) + + with open(PUBLIC_KEY_PATH, "w") as out_file: + out_file.write(PUBLIC_KEY) + os.chmod(PUBLIC_KEY_PATH, int('644', 8)) + + with open(SYSTEMD_SCRIPT_PATH, "w") as out_file: + out_file.write(SYSTEMD_SCRIPT) + + check_call(["systemctl", "enable", "lepidopter-updater"]) + check_call(["systemctl", "start", "lepidopter-updater"]) + +class InvalidLogLevel(Exception): + pass + +def _setup_logging(args): + log_file = args.log_file + + try: + log_level = getattr(logging, args.log_level) + except AttributeError: + raise InvalidLogLevel() + + logging.basicConfig(filename=log_file, level=log_level) + +def main(): + parser = argparse.ArgumentParser(description="Auto-update system for lepidopter") + parser.add_argument('--log-file', help="Specify the path to the logfile") + parser.add_argument('--log-level', help="Specify the loglevel (CRITICAL, ERROR, WARNING, INFO, DEBUG)", default="INFO") + + sub_parsers = parser.add_subparsers() + + parser_update = sub_parsers.add_parser('update') + parser_update.add_argument('--watch', + action='store_true', + help="Keep watching for changes in version and automatically update when a new version is available") + parser_update.add_argument('--interval', default='6h') + parser_update.add_argument('--skip-verification', + action='store_true', + help="Skip key verification (DANGER USE ONLY FOR TESTING))") + parser_update.set_defaults(func=update) + + parser_install = sub_parsers.add_parser('install') + parser_install.set_defaults(func=install) + + args = parser.parse_args() + _setup_logging(args) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 4c0bcfe..c465789 100644 --- a/setup.py +++ b/setup.py @@ -95,6 +95,7 @@ from ConfigParser import SafeConfigParser from os.path import join as pj from setuptools import setup from setuptools.command.install import install as InstallCommand +from subprocess import check_call from ooni import __version__, __author__ @@ -134,6 +135,18 @@ Topic :: System :: Networking :: Monitoring """ +def is_lepidopter(): + return os.path.exists('/etc/default/lepidopter') + + +def is_updater_installed(): + return os.path.exists('/etc/lepidopter-update/version') + + +def install_updater(): + check_call(["data/updater.py", "install"]) + + class OoniInstall(InstallCommand): def gen_config(self, share_path): config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample") @@ -183,15 +196,20 @@ class OoniInstall(InstallCommand): prefix = os.path.abspath(self.prefix) self.set_data_files(prefix) - def post_install(self): - pass - def run(self): self.pre_install() self.do_egg_install() - self.post_install() + def setup_package(): + if is_lepidopter() and not is_updater_installed(): + print("Lepidopter now requires that ooniprobe is installed via the " + "updater") + print("Let me install the auto-updater for you and we shall use that " + "for updates in the future.") + install_updater() + return + setup_requires = [] install_requires = [] dependency_links = []