September 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Add some debugging logs to the refresh deck list
by art＠torproject.org 19 Sep '16

19 Sep '16

commit a3700b5eabd923886f0e6951784196bdc451caaa Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 13 17:20:34 2016 +0200 Add some debugging logs to the refresh deck list --- ooni/agent/scheduler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index d6fbc31..2797fba 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -278,6 +278,7 @@ class RefreshDeckList(ScheduledTask): def task(self): self.scheduler.refresh_deck_list() + class SendHeartBeat(ScheduledTask): """ This task is used to send a heartbeat that the probe is still alive and @@ -345,6 +346,7 @@ class SchedulerService(service.MultiService): # If we are not initialized we should not enable anything if not config.is_initialized(): + log.msg("We are not initialized skipping setup of decks") to_enable = [] for scheduled_task in self._scheduled_tasks[:]: @@ -355,13 +357,16 @@ class SchedulerService(service.MultiService): if info in to_enable: # If the task is already scheduled there is no need to # enable it. + log.msg("The deck {0} is already scheduled".format(deck_id)) to_enable.remove(info) else: # If one of the tasks that is scheduled is no longer in the # scheduled tasks. We should disable it. + log.msg("The deck task {0} should be disabled".format(deck_id)) self.unschedule(scheduled_task) for deck_id, schedule in to_enable: + log.msg("Scheduling to run {0}".format(deck_id)) self.schedule(RunDeck(self.director, deck_id, schedule)) def _task_did_not_run(self, failure, task):

1 0

[ooni-probe/master] When we talk about time ALWAYS use UTC
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 46027a39a768efa6beabac9af62765970762f6bf Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 13 13:21:36 2016 +0200 When we talk about time ALWAYS use UTC --- ooni/agent/scheduler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 0a6cc0f..d6fbc31 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -50,11 +50,13 @@ class FileSystemlockAndMutex(object): # We use this date to indicate that the scheduled task has never run. # Easter egg, try to see what is special about this date :)? -CANARY_DATE = datetime(1957, 8, 4) +CANARY_DATE = datetime(1957, 8, 4, tzinfo=tz.tzutc()) + class DidNotRun(Exception): pass + class ScheduledTask(object): _time_format = "%Y-%m-%dT%H:%M:%SZ" schedule = None @@ -80,7 +82,7 @@ class ScheduledTask(object): @property def should_run(self): - current_time = datetime.utcnow() + current_time = datetime.utcnow().replace(tzinfo=tz.tzutc()) next_cycle = croniter(self.schedule, self.last_run).get_next(datetime) if next_cycle <= current_time: return True

1 0

[ooni-probe/master] Enable submission of reports when run through the web user interface
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 2ab3219a6044b27c47dd310c1e6cecec20f006fd Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 13 18:57:03 2016 +0200 Enable submission of reports when run through the web user interface --- ooni/ui/web/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index aed9951..48f21c7 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -458,7 +458,7 @@ class WebUIAPI(object): ] } try: - deck = NGDeck(no_collector=True) + deck = NGDeck() deck.load(deck_data) self.run_deck(deck)

1 0

[ooni-probe/master] Bump up the default timeout for tor to 400 seconds
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 3f25dad2ed4ede50798465eb284dbbc8f52e5ee4 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 13 17:42:33 2016 +0200 Bump up the default timeout for tor to 400 seconds --- ooni/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/settings.py b/ooni/settings.py index be0025d..14f135f 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -86,7 +86,7 @@ tor: # # This is the timeout after which we consider to to not have # bootstrapped properly. - #timeout: 200 + #timeout: 400 #torrc: #HTTPProxy: host:port #HTTPProxyAuthenticator: user:password @@ -141,7 +141,7 @@ defaults = { "control_port": None, "bridges": None, "data_dir": None, - "timeout": 200, + "timeout": 400, "torrc": {} } }

1 0

[ooni-probe/master] Make changes to the updater based on feedback by @bassosimone
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 45993baf6bf0ae5377dcc9f99d9bf1c19a050b0b Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 15 12:41:42 2016 +0200 Make changes to the updater based on feedback by @bassosimone --- MANIFEST.in | 2 +- data/lepidopter-update.py | 385 ++++++++++++++++++++++++++++++++++++++++++++++ data/updater.py | 357 ------------------------------------------ setup.py | 2 +- 4 files changed, 387 insertions(+), 359 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0528d4b..258459e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,7 +7,7 @@ include data/oonireport.1 include data/ooniresources.1 include data/ooniprobe.conf.sample -include data/updater.py +include data/lepidopter-update.py include ooni/settings.ini include ooni/ui/consent-form.md diff --git a/data/lepidopter-update.py b/data/lepidopter-update.py new file mode 100755 index 0000000..bdd1cb5 --- /dev/null +++ b/data/lepidopter-update.py @@ -0,0 +1,385 @@ +#!/usr/bin/env python2 +""" +This is the auto-updater script for lepidopter. + +It must be run from root and it takes care of downloading the most recent +updates and doing all the operations needed to perform the update. + +To run it expects systemd to be configured. + +This script includes a self-installer which can be run via: + +python updater.py install + +It then expects to be run as a systemd service with: + +python updater.py update --watch +""" + +from __future__ import print_function + +import os +import re +import imp # XPY3 this is deprecated in python3 +import sys +import time +import errno +import shutil +import getpass +import logging +import tempfile +import argparse + +from subprocess import check_output, check_call, CalledProcessError + +# The version number of the updater +__version__ = "1.0.0" + +LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s" +# UPDATE_BASE_URL/latest/version must return an integer containing the latest version number +# UPDATE_BASE_URL/VERSION/update.py must return the update script for VERSION +# UPDATE_BASE_URL/VERSION/update.py.asc must return a valid GPG signature for update.py +UPDATE_BASE_URL = "https://github.com/OpenObservatory/lepidopter-update/releases/download/" + +CURRENT_VERSION_PATH = "/etc/lepidopter-update/version" +UPDATER_PATH = "/opt/ooni/lepidopter-update/versions/" +SCRIPT_INSTALL_PATH = "/opt/ooni/lepidopter-update/updater.py" + +SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-update.service" +SYSTEMD_SCRIPT = """\ +[Unit] +Description=lepidopter-update service + +[Service] +Type=simple +ExecStart={0} --log-file /var/log/ooni/lepidopter-update.log update --watch +TimeoutStartSec=300 +Restart=on-failure + +[Install] +WantedBy=multi-user.target +""".format(SCRIPT_INSTALL_PATH) + +PUBLIC_KEY_PATH = "/opt/ooni/lepidopter-update/public.asc" +PUBLIC_KEY = """\ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: GPGTools - https://gpgtools.org + +mQINBFfEAKABEADNBPp2nD48xXRhMdKMVXS2qHgDzokSAn3hikA+cb2IL5ssde0o +9HHzMxSNCbQBWo1bpmg84zsHvZTL+yEVGJ+o8DjLfdKKdMUOPsLTc0O1rqD0M6L4 +35n6JjaeJp98HhVIRkmNqBG4pWMKLqvW1crEt5U8m/X7LWtTzsBt2DPi6UB6yDqw +520DLK051/0WKE+s7W8f8hYheHqyaUl35wtU6Qj7kjcDm0Kg57l7pY7gdYEeRizA +TECXy2c2mKJusql3p65FD/jNX6TncfHWiESvS8p31E8xx1hfgsgmh15JqrMTALm/ +7cn3/IDV5vPBzi2pf4IlVHo34QcE26uj7QaXjrlQUkuds5cAFy/4uozN6J2PbH2x +e1+oI9rGxSf9m7UfAbudC+QATAlMDNeH2ngeqA0tm4vrMk/ybj5efeUjGNGNW0c8 +6xfhbyhNJb6Rw2ScwdFUc/niWone3O1J3QkQ6CS6/gT3JCBMRVwLl+CkbeaALBTI +6We0CNQc1FXcWB84LI9F3UAHiR9jrmA3J/ck4R1oqv9STTrClTdWIvCK4sNa0sv7 +ra1fdEV4CK1Z0qKxbKCk/JTlD/9w/OqZQqyJLOrWXomYxR6I6lxNwhoC+3Ysj5EG +Mmagpi+nnqAK0oIBkPytts9e6e1D54hS9sEG4uaEQRm229e0yhmQNQOKNwARAQAB +tDZPT05JIHNvZnR3YXJlIHVwZGF0ZSBrZXkgPGNvbnRhY3RAb3Blbm9ic2VydmF0 +b3J5Lm9yZz6JAj0EEwEKACcFAlfEAKACGwMFCQHhM4AFCwkIBwMFFQoJCAsFFgID +AQACHgECF4AACgkQw+zcBCBPnSm6ug//eVOV7RiG8q5ry64TvgeTNfPlVF0R3y3d +2dUNaWy+4H0ay9UjW/ayxZNnSSreVZY+50pOiqsKWdV5bEgtOZXkDfth8NuCNddo +CYmVkV/x2Mvmpf3eTBXlXtmFn9j2an3GKSSHFscdfdZsPATUUv+YFyX8LK5K6vq+ +BdNEGpqqHxPEM0wyQm2/f2s0dmjkmPFNZpCGWnuBRpQQD5O2YwFKK316VNdBXvVA +i9+MA81vLtn40FsOKZ/kDLt65khEdgYTYj8lRXIEWGuWp1iPUuMmEL8dxtlY8K1R +qU2JbgHHOA7RHnAUqgg0Hjmyg4ZsQ/ZyWi2/3IoLn/7QGeV0HBdiGMFuShSfkFWx +bNNMuei9FVK4nwXRLcVfAMXv1GtqQU9jTeCYXzxgr81rkEivkdqlZ3Iins+KgWEQ +SbEEYAXOWp/oheTBOBQvLSZi+2vjMiUeIQHQUDNfhlp3/Mk6RTVLMml6thIY/NyL +f/vABO5V9oKAdIaFMu/70tYn8PxTqPE0uJ7FwcTa7awp10dkpXXk0tm5ywYsms8l +CA/vizq7VMiZC9G4JvZqa3vXNBT1yFe+4Ri+fLtdZw9IDgECi5ZdQlp7dx2Rei2i +S2XkUwWR4Qv3/WzvPDChr25BMlu0Pkb8MbynrxcMs5ODFxOuOiP2kL4YW2Qppo66 +U3Z92swhAIq5Ag0EV8QAoAEQAOQwsRo+2260kBYKnxRHr6rzTjStXtxsCsMUB08E +XS7eTElwDSE2C+pfeQjFe366f1zNTxY/CN6wCtd7wI4cVXWKLescFfCUrsg+S0Wf +ot85AXqCqrPKFtKwW8khUeVnQfmHwhQl1W+/t+bE2p4X+0OR8qugHsMnvYwl+KpK +sZ094LwkO8GRySB+LKm6KQtJ+WOnsvs3X8v8fSA6GwJjYdtKqNUzPBLpw8RrIH9l +eaT2pe9Ta48GqEwrU8wxwKyRBIfJJP/zq5n1rKcOBpvLZDVcyrVw+pIGa0zfmr/c +qWYG7znx2Xq3i22d36xPkfkZEyVnQcCJJ28hkAfXRYpp+gMnL0Zt4u3GgzSARSBS +VrcMyNlaft/aSOkojyjh3+2zF1PCfW1Nw9Sx50gdN3FfF0yEWjUoA1R/NW9CQZVG +4qh/n2k508PYfZRuJ74T2jABFJIztv2pmq3VpSA7hkHGl3nXrdqpsw3V9bkFqZa/ +ihhY7IpGwUWx4pDHh1gKhjJ0qPUVK5sOx3GZfEvMCCiH9XPk70fn3nuYupRr9WNr +HJwUSeLMhRvi4jTT+z5QLdYloFRZmDRwNg63csGZRkly9vjrAiMVHMpcJI0eCei/ +XgeKSxoiAmzNuc2J47SF2z7WIsDwHhwRj6tj4dOW3Ye0WIkcTIvHd7UTVX02v+oB +d5YhABEBAAGJAiUEGAEKAA8FAlfEAKACGwwFCQHhM4AACgkQw+zcBCBPnSn25BAA +xU7NKi6BokqnloYncvL74fuUpam3LJBwsOkFehuO2D+X9A2blIpbpXtUoWXwRc9V +Jf7nL/yhMKcOB9m1MHVPtxtN5JzV9p8k2BT04/X6fa09umsJ3hwg/zhXrkGFrMVE +hfAk6q8a0Y5oJKUOoJhzxqD9ItibxHPkqb+R/GSMfrDIRE0ecfIltDLIRQMlBF1b +z5WN5Dwv8cikeQrsK6DNvbUUuHAbG8RZYG9QxFdkbehp46bCA8CfINENBzIskckx +xwlTtVxcDD0Irql7EuIM1bpWdFxBZPlmcDyLrZgCYgSqPOe2mOK9V37jM3fsTqR9 +C9fmr3DDmAm8XrUBL9ORwMTFa6LIUpUoSSzG258h9qPiBySj7b51QbrJ9WXiISsQ +5X/V8u6Qbp9PJnEUXXfIE9YVspF3+Zrfn0XqqjEND+bWi52cgDvwROSFf4KQnAto +FdlxHtxlcZgJEFeHLyb370XI4rcNs7zMoD3B06Eyay5oY98w9JFYwrm6bVvNzis7 +CrK2K46QzSUZm1BHTOi0AlnqpYtLaJnMSQCxTmjyRBaJM5DGXs/86y/OYdBcpiYG +aV0q9EcvDLQnSpa0vlsVM0AfMY+To1RTCMv+TGKMZHHWeV2yUABWK52raBpQHtjs +7SonBqg04+2E4w1WmZEx1u9QyDXmlaLxnR+YIqilM7g= +=RbWA +-----END PGP PUBLIC KEY BLOCK----- +""" + + +class RequestFailed(Exception): + pass + +def get_request(url, follow_redirects=True): + cmd = ["curl", "-q"] + if follow_redirects is True: + cmd.append("-L") + cmd.append(url) + + tmp_file = tempfile.TemporaryFile() + + try: + check_call(cmd, stdout=tmp_file) + except CalledProcessError: + raise RequestFailed + + tmp_file.seek(0) + + return tmp_file.read() + +def get_current_version(): + if not os.path.exists(CURRENT_VERSION_PATH): + return 0 + with open(CURRENT_VERSION_PATH) as in_file: + version = in_file.read() + return int(version) + +def get_latest_version(): + version = get_request(UPDATE_BASE_URL + "latest/version") + return int(version) + +class InvalidSignature(Exception): + pass + +class InvalidPublicKey(Exception): + pass + + +def verify_file(signature_path, file_path, signer_pk_path): + tmp_dir = tempfile.mkdtemp() + tmp_key = os.path.join(tmp_dir, "signing-key.gpg") + + try: + try: + check_call(["gpg", "--batch", "--yes", "-o", tmp_key, + "--dearmor", signer_pk_path]) + except CalledProcessError: + raise InvalidPublicKey + + try: + output = check_output(["gpg", "--batch", "--status-fd", "1", + "--no-default-keyring", "--keyring", + tmp_key, "--trust-model", "always", + "--verify", signature_path, file_path]) + except CalledProcessError: + raise InvalidSignature + + except Exception as e: + raise e + + finally: + shutil.rmtree(tmp_dir) + + return output + +class UpdateFailed(Exception): + pass + +def perform_update(version, skip_verification=False): + try: + updater = get_request(UPDATE_BASE_URL + "{0}/update.py".format(version)) + updater_path = os.path.join(UPDATER_PATH, "update-{0}.py".format(version)) + except RequestFailed: + logging.error("Failed to download update file") + raise UpdateFailed + + if skip_verification is not True: + try: + updater_sig = get_request(UPDATE_BASE_URL + "{0}/update.py.asc".format(version)) + updater_sig_path = os.path.join(UPDATER_PATH, "update-{0}.py.asc".format(version)) + except RequestFailed: + logging.error("Failed to download update file") + raise UpdateFailed + + with open(updater_path, "w+") as out_file: + out_file.write(updater) + + if skip_verification is not True: + with open(updater_sig_path, "w+") as out_file: + out_file.write(updater_sig) + + if skip_verification is not True: + try: + verify_file(updater_sig_path, updater_path, PUBLIC_KEY_PATH) + except InvalidSignature: + logging.error("Found an invalid signature. Bailing") + raise UpdateFailed + + updater = imp.load_source('updater_{0}'.format(version), + updater_path) + + try: + logging.info("Running install script") + if updater.__version__ != str(version): + logging.error("There is a version mismatch in the updater file. This could be a sign of a replay attack.") + raise UpdateFailed + updater.run() + except Exception: + logging.exception("Failed to run the version update script for version {0}".format(version)) + raise UpdateFailed + + current_version_dir = os.path.dirname(CURRENT_VERSION_PATH) + try: + os.makedirs(current_version_dir) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise + + # Update the current version number + with open(CURRENT_VERSION_PATH, "w+") as out_file: + out_file.write(str(version)) + + logging.info("Updated to version {0}".format(version)) + +def update_to_version(from_version, to_version, skip_verification=False): + versions = range(from_version + 1, to_version + 1) + for version in versions: + try: + perform_update(version, skip_verification) + except UpdateFailed: + logging.error("Failed to update to version {0}".format(version)) + return + +def check_for_update(skip_verification=False): + logging.info("Checking for update") + current_version = get_current_version() + try: + latest_version = get_latest_version() + except RequestFailed: + logging.error("Failed to learn the latest version") + return + + if current_version < latest_version: + logging.info("Updating {0}->{1}".format(current_version, latest_version)) + update_to_version(current_version, latest_version, skip_verification) + else: + logging.info("Already up to date") + +class InvalidInterval(Exception): + pass + +def _get_interval(interval): + """ + Returns the interval in seconds. + """ + seconds = 0 + INTERVAL_REGEXP = re.compile("(\d+d)?(\d+h)?(\d+m)?") + m = INTERVAL_REGEXP.match(interval) + days, hours, minutes = m.groups() + + if days is not None: + seconds += int(days[:-1]) * 24 * 60 * 60 + if hours is not None: + seconds += int(hours[:-1]) * 60 * 60 + if minutes is not None: + seconds += int(minutes[:-1]) * 60 + + if seconds == 0: + try: + seconds = int(interval) + except ValueError: + raise InvalidInterval + return seconds + + +def update(args): + """ + This command fires the updater. + """ + if args.watch is True: + seconds = _get_interval(args.interval) + while True: + check_for_update(skip_verification=args.skip_verification) + time.sleep(seconds) + else: + check_for_update(skip_verification=args.skip_verification) + + +def install(args): + """ + This command installs the updater. + """ + directories = [ + UPDATER_PATH, + os.path.dirname(CURRENT_VERSION_PATH) + ] + for path in directories: + try: + os.makedirs(path) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise + + with open(CURRENT_VERSION_PATH, "w") as out_file: + out_file.write("0") + + # Copy myself over to the SCRIPT_INSTALL_PATH + shutil.copyfile(__file__, SCRIPT_INSTALL_PATH) + os.chmod(SCRIPT_INSTALL_PATH, int('744', 8)) + + with open(PUBLIC_KEY_PATH, "w") as out_file: + out_file.write(PUBLIC_KEY) + os.chmod(PUBLIC_KEY_PATH, int('644', 8)) + + with open(SYSTEMD_SCRIPT_PATH, "w") as out_file: + out_file.write(SYSTEMD_SCRIPT) + + check_call(["systemctl", "enable", "lepidopter-update"]) + check_call(["systemctl", "start", "lepidopter-update"]) + +class InvalidLogLevel(Exception): + pass + +def _setup_logging(args): + log_file = args.log_file + + try: + log_level = getattr(logging, args.log_level) + except AttributeError: + raise InvalidLogLevel() + + logging.basicConfig(filename=log_file, level=log_level, format=LOG_FORMAT) + +def _check_user(): + if getpass.getuser() != 'root': + print("ERROR: this script must be run as root!") + sys.exit(1) + +def main(): + parser = argparse.ArgumentParser(description="Auto-update system for lepidopter") + parser.add_argument('--log-file', help="Specify the path to the logfile") + parser.add_argument('--log-level', help="Specify the loglevel (CRITICAL, ERROR, WARNING, INFO, DEBUG)", default="INFO") + + sub_parsers = parser.add_subparsers() + + parser_update = sub_parsers.add_parser('update') + parser_update.add_argument('--watch', + action='store_true', + help="Keep watching for changes in version and automatically update when a new version is available") + parser_update.add_argument('--interval', default='6h') + parser_update.add_argument('--skip-verification', + action='store_true', + help="Skip key verification (DANGER USE ONLY FOR TESTING))") + parser_update.set_defaults(func=update) + + parser_install = sub_parsers.add_parser('install') + parser_install.set_defaults(func=install) + + args = parser.parse_args() + _setup_logging(args) + _check_user() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/data/updater.py b/data/updater.py deleted file mode 100755 index 4cacf3d..0000000 --- a/data/updater.py +++ /dev/null @@ -1,357 +0,0 @@ -#!/usr/bin/env python2 - -from __future__ import print_function - -import os -import re -import imp # XPY3 this is deprecated in python3 -import time -import errno -import shutil -import logging -import tempfile -import argparse - -from subprocess import check_output, check_call, CalledProcessError - -# UPDATE_BASE_URL/latest/version must return an integer containing the latest version number -# UPDATE_BASE_URL/VERSION/update.py must return the update script for VERSION -# UPDATE_BASE_URL/VERSION/update.py.asc must return a valid GPG signature for update.py -UPDATE_BASE_URL = "https://github.com/OpenObservatory/lepidopter-update/releases/download/" - -CURRENT_VERSION_PATH = "/etc/lepidopter-update/version" -UPDATER_PATH = "/opt/ooni/lepidopter-update/versions/" -SCRIPT_INSTALL_PATH = "/opt/ooni/lepidopter-update/updater.py" - -SYSTEMD_SCRIPT_PATH = "/etc/systemd/system/lepidopter-update.service" -SYSTEMD_SCRIPT = """\ -[Unit] -Description=lepidopter-update service - -[Service] -Type=simple -ExecStart={0} --log-file /var/log/ooni/lepidopter-update.log update --watch -TimeoutStartSec=300 -Restart=on-failure - -[Install] -WantedBy=multi-user.target -""".format(SCRIPT_INSTALL_PATH) - -PUBLIC_KEY_PATH = "/opt/ooni/lepidopter-update/public.asc" -PUBLIC_KEY = """\ ------BEGIN PGP PUBLIC KEY BLOCK----- -Comment: GPGTools - https://gpgtools.org - -mQINBFfEAKABEADNBPp2nD48xXRhMdKMVXS2qHgDzokSAn3hikA+cb2IL5ssde0o -9HHzMxSNCbQBWo1bpmg84zsHvZTL+yEVGJ+o8DjLfdKKdMUOPsLTc0O1rqD0M6L4 -35n6JjaeJp98HhVIRkmNqBG4pWMKLqvW1crEt5U8m/X7LWtTzsBt2DPi6UB6yDqw -520DLK051/0WKE+s7W8f8hYheHqyaUl35wtU6Qj7kjcDm0Kg57l7pY7gdYEeRizA -TECXy2c2mKJusql3p65FD/jNX6TncfHWiESvS8p31E8xx1hfgsgmh15JqrMTALm/ -7cn3/IDV5vPBzi2pf4IlVHo34QcE26uj7QaXjrlQUkuds5cAFy/4uozN6J2PbH2x -e1+oI9rGxSf9m7UfAbudC+QATAlMDNeH2ngeqA0tm4vrMk/ybj5efeUjGNGNW0c8 -6xfhbyhNJb6Rw2ScwdFUc/niWone3O1J3QkQ6CS6/gT3JCBMRVwLl+CkbeaALBTI -6We0CNQc1FXcWB84LI9F3UAHiR9jrmA3J/ck4R1oqv9STTrClTdWIvCK4sNa0sv7 -ra1fdEV4CK1Z0qKxbKCk/JTlD/9w/OqZQqyJLOrWXomYxR6I6lxNwhoC+3Ysj5EG -Mmagpi+nnqAK0oIBkPytts9e6e1D54hS9sEG4uaEQRm229e0yhmQNQOKNwARAQAB -tDZPT05JIHNvZnR3YXJlIHVwZGF0ZSBrZXkgPGNvbnRhY3RAb3Blbm9ic2VydmF0 -b3J5Lm9yZz6JAj0EEwEKACcFAlfEAKACGwMFCQHhM4AFCwkIBwMFFQoJCAsFFgID -AQACHgECF4AACgkQw+zcBCBPnSm6ug//eVOV7RiG8q5ry64TvgeTNfPlVF0R3y3d -2dUNaWy+4H0ay9UjW/ayxZNnSSreVZY+50pOiqsKWdV5bEgtOZXkDfth8NuCNddo -CYmVkV/x2Mvmpf3eTBXlXtmFn9j2an3GKSSHFscdfdZsPATUUv+YFyX8LK5K6vq+ -BdNEGpqqHxPEM0wyQm2/f2s0dmjkmPFNZpCGWnuBRpQQD5O2YwFKK316VNdBXvVA -i9+MA81vLtn40FsOKZ/kDLt65khEdgYTYj8lRXIEWGuWp1iPUuMmEL8dxtlY8K1R -qU2JbgHHOA7RHnAUqgg0Hjmyg4ZsQ/ZyWi2/3IoLn/7QGeV0HBdiGMFuShSfkFWx -bNNMuei9FVK4nwXRLcVfAMXv1GtqQU9jTeCYXzxgr81rkEivkdqlZ3Iins+KgWEQ -SbEEYAXOWp/oheTBOBQvLSZi+2vjMiUeIQHQUDNfhlp3/Mk6RTVLMml6thIY/NyL -f/vABO5V9oKAdIaFMu/70tYn8PxTqPE0uJ7FwcTa7awp10dkpXXk0tm5ywYsms8l -CA/vizq7VMiZC9G4JvZqa3vXNBT1yFe+4Ri+fLtdZw9IDgECi5ZdQlp7dx2Rei2i -S2XkUwWR4Qv3/WzvPDChr25BMlu0Pkb8MbynrxcMs5ODFxOuOiP2kL4YW2Qppo66 -U3Z92swhAIq5Ag0EV8QAoAEQAOQwsRo+2260kBYKnxRHr6rzTjStXtxsCsMUB08E -XS7eTElwDSE2C+pfeQjFe366f1zNTxY/CN6wCtd7wI4cVXWKLescFfCUrsg+S0Wf -ot85AXqCqrPKFtKwW8khUeVnQfmHwhQl1W+/t+bE2p4X+0OR8qugHsMnvYwl+KpK -sZ094LwkO8GRySB+LKm6KQtJ+WOnsvs3X8v8fSA6GwJjYdtKqNUzPBLpw8RrIH9l -eaT2pe9Ta48GqEwrU8wxwKyRBIfJJP/zq5n1rKcOBpvLZDVcyrVw+pIGa0zfmr/c -qWYG7znx2Xq3i22d36xPkfkZEyVnQcCJJ28hkAfXRYpp+gMnL0Zt4u3GgzSARSBS -VrcMyNlaft/aSOkojyjh3+2zF1PCfW1Nw9Sx50gdN3FfF0yEWjUoA1R/NW9CQZVG -4qh/n2k508PYfZRuJ74T2jABFJIztv2pmq3VpSA7hkHGl3nXrdqpsw3V9bkFqZa/ -ihhY7IpGwUWx4pDHh1gKhjJ0qPUVK5sOx3GZfEvMCCiH9XPk70fn3nuYupRr9WNr -HJwUSeLMhRvi4jTT+z5QLdYloFRZmDRwNg63csGZRkly9vjrAiMVHMpcJI0eCei/ -XgeKSxoiAmzNuc2J47SF2z7WIsDwHhwRj6tj4dOW3Ye0WIkcTIvHd7UTVX02v+oB -d5YhABEBAAGJAiUEGAEKAA8FAlfEAKACGwwFCQHhM4AACgkQw+zcBCBPnSn25BAA -xU7NKi6BokqnloYncvL74fuUpam3LJBwsOkFehuO2D+X9A2blIpbpXtUoWXwRc9V -Jf7nL/yhMKcOB9m1MHVPtxtN5JzV9p8k2BT04/X6fa09umsJ3hwg/zhXrkGFrMVE -hfAk6q8a0Y5oJKUOoJhzxqD9ItibxHPkqb+R/GSMfrDIRE0ecfIltDLIRQMlBF1b -z5WN5Dwv8cikeQrsK6DNvbUUuHAbG8RZYG9QxFdkbehp46bCA8CfINENBzIskckx -xwlTtVxcDD0Irql7EuIM1bpWdFxBZPlmcDyLrZgCYgSqPOe2mOK9V37jM3fsTqR9 -C9fmr3DDmAm8XrUBL9ORwMTFa6LIUpUoSSzG258h9qPiBySj7b51QbrJ9WXiISsQ -5X/V8u6Qbp9PJnEUXXfIE9YVspF3+Zrfn0XqqjEND+bWi52cgDvwROSFf4KQnAto -FdlxHtxlcZgJEFeHLyb370XI4rcNs7zMoD3B06Eyay5oY98w9JFYwrm6bVvNzis7 -CrK2K46QzSUZm1BHTOi0AlnqpYtLaJnMSQCxTmjyRBaJM5DGXs/86y/OYdBcpiYG -aV0q9EcvDLQnSpa0vlsVM0AfMY+To1RTCMv+TGKMZHHWeV2yUABWK52raBpQHtjs -7SonBqg04+2E4w1WmZEx1u9QyDXmlaLxnR+YIqilM7g= -=RbWA ------END PGP PUBLIC KEY BLOCK----- -""" - - -class RequestFailed(Exception): - pass - -def get_request(url, follow_redirects=True): - cmd = ["curl", "-q"] - if follow_redirects is True: - cmd.append("-L") - cmd.append(url) - - tmp_file = tempfile.TemporaryFile() - - try: - check_call(cmd, stdout=tmp_file) - except CalledProcessError: - raise RequestFailed - - tmp_file.seek(0) - - return tmp_file.read() - -def get_current_version(): - if not os.path.exists(CURRENT_VERSION_PATH): - return 0 - with open(CURRENT_VERSION_PATH) as in_file: - version = in_file.read() - return int(version) - -def get_latest_version(): - version = get_request(UPDATE_BASE_URL + "latest/version") - return int(version) - -class InvalidSignature(Exception): - pass - -class InvalidPublicKey(Exception): - pass - - -def verify_file(signature_path, file_path, signer_pk_path): - tmp_dir = tempfile.mkdtemp() - tmp_key = os.path.join(tmp_dir, "signing-key.gpg") - - try: - try: - check_call(["gpg", "--batch", "--yes", "-o", tmp_key, - "--dearmor", signer_pk_path]) - except CalledProcessError: - raise InvalidPublicKey - - try: - output = check_output(["gpg", "--batch", "--status-fd", "1", - "--no-default-keyring", "--keyring", - tmp_key, "--trust-model", "always", - "--verify", signature_path, file_path]) - except CalledProcessError: - raise InvalidSignature - - except Exception as e: - raise e - - finally: - shutil.rmtree(tmp_dir) - - return output - -class UpdateFailed(Exception): - pass - -def perform_update(version, skip_verification=False): - try: - updater = get_request(UPDATE_BASE_URL + "{0}/update.py".format(version)) - updater_path = os.path.join(UPDATER_PATH, "update-{0}.py".format(version)) - except RequestFailed: - logging.error("Failed to download update file") - raise UpdateFailed - - if skip_verification is not True: - try: - updater_sig = get_request(UPDATE_BASE_URL + "{0}/update.py.asc".format(version)) - updater_sig_path = os.path.join(UPDATER_PATH, "update-{0}.py.asc".format(version)) - except RequestFailed: - logging.error("Failed to download update file") - raise UpdateFailed - - with open(updater_path, "w+") as out_file: - out_file.write(updater) - - if skip_verification is not True: - with open(updater_sig_path, "w+") as out_file: - out_file.write(updater_sig) - - if skip_verification is not True: - try: - verify_file(updater_sig_path, updater_path, PUBLIC_KEY_PATH) - except InvalidSignature: - logging.error("Found an invalid signature. Bailing") - raise UpdateFailed - - updater = imp.load_source('updater_{0}'.format(version), - updater_path) - - try: - logging.info("Running install script") - if updater.__version__ != str(version): - logging.error("There is a version mismatch in the updater file. This could be a sign of a replay attack.") - raise UpdateFailed - updater.run() - except Exception: - logging.exception("Failed to run the version update script for version {0}".format(version)) - raise UpdateFailed - - current_version_dir = os.path.dirname(CURRENT_VERSION_PATH) - try: - os.makedirs(current_version_dir) - except OSError as ose: - if ose.errno != errno.EEXIST: - raise - - # Update the current version number - with open(CURRENT_VERSION_PATH, "w+") as out_file: - out_file.write(str(version)) - - logging.info("Updated to version {0}".format(version)) - -def update_to_version(from_version, to_version, skip_verification=False): - versions = range(from_version + 1, to_version + 1) - for version in versions: - try: - perform_update(version, skip_verification) - except UpdateFailed: - logging.error("Failed to update to version {0}".format(version)) - return - -def check_for_update(skip_verification=False): - logging.info("Checking for update") - current_version = get_current_version() - try: - latest_version = get_latest_version() - except RequestFailed: - logging.error("Failed to learn the latest version") - return - - if current_version < latest_version: - logging.info("Updating {0}->{1}".format(current_version, latest_version)) - update_to_version(current_version, latest_version, skip_verification) - else: - logging.info("Already up to date") - -class InvalidInterval(Exception): - pass - -def _get_interval(interval): - """ - Returns the interval in seconds. - """ - seconds = 0 - INTERVAL_REGEXP = re.compile("(\d+d)?(\d+h)?(\d+m)?") - m = INTERVAL_REGEXP.match(interval) - days, hours, minutes = m.groups() - - if days is not None: - seconds += int(days[:-1]) * 24 * 60 * 60 - if hours is not None: - seconds += int(hours[:-1]) * 60 * 60 - if minutes is not None: - seconds += int(minutes[:-1]) * 60 - - if seconds == 0: - try: - seconds = int(interval) - except ValueError: - raise InvalidInterval - return seconds - - -def update(args): - """ - This command fires the updater. - """ - if args.watch is True: - seconds = _get_interval(args.interval) - while True: - check_for_update(skip_verification=args.skip_verification) - time.sleep(seconds) - else: - check_for_update(skip_verification=args.skip_verification) - - -def install(args): - """ - This command installs the updater. - """ - directories = [ - UPDATER_PATH, - os.path.dirname(CURRENT_VERSION_PATH) - ] - for path in directories: - try: - os.makedirs(path) - except OSError as ose: - if ose.errno != errno.EEXIST: - raise - - with open(CURRENT_VERSION_PATH, "w") as out_file: - out_file.write("0") - - # Copy myself over to the SCRIPT_INSTALL_PATH - shutil.copyfile(__file__, SCRIPT_INSTALL_PATH) - os.chmod(SCRIPT_INSTALL_PATH, int('744', 8)) - - with open(PUBLIC_KEY_PATH, "w") as out_file: - out_file.write(PUBLIC_KEY) - os.chmod(PUBLIC_KEY_PATH, int('644', 8)) - - with open(SYSTEMD_SCRIPT_PATH, "w") as out_file: - out_file.write(SYSTEMD_SCRIPT) - - check_call(["systemctl", "enable", "lepidopter-update"]) - check_call(["systemctl", "start", "lepidopter-update"]) - -class InvalidLogLevel(Exception): - pass - -def _setup_logging(args): - log_file = args.log_file - - try: - log_level = getattr(logging, args.log_level) - except AttributeError: - raise InvalidLogLevel() - - logging.basicConfig(filename=log_file, level=log_level) - -def main(): - parser = argparse.ArgumentParser(description="Auto-update system for lepidopter") - parser.add_argument('--log-file', help="Specify the path to the logfile") - parser.add_argument('--log-level', help="Specify the loglevel (CRITICAL, ERROR, WARNING, INFO, DEBUG)", default="INFO") - - sub_parsers = parser.add_subparsers() - - parser_update = sub_parsers.add_parser('update') - parser_update.add_argument('--watch', - action='store_true', - help="Keep watching for changes in version and automatically update when a new version is available") - parser_update.add_argument('--interval', default='6h') - parser_update.add_argument('--skip-verification', - action='store_true', - help="Skip key verification (DANGER USE ONLY FOR TESTING))") - parser_update.set_defaults(func=update) - - parser_install = sub_parsers.add_parser('install') - parser_install.set_defaults(func=install) - - args = parser.parse_args() - _setup_logging(args) - args.func(args) - - -if __name__ == "__main__": - main() diff --git a/setup.py b/setup.py index e6031c6..4acbe77 100644 --- a/setup.py +++ b/setup.py @@ -145,7 +145,7 @@ def is_updater_installed(): def install_lepidopter_update(): - check_call(["data/updater.py", "install"]) + check_call(["data/lepidopter-update.py", "install"]) def mkdir_p(path):

1 0

[ooni-probe/master] Force creation of decks-available directory.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 28a441b1da39b8b8de28752dcfc0578352f6911c Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 13 15:28:02 2016 +0200 Force creation of decks-available directory. Failing to do so will lead to the decks not being copied over and pip will silently fail. --- setup.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 60d08a5..753cf07 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ Have fun! from __future__ import print_function import os +import errno import tempfile from glob import glob @@ -147,6 +148,13 @@ def install_lepidopter_update(): check_call(["data/updater.py", "install"]) +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as ose: + if ose != errno.EEXIST: + raise + class OoniInstall(InstallCommand): def gen_config(self, share_path): config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample") @@ -183,14 +191,9 @@ class OoniInstall(InstallCommand): with open("ooni/settings.ini", "w+") as fp: settings.write(fp) - try: - os.makedirs(pj(var_path, 'ooni')) - except OSError: - pass - try: - os.makedirs(pj(share_path, 'ooni')) - except OSError: - pass + mkdir_p(pj(var_path, 'ooni')) + mkdir_p(pj(share_path, 'ooni')) + mkdir_p(pj(share_path, 'ooni', 'decks-available')) def pre_install(self): prefix = os.path.abspath(self.prefix)

1 0

[ooni-probe/master] Move check for running process into utility function
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 8c3a65edc75e6830f4e5c7a6c2e1afd55851de3e Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 15 16:22:12 2016 +0200 Move check for running process into utility function * More fixes based on review by @bassosimone --- ooni/measurements.py | 18 +++--------------- ooni/nettest.py | 2 +- ooni/scripts/ooniprobe_agent.py | 16 ++++------------ ooni/utils/__init__.py | 15 +++++++++++++++ 4 files changed, 23 insertions(+), 28 deletions(-) diff --git a/ooni/measurements.py b/ooni/measurements.py index 294b182..422fc55 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -1,9 +1,7 @@ -import os import json -import signal from twisted.python.filepath import FilePath -from ooni.utils import log +from ooni.utils import log, is_process_running from ooni.utils.files import directory_usage from ooni.settings import config @@ -75,14 +73,11 @@ def get_measurement(measurement_id, compute_size=False): stale = False if measurement.child("measurements.njson.progress").exists(): completed = False - # XXX this is done quite often around the code, probably should - # be moved into some utility function. pid = measurement.child("running.pid").open("r").read() pid = int(pid) - try: - os.kill(pid, signal.SIG_DFL) + if is_process_running(pid): running = True - except OSError: + else: stale = True if measurement.child("keep").exists(): @@ -136,10 +131,3 @@ def list_measurements(compute_size=False): except: log.err("Failed to get metadata for measurement {0}".format(measurement_id)) return measurements - -if __name__ == "__main__": - import sys - if len(sys.argv) != 3: - print("Usage: {0} [input_file] [output_file]".format(sys.argv[0])) - sys.exit(1) - generate_summary(sys.argv[1], sys.argv[2]) diff --git a/ooni/nettest.py b/ooni/nettest.py index 4ea3329..d6784f4 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -242,7 +242,7 @@ class NetTestLoader(object): m = ONION_INPUT_REGEXP.match(filename) if m: raise e.InvalidInputFile("Input files hosted on hidden services " - "are not longer supported") + "are no longer supported") else: input_file['filename'] = filename self.inputFiles.append(input_file) diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index 3eb1d22..c455f9b 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -8,7 +8,7 @@ import signal from twisted.scripts import twistd from twisted.python import usage -from ooni.utils import log +from ooni.utils import log, is_process_running from ooni.settings import config from ooni.agent.agent import AgentService @@ -102,18 +102,10 @@ def get_running_pidfile(): continue pid = open(pidfile, "r").read() pid = int(pid) - try: - os.kill(pid, signal.SIG_DFL) + if is_process_running(pid): running_pidfile = pidfile - break - except OSError as ose: - if ose.errno == errno.ESRCH: - # Found pid, but isn't running - continue - elif ose.errno == errno.EPERM: - # The process is owned by root. We assume it's running - running_pidfile = pidfile - break + else: + continue if running_pidfile is None: raise NotRunning return running_pidfile diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 247758f..a894daf 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -1,6 +1,8 @@ import shutil import string import random +import signal +import errno import gzip import os @@ -160,3 +162,16 @@ def gunzip(file_path): def get_ooni_root(): script = os.path.join(__file__, '..') return os.path.dirname(os.path.realpath(script)) + +def is_process_running(pid): + try: + os.kill(pid, signal.SIG_DFL) + running = True + except OSError as ose: + if ose.errno == errno.EPERM: + running = True + elif ose.errno == errno.ESRCH: + running = False + else: + raise + return running

1 0

[ooni-probe/master] The last runtime should be determined based on the time we started the task
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 5c0fd074630b7fdd4e066c0b783242880d0838a3 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Sep 12 14:53:32 2016 +0200 The last runtime should be determined based on the time we started the task --- ooni/agent/scheduler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 4002369..a7713e0 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -93,10 +93,9 @@ class ScheduledTask(object): date_str = in_file.read() return datetime.strptime(date_str, self._time_format) - def _update_last_run(self): + def _update_last_run(self, last_run_time): with self._last_run.open('w') as out_file: - current_time = datetime.utcnow() - out_file.write(current_time.strftime(self._time_format)) + out_file.write(last_run_time.strftime(self._time_format)) def task(self): raise NotImplementedError @@ -117,8 +116,9 @@ class ScheduledTask(object): try: if self.last_run == CANARY_DATE: yield defer.maybeDeferred(self.first_run) + last_run_time = datetime.utcnow() yield self.task() - self._update_last_run() + self._update_last_run(last_run_time) except: raise finally:

1 0

[ooni-probe/master] Add keyword arguments to ProbeIP.lookup method
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 60fd4ec19818f77f68824c59db17739f86c87914 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 16 11:55:07 2016 +0200 Add keyword arguments to ProbeIP.lookup method --- ooni/geoip.py | 17 ++++++++++------- ooni/scripts/oonideckgen.py | 3 +-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ooni/geoip.py b/ooni/geoip.py index 40fad25..5e6b275 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -192,20 +192,23 @@ class ProbeIP(object): self._reset_state() return failure - def resolveGeodata(self): + def resolveGeodata(self, + include_ip=None, + include_asn=None, + include_country=None): from ooni.settings import config self.geodata = ip_to_location(self.address) self.geodata['ip'] = self.address - if not config.privacy.includeasn: + if not config.privacy.includeasn and include_asn is not True: self.geodata['asn'] = 'AS0' - if not config.privacy.includecountry: + if not config.privacy.includecountry and include_country is not True: self.geodata['countrycode'] = 'ZZ' - if not config.privacy.includeip: + if not config.privacy.includeip and include_ip is not True: self.geodata['ip'] = '127.0.0.1' @defer.inlineCallbacks - def lookup(self): + def lookup(self, include_ip=None, include_asn=None, include_country=None): if self._state == IN_PROGRESS: yield self._looking_up elif self._last_lookup < time.time() - self._expire_in: @@ -218,7 +221,7 @@ class ProbeIP(object): try: yield self.askTor() log.msg("Found your IP via Tor") - self.resolveGeodata() + self.resolveGeodata(include_ip, include_asn, include_country) self._looking_up.callback(self.address) defer.returnValue(self.address) except errors.TorStateNotFound: @@ -229,7 +232,7 @@ class ProbeIP(object): try: yield self.askGeoIPService() log.msg("Found your IP via a GeoIP service") - self.resolveGeodata() + self.resolveGeodata(include_ip, include_asn, include_country) self._looking_up.callback(self.address) defer.returnValue(self.address) except Exception as exc: diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py index 6c2882c..4ad0f59 100644 --- a/ooni/scripts/oonideckgen.py +++ b/ooni/scripts/oonideckgen.py @@ -84,8 +84,7 @@ def generate_deck(options): @defer.inlineCallbacks def get_user_country_code(): - config.privacy.includecountry = True - yield probe_ip.lookup() + yield probe_ip.lookup(include_country=True) defer.returnValue(probe_ip.geodata['countrycode'])

1 0

[ooni-probe/master] Add note about locking logic for run() of tasks
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 6d13ade8bf4faae42e18c4e2c79dc695891d75c4 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Sep 15 19:39:17 2016 +0200 Add note about locking logic for run() of tasks --- ooni/agent/scheduler.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index c86e3b6..e8fef53 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -398,6 +398,15 @@ class SchedulerService(service.MultiService): """ This function is called every self.interval seconds to check which periodic tasks should be run. + + Note: the task will wait on the lock if there is already a task of + that type running. This means that if a task is very long running + there can potentially be a pretty large backlog of accumulated + periodic tasks waiting to know if they should run. + XXX + We may want to do something like not wait on the lock if there is + already a queue that is larger than a certain amount or something + smarter if still starts to become a memory usage concern. """ for task in self._scheduled_tasks: log.debug("Running task {0}".format(task.identifier))

1 0