commit f7fec4b6d0aca457cfafdb0617218f2b4c0f0345 Author: Arturo Filastò <arturo@filasto.net> Date: Sat Jul 30 17:53:32 2016 +0200 Run the check_for_update function when installation is complete --- Vagrantfile | 2 +- ooni/agent/scheduler.py | 6 ++- setup.py | 114 +++--------------------------------------------- 3 files changed, 10 insertions(+), 112 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index cf73244..0e75d55 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,7 +13,7 @@ apt-get update apt-get install -y tor deb.torproject.org-keyring # Setup for sniffer subsystem -apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip +apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip libssl-dev cd /data/ooni-probe python setup.py install diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 7a77afb..0d8689c 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -1,7 +1,8 @@ from datetime import datetime from twisted.application import service -from twisted.internet import task, defer +from twisted.internet import defer +from twisted.internet.task import LoopingCall from twisted.python.filepath import FilePath from ooni.scripts import oonireport @@ -72,6 +73,7 @@ class ScheduledTask(object): finally: self._last_run_lock.unlock() + class UpdateInputsAndResources(ScheduledTask): identifier = "update-inputs" schedule = "@daily" @@ -177,7 +179,7 @@ class SchedulerService(service.MultiService): service.MultiService.__init__(self) self.director = director self.interval = interval - self._looping_call = task.LoopingCall(self._should_run) + self._looping_call = LoopingCall(self._should_run) self._scheduled_tasks = [] def schedule(self, task): diff --git a/setup.py b/setup.py index 6940f66..da48323 100644 --- a/setup.py +++ b/setup.py @@ -89,16 +89,14 @@ Have fun! from __future__ import print_function import os -import sys import shutil import tempfile import subprocess from ConfigParser import SafeConfigParser from os.path import join as pj -from setuptools import setup, Command +from setuptools import setup from setuptools.command.install import install -from distutils.spawn import find_executable from ooni import __version__, __author__ @@ -177,11 +175,9 @@ class OoniInstall(install): pass def ooniresources(self): - ooniresources = find_executable("ooniresources") - process = subprocess.Popen([ooniresources], - stdout=sys.stdout.fileno(), - stderr=sys.stderr.fileno()) - process.wait() + from ooni.resources import check_for_update + from twisted.internet import task + task.react(lambda _: check_for_update()) def update_lepidopter_config(self): try: @@ -200,105 +196,6 @@ class OoniInstall(install): if is_lepidopter(): self.update_lepidopter_config() -class ExecutableNotFound(Exception): - pass - -class CreateOoniResources(Command): - description = ("Create ooni-resources.tar.gz containing test-lists and " - "GeoIP data files") - user_options = [] - - def initialize_options(self): - pass - def finalize_options(self): - pass - def download(self, url, directory, filename): - dst_path = pj(directory, filename) - args = [ - self.wget, - "-O", - dst_path, - url - ] - out = run_command(args) - if out is None: - raise Exception("Failed to download {0}".format(url)) - return dst_path - - def find_executables(self): - self.wget = find_executable("wget") - if not self.wget: - raise ExecutableNotFound("wget") - self.tar = find_executable("tar") - if not self.tar: - raise ExecutableNotFound("tar") - self.unzip = find_executable("unzip") - if not self.unzip: - raise ExecutableNotFound("unzip") - self.gunzip = find_executable("gunzip") - if not self.gunzip: - raise ExecutableNotFound("gunzip") - - def run(self): - dst_path = "dist/ooni-resources.tar.gz" - - try: - self.find_executables() - except ExecutableNotFound as enf: - print("ERR: Could not find '{0}'".format(enf.message)) - return - - tmp_dir = tempfile.mkdtemp() - pkg_dir = tempfile.mkdtemp() - - os.mkdir(pj(pkg_dir, "resources")) - os.mkdir(pj(pkg_dir, "GeoIP")) - - try: - geoip_asn_path = self.download(GEOIP_ASN_URL, tmp_dir, "GeoIPASNum.dat.gz") - except Exception as exc: - print(exc.message) - return - try: - geoip_path = self.download(GEOIP_URL, tmp_dir, "GeoIP.dat.gz") - except Exception as exc: - print(exc.message) - return - try: - test_lists_path = self.download(TEST_LISTS_URL, tmp_dir, "master.zip") - except Exception as exc: - print(exc.message) - return - - run_command([self.gunzip, geoip_asn_path]) - run_command([self.gunzip, geoip_path]) - run_command([self.unzip, "-d", tmp_dir, test_lists_path]) - - shutil.move(pj(tmp_dir, "GeoIP.dat"), - pj(pkg_dir, "GeoIP", "GeoIP.dat")) - shutil.move(pj(tmp_dir, "GeoIPASNum.dat"), - pj(pkg_dir, "GeoIP", "GeoIPASNum.dat")) - shutil.move(pj(tmp_dir, "test-lists-master", "lists"), - pj(pkg_dir, "resources", "citizenlab-test-lists")) - # Don't include services and official lists - shutil.rmtree( - pj(pkg_dir, - "resources", - "citizenlab-test-lists", - "services"), - ignore_errors=True) - shutil.rmtree( - pj(pkg_dir, - "resources", - "citizenlab-test-lists", - "official"), - ignore_errors=True) - run_command([self.tar, "cvzf", dst_path, "-C", pkg_dir, "."]) - - # Cleanup - shutil.rmtree(pkg_dir, ignore_errors=True) - shutil.rmtree(tmp_dir, ignore_errors=True) - print("Written ooniresources to {0}".format(dst_path)) install_requires = [] dependency_links = [] @@ -363,8 +260,7 @@ setup( ] }, cmdclass={ - "install": OoniInstall, - "create_ooniresources": CreateOoniResources + "install": OoniInstall }, classifiers=( "Development Status :: 5 - Production/Stable",