commit 42dde01be494c7902ad6860877e067ffb37f7513 Author: Arturo Filastò arturo@filasto.net Date: Fri Aug 5 16:29:07 2016 +0200
Various bugfixes and stability fixes. --- Vagrantfile | 30 +++++++++++++----------------- ooni/agent/scheduler.py | 3 +++ ooni/deck/deck.py | 13 +++++++------ ooni/measurements.py | 6 +++++- ooni/settings.py | 4 ++-- ooni/ui/cli.py | 19 ------------------- ooni/ui/web/server.py | 2 +- setup.py | 2 +- 8 files changed, 32 insertions(+), 47 deletions(-)
diff --git a/Vagrantfile b/Vagrantfile index 0e75d55..71f7c7a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -17,6 +17,12 @@ apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libff cd /data/ooni-probe python setup.py install
+ +echo "To run ooniprobe:" +echo "1. vagrant ssh probe" +echo "2. ooniprobe-agent start" +echo "3. connect to the web UI on your host machine at http://localhost:8042/" + SCRIPT
$setup_oonibackend = <<SCRIPT @@ -47,6 +53,13 @@ cp oonib.conf.example /etc/oonibackend.conf echo "Installing ooni-backend" python setup.py install
+echo "" +echo "To run oonibackend:" +echo "1. vagrant ssh backend" +echo "2. vi /etc/oonibackend.conf # possibly" +echo "3. cd /data/ooni-backend" +echo "4. sudo ./bin/oonib -c /etc/oonibackend.conf" + SCRIPT
Vagrant.configure("2") do |config| @@ -68,21 +81,4 @@ Vagrant.configure("2") do |config| end end
- if File.directory?("../ooni-backend") - config.vm.provision "shell", inline: <<-EOF - echo "To run oonibackend:" - echo "1. vagrant ssh backend" - echo "2. vi /etc/oonibackend.conf # possibly" - echo "3. cd /data/ooni-backend" - echo "4. sudo ./bin/oonib -c /etc/oonibackend.conf" - EOF - end - - config.vm.provision "shell", inline: <<-EOF - echo "To run ooniprobe:" - echo "1. vagrant ssh probe" - echo "2. oonideckgen" - echo "3. ooniprobe -i deck-*/*.deck" - EOF - end diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 74a1688..3389db1 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -70,6 +70,7 @@ class ScheduledTask(object):
@property def last_run(self): + self._last_run.restat(False) if not self._last_run.exists(): return datetime.fromtimestamp(0) with self._last_run.open('r') as in_file: @@ -208,6 +209,8 @@ def run_system_tasks(no_input_store=False): log.debug("Running task {0}".format(task.identifier)) try: yield task.run() + except DidNotRun: + log.debug("Did not run {0}".format(task.identifier)) except Exception as exc: log.err("Failed to run task {0}".format(task.identifier)) log.exception(exc) diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index 75d6366..037c5d6 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -259,8 +259,9 @@ class NGDeck(object): try: yield task.setup() except InputNotFound: - log.msg("Skipping this task because the input cannot be found") - self._skip = True + log.msg("Skipping the task {0} because the input cannot be " + "found".format(task.id)) + task.skip = True self._is_setup = True
@defer.inlineCallbacks @@ -271,8 +272,8 @@ class NGDeck(object): yield director.start_tor() yield self.query_bouncer() for task in self._tasks: - if task._skip is True: - log.msg("Skipping running {0}".format(task.name)) + if task.skip is True: + log.msg("Skipping running {0}".format(task.id)) continue if task.type == "ooni": yield self._run_ooni_task(task, director) @@ -294,9 +295,9 @@ class DeckTask(object): self.cwd = cwd self.data = deepcopy(data)
- self._skip = False + self.skip = False
- self.id = "" + self.id = "invalid"
self.type = None self.metadata = {} diff --git a/ooni/measurements.py b/ooni/measurements.py index cdf6b14..6d90c1b 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -3,6 +3,7 @@ import json import signal
from twisted.python.filepath import FilePath +from ooni.utils import log from ooni.settings import config
class MeasurementInProgress(Exception): @@ -118,7 +119,10 @@ def list_measurements(): measurements = [] measurement_path = FilePath(config.measurements_directory) for measurement_id in measurement_path.listdir(): - measurements.append(get_measurement(measurement_id)) + try: + measurements.append(get_measurement(measurement_id)) + except: + log.err("Failed to get metadata for measurement {0}".format(measurement_id)) return measurements
if __name__ == "__main__": diff --git a/ooni/settings.py b/ooni/settings.py index e7174e2..16de94d 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -9,9 +9,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
from os.path import abspath, expanduser
-from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol from ooni.utils import Storage, log, get_ooni_root -from ooni import errors
CONFIG_FILE_TEMPLATE = """\ # This is the configuration file for OONIProbe @@ -380,6 +378,7 @@ class OConfig(object): self.log_incoherences(incoherent)
def log_incoherences(self, incoherences): + from ooni import errors if len(incoherences) > 0: if len(incoherences) > 1: incoherent_pretty = ", ".join(incoherences[:-1]) + ' and ' + incoherences[-1] @@ -393,6 +392,7 @@ class OConfig(object): """ Called only when we must start tor by director.start """ + from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol incoherent = [] if not self.advanced.start_tor: if self.tor.socks_port is None: diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 65e4a0f..61f254b 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -74,15 +74,6 @@ class Options(usage.Options): def __init__(self): usage.Options.__init__(self)
- def getUsage(self, width=None): - return super(Options, self).getUsage(width) + """ -To get started you may want to run: - -$ oonideckgen - -This will tell you how to run ooniprobe :) -""" - def opt_spew(self): """ Print an insanely verbose log of everything that happens. @@ -109,9 +100,6 @@ This will tell you how to run ooniprobe :)
def parseOptions(): - print "WARNING: running ooniprobe involves some risk that varies greatly" - print " from country to country. You should be aware of this when" - print " running the tool. Read more about this in the manpage or README." cmd_line_options = Options() if len(sys.argv) == 1: cmd_line_options.getUsage() @@ -182,13 +170,6 @@ def director_startup_other_failures(failure):
def initializeOoniprobe(global_options): - print(""" - _ _ _ - __ _ _ _ ___ ___| |_(_)_ _ __ _ __| | - / _` | '_/ -_) -_) _| | ' / _` (_-<_| - __, |_| ______|__|_|_||___, /__(_) - |___/ |___/ ) - """) print("It looks like this is the first time you are running ooniprobe") print("Please take a minute to read through the informed consent documentation and " "understand what are the risks associated with running ooniprobe.") diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index bdafe75..ee33fa2 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -196,7 +196,7 @@ class WebUIAPI(object): }
def handle_director_event(self, event): - log.msg("Handling event {0}".format(event.type)) + log.debug("Handling event {0}".format(event.type)) self.director_event_poller.notify(event)
def director_started(self, _): diff --git a/setup.py b/setup.py index abbbf21..7612b39 100644 --- a/setup.py +++ b/setup.py @@ -188,7 +188,7 @@ class OoniInstall(install): if is_lepidopter(): self.update_lepidopter_config()
-setup_requires = ['twisted'] +setup_requires = ['twisted', 'pyyaml'] install_requires = [] dependency_links = [] data_files = []