[tor-commits] [ooni-probe/master] Various bugfixes and stability fixes.
art at torproject.org
art at torproject.org
Mon Sep 19 12:14:24 UTC 2016
commit 42dde01be494c7902ad6860877e067ffb37f7513
Author: Arturo Filastò <arturo at filasto.net>
Date: Fri Aug 5 16:29:07 2016 +0200
Various bugfixes and stability fixes.
---
Vagrantfile | 30 +++++++++++++-----------------
ooni/agent/scheduler.py | 3 +++
ooni/deck/deck.py | 13 +++++++------
ooni/measurements.py | 6 +++++-
ooni/settings.py | 4 ++--
ooni/ui/cli.py | 19 -------------------
ooni/ui/web/server.py | 2 +-
setup.py | 2 +-
8 files changed, 32 insertions(+), 47 deletions(-)
diff --git a/Vagrantfile b/Vagrantfile
index 0e75d55..71f7c7a 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -17,6 +17,12 @@ apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libff
cd /data/ooni-probe
python setup.py install
+
+echo "To run ooniprobe:"
+echo "1. vagrant ssh probe"
+echo "2. ooniprobe-agent start"
+echo "3. connect to the web UI on your host machine at http://localhost:8042/"
+
SCRIPT
$setup_oonibackend = <<SCRIPT
@@ -47,6 +53,13 @@ cp oonib.conf.example /etc/oonibackend.conf
echo "Installing ooni-backend"
python setup.py install
+echo ""
+echo "To run oonibackend:"
+echo "1. vagrant ssh backend"
+echo "2. vi /etc/oonibackend.conf # possibly"
+echo "3. cd /data/ooni-backend"
+echo "4. sudo ./bin/oonib -c /etc/oonibackend.conf"
+
SCRIPT
Vagrant.configure("2") do |config|
@@ -68,21 +81,4 @@ Vagrant.configure("2") do |config|
end
end
- if File.directory?("../ooni-backend")
- config.vm.provision "shell", inline: <<-EOF
- echo "To run oonibackend:"
- echo "1. vagrant ssh backend"
- echo "2. vi /etc/oonibackend.conf # possibly"
- echo "3. cd /data/ooni-backend"
- echo "4. sudo ./bin/oonib -c /etc/oonibackend.conf"
- EOF
- end
-
- config.vm.provision "shell", inline: <<-EOF
- echo "To run ooniprobe:"
- echo "1. vagrant ssh probe"
- echo "2. oonideckgen"
- echo "3. ooniprobe -i deck-*/*.deck"
- EOF
-
end
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 74a1688..3389db1 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -70,6 +70,7 @@ class ScheduledTask(object):
@property
def last_run(self):
+ self._last_run.restat(False)
if not self._last_run.exists():
return datetime.fromtimestamp(0)
with self._last_run.open('r') as in_file:
@@ -208,6 +209,8 @@ def run_system_tasks(no_input_store=False):
log.debug("Running task {0}".format(task.identifier))
try:
yield task.run()
+ except DidNotRun:
+ log.debug("Did not run {0}".format(task.identifier))
except Exception as exc:
log.err("Failed to run task {0}".format(task.identifier))
log.exception(exc)
diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 75d6366..037c5d6 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -259,8 +259,9 @@ class NGDeck(object):
try:
yield task.setup()
except InputNotFound:
- log.msg("Skipping this task because the input cannot be found")
- self._skip = True
+ log.msg("Skipping the task {0} because the input cannot be "
+ "found".format(task.id))
+ task.skip = True
self._is_setup = True
@defer.inlineCallbacks
@@ -271,8 +272,8 @@ class NGDeck(object):
yield director.start_tor()
yield self.query_bouncer()
for task in self._tasks:
- if task._skip is True:
- log.msg("Skipping running {0}".format(task.name))
+ if task.skip is True:
+ log.msg("Skipping running {0}".format(task.id))
continue
if task.type == "ooni":
yield self._run_ooni_task(task, director)
@@ -294,9 +295,9 @@ class DeckTask(object):
self.cwd = cwd
self.data = deepcopy(data)
- self._skip = False
+ self.skip = False
- self.id = ""
+ self.id = "invalid"
self.type = None
self.metadata = {}
diff --git a/ooni/measurements.py b/ooni/measurements.py
index cdf6b14..6d90c1b 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -3,6 +3,7 @@ import json
import signal
from twisted.python.filepath import FilePath
+from ooni.utils import log
from ooni.settings import config
class MeasurementInProgress(Exception):
@@ -118,7 +119,10 @@ def list_measurements():
measurements = []
measurement_path = FilePath(config.measurements_directory)
for measurement_id in measurement_path.listdir():
- measurements.append(get_measurement(measurement_id))
+ try:
+ measurements.append(get_measurement(measurement_id))
+ except:
+ log.err("Failed to get metadata for measurement {0}".format(measurement_id))
return measurements
if __name__ == "__main__":
diff --git a/ooni/settings.py b/ooni/settings.py
index e7174e2..16de94d 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -9,9 +9,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
from os.path import abspath, expanduser
-from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol
from ooni.utils import Storage, log, get_ooni_root
-from ooni import errors
CONFIG_FILE_TEMPLATE = """\
# This is the configuration file for OONIProbe
@@ -380,6 +378,7 @@ class OConfig(object):
self.log_incoherences(incoherent)
def log_incoherences(self, incoherences):
+ from ooni import errors
if len(incoherences) > 0:
if len(incoherences) > 1:
incoherent_pretty = ", ".join(incoherences[:-1]) + ' and ' + incoherences[-1]
@@ -393,6 +392,7 @@ class OConfig(object):
"""
Called only when we must start tor by director.start
"""
+ from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol
incoherent = []
if not self.advanced.start_tor:
if self.tor.socks_port is None:
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 65e4a0f..61f254b 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -74,15 +74,6 @@ class Options(usage.Options):
def __init__(self):
usage.Options.__init__(self)
- def getUsage(self, width=None):
- return super(Options, self).getUsage(width) + """
-To get started you may want to run:
-
-$ oonideckgen
-
-This will tell you how to run ooniprobe :)
-"""
-
def opt_spew(self):
"""
Print an insanely verbose log of everything that happens.
@@ -109,9 +100,6 @@ This will tell you how to run ooniprobe :)
def parseOptions():
- print "WARNING: running ooniprobe involves some risk that varies greatly"
- print " from country to country. You should be aware of this when"
- print " running the tool. Read more about this in the manpage or README."
cmd_line_options = Options()
if len(sys.argv) == 1:
cmd_line_options.getUsage()
@@ -182,13 +170,6 @@ def director_startup_other_failures(failure):
def initializeOoniprobe(global_options):
- print("""
- _ _ _
- __ _ _ _ ___ ___| |_(_)_ _ __ _ __| |
- / _` | '_/ -_) -_) _| | ' \/ _` (_-<_|
- \__, |_| \___\___|\__|_|_||_\__, /__(_)
- |___/ |___/ )
- """)
print("It looks like this is the first time you are running ooniprobe")
print("Please take a minute to read through the informed consent documentation and "
"understand what are the risks associated with running ooniprobe.")
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index bdafe75..ee33fa2 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -196,7 +196,7 @@ class WebUIAPI(object):
}
def handle_director_event(self, event):
- log.msg("Handling event {0}".format(event.type))
+ log.debug("Handling event {0}".format(event.type))
self.director_event_poller.notify(event)
def director_started(self, _):
diff --git a/setup.py b/setup.py
index abbbf21..7612b39 100644
--- a/setup.py
+++ b/setup.py
@@ -188,7 +188,7 @@ class OoniInstall(install):
if is_lepidopter():
self.update_lepidopter_config()
-setup_requires = ['twisted']
+setup_requires = ['twisted', 'pyyaml']
install_requires = []
dependency_links = []
data_files = []
More information about the tor-commits
mailing list