September 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Add commands for managing the lifecycle of the ooniprobe-agent
by art＠torproject.org 19 Sep '16

19 Sep '16

commit fe64930f836dccf76a87b9e9bcfe655bada49f90 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 29 16:47:32 2016 +0200 Add commands for managing the lifecycle of the ooniprobe-agent * Add command to start the web-ui from ooniprobe * Improvements to the measurements API * Add klein dependency --- ooni/agent/agent.py | 4 +- ooni/measurements.py | 93 ++++++++++++++++++++++------ ooni/reporter.py | 13 +++- ooni/scripts/ooniprobe.py | 15 ++++- ooni/scripts/ooniprobe_agent.py | 130 +++++++++++++++++++++++++++++++++++++++- ooni/ui/cli.py | 5 +- ooni/ui/web/client/index.html | 2 +- ooni/ui/web/server.py | 53 +++++----------- requirements.txt | 1 + 9 files changed, 247 insertions(+), 69 deletions(-) diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py index f5322ed..ff9a2dd 100644 --- a/ooni/agent/agent.py +++ b/ooni/agent/agent.py @@ -7,7 +7,7 @@ from ooni.ui.web.web import WebUIService from ooni.agent.scheduler import SchedulerService class AgentService(service.MultiService): - def __init__(self): + def __init__(self, web_ui_port): service.MultiService.__init__(self) director = Director() @@ -15,7 +15,7 @@ class AgentService(service.MultiService): config.initialize_ooni_home() config.read_config_file() - self.web_ui_service = WebUIService(director) + self.web_ui_service = WebUIService(director, web_ui_port) self.web_ui_service.setServiceParent(self) self.scheduler_service = SchedulerService(director) diff --git a/ooni/measurements.py b/ooni/measurements.py index f830a99..50efd87 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -1,10 +1,14 @@ +import os import json +import signal + from twisted.python.filepath import FilePath from ooni.settings import config class Process(): supported_tests = [ - "web_connectivity" + "web_connectivity", + "http_requests" ] @staticmethod def web_connectivity(entry): @@ -15,6 +19,22 @@ class Process(): result['url'] = entry['input'] return result + @staticmethod + def http_requests(entry): + result = {} + test_keys = entry['test_keys'] + anomaly = ( + test_keys['body_length_match'] and + test_keys['headers_match'] and + ( + test_keys['control_failure'] != + test_keys['experiment_failure'] + ) + ) + result['anomaly'] = anomaly + result['url'] = entry['input'] + return result + def generate_summary(input_file, output_file): results = {} with open(input_file) as in_file: @@ -34,29 +54,64 @@ def generate_summary(input_file, output_file): with open(output_file, "w") as fw: json.dump(results, fw) +class MeasurementNotFound(Exception): + pass + +def get_measurement(measurement_id): + measurement_path = FilePath(config.measurements_directory) + measurement = measurement_path.child(measurement_id) + if not measurement.exists(): + raise MeasurementNotFound + + running = False + completed = True + keep = False + if measurement.child("measurement.njson.progress").exists(): + completed = False + # XXX this is done quite often around the code, probably should + # be moved into some utility function. + pid = measurement.child("running.pid").open("r").read() + pid = int(pid) + try: + os.kill(pid, signal.SIG_DFL) + running = True + except OSError: + pass + + if measurement.child("keep").exists(): + keep = True + test_start_time, country_code, asn, test_name = \ + measurement_id.split("-")[:4] + return { + "test_name": test_name, + "country_code": country_code, + "asn": asn, + "test_start_time": test_start_time, + "id": measurement_id, + "completed": completed, + "keep": keep, + "running": running + } + + +def get_summary(measurement_id): + measurement_path = FilePath(config.measurements_directory) + measurement = measurement_path.child(measurement_id) + summary = measurement.child("summary.json") + if not summary.exists(): + generate_summary( + measurement.child("measurement.njson").path, + summary.path + ) + + with summary.open("r") as f: + return json.load(f) def list_measurements(): measurements = [] measurement_path = FilePath(config.measurements_directory) for measurement_id in measurement_path.listdir(): - measurement = measurement_path.child(measurement_id) - completed = True - keep = False - if measurement.child("measurement.njson.progress").exists(): - completed = False - if measurement.child("keep").exists(): - keep = True - test_start_time, country_code, asn, test_name = \ - measurement_id.split("-")[:4] - measurements.append({ - "test_name": test_name, - "country_code": country_code, - "asn": asn, - "test_start_time": test_start_time, - "id": measurement_id, - "completed": completed, - "keep": keep - }) + measurements.append(get_measurement(measurement_id)) return measurements if __name__ == "__main__": diff --git a/ooni/reporter.py b/ooni/reporter.py index cf5341d..cfdce08 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -6,7 +6,6 @@ import os from copy import deepcopy from datetime import datetime -from contextlib import contextmanager from yaml.representer import SafeRepresenter from yaml.emitter import Emitter @@ -36,6 +35,7 @@ from ooni.utils import generate_filename from ooni.settings import config from ooni.tasks import ReportEntry +from ooni.measurements import list_measurements def createPacketReport(packet_list): @@ -413,9 +413,10 @@ class OONIBReportLog(object): @defer.inlineCallbacks def get_report_log_entries(self): entries = [] - for measurement_id in self.measurement_dir.listdir(): + for measurement in list_measurements(): try: - entry = yield self.get_report_log(measurement_id) + entry = yield self.get_report_log(measurement['id']) + entry['completed'] = measurement['completed'] entries.append(entry) except NoReportLog: continue @@ -453,6 +454,9 @@ class OONIBReportLog(object): incomplete_reports = [] all_entries = yield self.get_report_log_entries() for entry in all_entries[:]: + # This means that the measurement itself is incomplete + if entry['completed'] is False: + continue if entry['status'] in ('created',): try: os.kill(entry['pid'], 0) @@ -486,6 +490,9 @@ class OONIBReportLog(object): to_upload_reports = [] all_entries = yield self.get_report_log_entries() for entry in all_entries[:]: + # This means that the measurement itself is incomplete + if entry['completed'] is False: + continue if entry['status'] in ('creation-failed', 'not-created'): to_upload_reports.append( (entry['measurements_path'], entry) diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py index 24493da..f5d5b59 100644 --- a/ooni/scripts/ooniprobe.py +++ b/ooni/scripts/ooniprobe.py @@ -1,5 +1,8 @@ #!/usr/bin/env python -from twisted.internet import task +import webbrowser +from multiprocessing import Process + +from twisted.internet import task, defer def ooniprobe(reactor): from ooni.ui.cli import runWithDaemonDirector, runWithDirector @@ -9,6 +12,16 @@ def ooniprobe(reactor): check_incoherences=True) if global_options['queue']: return runWithDaemonDirector(global_options) + elif global_options['web-ui']: + from ooni.scripts.ooniprobe_agent import WEB_UI_URL + from ooni.scripts.ooniprobe_agent import status_agent, start_agent + if status_agent() != 0: + p = Process(target=start_agent) + p.start() + p.join() + print("Started ooniprobe-agent") + webbrowser.open_new(WEB_UI_URL) + return defer.succeed(None) else: return runWithDirector(global_options) diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index 7833308..3f8efc8 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -1,22 +1,58 @@ +from __future__ import print_function + +import os +import time +import signal + from twisted.scripts import twistd from twisted.python import usage +from ooni.settings import config from ooni.agent.agent import AgentService +WEB_UI_PORT = 8842 +WEB_UI_URL = "http://127.0.0.1:{0}".format(WEB_UI_PORT) + class StartOoniprobeAgentPlugin: tapname = "ooniprobe" def makeService(self, so): - return AgentService() + return AgentService(WEB_UI_PORT) class OoniprobeTwistdConfig(twistd.ServerOptions): subCommands = [ ("StartOoniprobeAgent", None, usage.Options, "ooniprobe agent") ] -def run(): - twistd_args = ["--nodaemon"] +class StartOptions(usage.Options): + pass + +class StopOptions(usage.Options): + pass + +class StatusOptions(usage.Options): + pass + +class RunOptions(usage.Options): + pass + +class AgentOptions(usage.Options): + subCommands = [ + ['start', None, StartOptions, "Start the ooniprobe-agent in the " + "background"], + ['stop', None, StopOptions, "Stop the ooniprobe-agent"], + ['status', None, StatusOptions, "Show status of the ooniprobe-agent"], + ['run', None, RunOptions, "Run the ooniprobe-agent in the foreground"] + ] + def postOptions(self): + self.twistd_args = [] + +def start_agent(options=None): + os.chdir(config.ooni_home) + twistd_args = [] twistd_config = OoniprobeTwistdConfig() + if options is not None: + twistd_args.extend(options.twistd_args) twistd_args.append("StartOoniprobeAgent") try: twistd_config.parseOptions(twistd_args) @@ -25,8 +61,96 @@ def run(): twistd_config.loadedPlugins = { "StartOoniprobeAgent": StartOoniprobeAgentPlugin() } + print("Starting ooniprobe agent.") + print("To view the GUI go to %s" % WEB_UI_URL) twistd.runApp(twistd_config) return 0 +def status_agent(): + pidfile = os.path.join( + config.ooni_home, + 'twistd.pid' + ) + if not os.path.exists(pidfile): + print("ooniprobe-agent is NOT running") + return 1 + pid = open(pidfile, "r").read() + pid = int(pid) + try: + os.kill(pid, signal.SIG_DFL) + except OSError, oserr: + if oserr.errno == 3: + print("ooniprobe-agent is NOT running") + return 1 + print("ooniprobe-agent is running") + return 0 + +def stop_agent(): + # This function is borrowed from tahoe + pidfile = os.path.join( + config.ooni_home, + 'twistd.pid' + ) + if not os.path.exists(pidfile): + print("It seems like ooniprobe-agent is not running") + return 2 + pid = open(pidfile, "r").read() + pid = int(pid) + try: + os.kill(pid, signal.SIGKILL) + except OSError, oserr: + if oserr.errno == 3: + print("No process was running. Cleaning up.") + # the process didn't exist, so wipe the pid file + os.remove(pidfile) + return 2 + else: + raise + try: + os.remove(pidfile) + except EnvironmentError: + pass + start = time.time() + time.sleep(0.1) + wait = 40 + first_time = True + while True: + # poll once per second until we see the process is no longer running + try: + os.kill(pid, 0) + except OSError: + print("process %d is dead" % pid) + return + wait -= 1 + if wait < 0: + if first_time: + print("It looks like pid %d is still running " + "after %d seconds" % (pid, (time.time() - start))) + print("I will keep watching it until you interrupt me.") + wait = 10 + first_time = False + else: + print("pid %d still running after %d seconds" % \ + (pid, (time.time() - start))) + wait = 10 + time.sleep(1) + # we define rc=1 to mean "I think something is still running, sorry" + return 1 + +def run(): + options = AgentOptions() + options.parseOptions() + + if options.subCommand == "run": + options.twistd_args += ("--nodaemon",) + + if options.subCommand == "stop": + return stop_agent() + + if options.subCommand == "status": + return status_agent() + + return start_agent(options) + if __name__ == "__main__": run() diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 2b5d844..3eccf9a 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -29,7 +29,8 @@ class Options(usage.Options): ["no-geoip", "g", "Disable geoip lookup on start"], ["list", "s", "List the currently installed ooniprobe " "nettests"], - ["verbose", "v", "Show more verbose information"] + ["verbose", "v", "Show more verbose information"], + ["web-ui", "w", "Start the web UI"] ] optParameters = [ @@ -96,7 +97,7 @@ This will tell you how to run ooniprobe :) sys.exit(0) def parseArgs(self, *args): - if self['testdeck'] or self['list']: + if self['testdeck'] or self['list'] or self['web-ui']: return try: self['test_file'] = args[0] diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index 7ba33fb..ebed106 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?afba2f26969b4c8f00ec"></script></body> + <script type="text/javascript" src="app.bundle.js?2777836bc218e75c3be5"></script></body> </html> diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 9d98e62..e1a6398 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -20,7 +20,8 @@ from ooni.deck import NGDeck from ooni.settings import config from ooni.utils import log from ooni.director import DirectorEvent -from ooni.measurements import generate_summary +from ooni.measurements import get_summary, get_measurement +from ooni.measurements import list_measurements, MeasurementNotFound from ooni.geoip import probe_ip config.advanced.debug = True @@ -311,61 +312,40 @@ class WebUIAPI(object): @app.route('/api/measurement', methods=["GET"]) @xsrf_protect(check=False) def api_measurement_list(self, request): - measurements = [] - for measurement_id in self.measurement_path.listdir(): - measurement = self.measurement_path.child(measurement_id) - completed = True - if measurement.child("measurement.njson.progress").exists(): - completed = False - test_start_time, country_code, asn, test_name = \ - measurement_id.split("-")[:4] - measurements.append({ - "test_name": test_name, - "country_code": country_code, - "asn": asn, - "test_start_time": test_start_time, - "id": measurement_id, - "completed": completed - }) + measurements = list_measurements() return self.render_json({"measurements": measurements}, request) @app.route('/api/measurement/<string:measurement_id>', methods=["GET"]) @xsrf_protect(check=False) def api_measurement_summary(self, request, measurement_id): try: - measurement_dir = self.measurement_path.child(measurement_id) + measurement = get_measurement(measurement_id) except InsecurePath: raise WebUIError(500, "invalid measurement id") + except MeasurementNotFound: + raise WebUIError(404, "measurement not found") - if measurement_dir.child("measurements.njson.progress").exists(): - raise WebUIError(400, "measurement in progress") - - if not measurement_dir.child("summary.json").exists(): - # XXX we can perhaps remove this. - generate_summary( - measurement_dir.child("measurements.njson").path, - measurement_dir.child("summary.json").path - ) + if measurement['completed'] is False: raise WebUIError(400, "measurement in progress") - summary = measurement_dir.child("summary.json") - with summary.open("r") as f: - r = json.load(f) - - return self.render_json(r, request) + summary = get_summary(measurement_id) + return self.render_json(summary, request) @app.route('/api/measurement/<string:measurement_id>', methods=["DELETE"]) @xsrf_protect(check=True) def api_measurement_delete(self, request, measurement_id): try: - measurement_dir = self.measurement_path.child(measurement_id) + measurement = get_measurement(measurement_id) except InsecurePath: raise WebUIError(500, "invalid measurement id") + except MeasurementNotFound: + raise WebUIError(404, "measurement not found") - if measurement_dir.child("measurements.njson.progress").exists(): - raise WebUIError(400, "measurement in progress") + if measurement['running'] is True: + raise WebUIError(400, "Measurement running") try: + measurement_dir = self.measurement_path.child(measurement_id) measurement_dir.remove() except: raise WebUIError(400, "Failed to delete report") @@ -380,9 +360,6 @@ class WebUIAPI(object): except InsecurePath: raise WebUIError(500, "invalid measurement id") - if measurement_dir.child("measurements.njson.progress").exists(): - raise WebUIError(400, "measurement in progress") - summary = measurement_dir.child("keep") with summary.open("w+") as f: pass diff --git a/requirements.txt b/requirements.txt index 410566e..c05ebcb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ service-identity pydumbnet zope.interface certifi +klein

1 0

[ooni-probe/master] Fix unittests of oonireport
by art＠torproject.org 19 Sep '16

19 Sep '16

commit e490533b48c7c62da5e6d28832434dbc51fb85f0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 29 23:49:32 2016 +0200 Fix unittests of oonireport --- ooni/tests/test_oonireport.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/ooni/tests/test_oonireport.py b/ooni/tests/test_oonireport.py index 4769682..fe5e420 100644 --- a/ooni/tests/test_oonireport.py +++ b/ooni/tests/test_oonireport.py @@ -1,5 +1,3 @@ -import yaml - from mock import patch, MagicMock from twisted.internet import defer @@ -9,15 +7,6 @@ mock_tor_check = MagicMock(return_value=True) class TestOONIReport(ConfigTestCase): - def _create_reporting_yaml(self, filename): - from ooni.settings import config - with open(config.report_log_file, 'w+') as f: - yaml.dump({ - filename: { - "collector": "httpo://thirteenchars123.onion" - } - }, f) - def _write_dummy_report(self, filename): from ooni.reporter import YAMLReporter from .test_reporter import test_details @@ -65,10 +54,9 @@ class TestOONIReport(ConfigTestCase): mock_oonib_report_log_i.closed.return_value = defer.succeed(True) report_name = "dummy_report.yaml" - self._create_reporting_yaml(report_name) self._write_dummy_report(report_name) from ooni.scripts import oonireport - d = oonireport.upload(report_name) + d = oonireport.upload(report_name, collector='httpo://thirteenchars123.onion') @d.addCallback def cb(result): mock_oonib_reporter_i.writeReportEntry.assert_called_with( @@ -90,14 +78,13 @@ class TestOONIReport(ConfigTestCase): mock_oonib_report_log_i = mock_oonib_report_log.return_value mock_oonib_report_log_i.created.return_value = defer.succeed(True) mock_oonib_report_log_i.closed.return_value = defer.succeed(True) - mock_oonib_report_log_i.reports_to_upload = [("dummy_report.yaml", None)] + mock_oonib_report_log_i.get_to_upload.return_value = defer.succeed([("dummy_report.yaml", {'measurement_id': 'XX'})]) report_name = "dummy_report.yaml" - self._create_reporting_yaml(report_name) self._write_dummy_report(report_name) from ooni.scripts import oonireport - d = oonireport.upload_all() + d = oonireport.upload_all(collector='httpo://thirteenchars123.onion') @d.addCallback def cb(result): mock_oonib_reporter_i.writeReportEntry.assert_called_with(

1 0

[ooni-probe/master] Add support for loglevels.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 5a6a483bd0e1f1443f170955085543ef11aac34f Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Jul 27 23:47:45 2016 +0200 Add support for loglevels. Small refactor of the logging facilities Note: now all messages printed with print() are not output to stdout by default and all messages that are not printed via the ooni logging infrastructure that are not critical are also ignored. This is useful to avoid cluttering the CLI with too many messages and also to reduce the size of the logs. This implements: #502 --- ooni/nettest.py | 8 +-- ooni/settings.py | 25 ++++++--- ooni/ui/cli.py | 12 ++-- ooni/utils/log.py | 160 +++++++++++++++++++++++++++++++++++------------------- 4 files changed, 133 insertions(+), 72 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 566c391..1978c68 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -148,7 +148,7 @@ def usageOptionsFactory(test_name, test_version): """ Display the net_test version and exit. """ - print "{} version: {}".format(test_name, test_version) + log.msg("{} version: {}".format(test_name, test_version)) sys.exit(0) return UsageOptions @@ -513,13 +513,13 @@ class NetTest(object): def doneNetTest(self, result): if self.summary: - print "Summary for %s" % self.testDetails['test_name'] - print "------------" + "-"*len(self.testDetails['test_name']) + log.msg("Summary for %s" % self.testDetails['test_name']) + log.msg("------------" + "-"*len(self.testDetails['test_name'])) for test_class in self.uniqueClasses(): test_instance = test_class() test_instance.displaySummary(self.summary) if self.testDetails["report_id"]: - print "Report ID: %s" % self.testDetails["report_id"] + log.msg("Report ID: %s" % self.testDetails["report_id"]) def doneReport(self, report_results): """ diff --git a/ooni/settings.py b/ooni/settings.py index acb1895..194729a 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -9,7 +9,6 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from os.path import abspath, expanduser from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol -from ooni import geoip from ooni.utils import Storage, log, get_ooni_root from ooni import errors @@ -101,8 +100,6 @@ class OConfig(object): self.inputs_directory = os.path.join(self.ooni_home, 'inputs') self.scheduler_directory = os.path.join(self.ooni_home, 'scheduler') - if not os.path.exists(self.scheduler_directory): - os.mkdir(self.scheduler_directory) if self.advanced.decks_dir: self.decks_directory = self.advanced.decks_dir @@ -136,11 +133,25 @@ class OConfig(object): ooni_home = self.ooni_home if not os.path.isdir(ooni_home): - print "Ooni home directory does not exist." - print "Creating it in '%s'." % ooni_home + log.msg("Ooni home directory does not exist") + log.msg("Creating it in '%s'" % ooni_home) os.mkdir(ooni_home) - os.mkdir(self.inputs_directory) - os.mkdir(self.decks_directory) + + # also ensure the subdirectories exist + sub_directories = [ + self.inputs_directory, + self.decks_directory, + self.scheduler_directory, + self.measurements_directory, + self.resources_directory + ] + for path in sub_directories: + try: + os.mkdir(path) + except OSError as exc: + if exc.errno != 17: + raise + def _create_config_file(self): target_config_file = self.config_file diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 51ed3b9..2f5c467 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -339,9 +339,9 @@ def runWithDirector(global_options, create_input_store=True): director = Director() if global_options['list']: net_tests = [net_test for net_test in director.getNetTests().items()] - print "" - print "Installed nettests" - print "==================" + log.msg("") + log.msg("Installed nettests") + log.msg("==================") for net_test_id, net_test in net_tests: optList = [] for name, details in net_test['arguments'].items(): @@ -359,9 +359,9 @@ def runWithDirector(global_options, create_input_store=True): '\n\n' + ''.join(usage.docMakeChunks(optList)) ) - print desc - print "Note: Third party tests require an external "\ - "application to run properly." + map(log.msg, desc.split("\n")) + log.msg("Note: Third party tests require an external " + "application to run properly.") raise SystemExit(0) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index fd80594..c8a7360 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -2,11 +2,8 @@ import os import sys import codecs import logging -import traceback -from twisted.python import log as txlog -from twisted.python import util -from twisted.python.failure import Failure +from twisted.python import log as tw_log from twisted.python.logfile import DailyLogFile from ooni import otime @@ -15,7 +12,6 @@ from ooni import otime # IPv6 destination warnings": logging.getLogger("scapy.runtime").setLevel(logging.ERROR) - def log_encode(logmsg): """ I encode logmsg (a str or unicode) as printable ASCII. Each case @@ -38,17 +34,97 @@ def log_encode(logmsg): repr(logmsg))) -class LogWithNoPrefix(txlog.FileLogObserver): +levels = { + 'NONE': 9999, + 'CRITICAL': 50, + 'ERROR': 40, + 'WARNING': 30, + # This is the name twisted gives it + 'WARN': 30, + 'NOTICE': 25, + 'INFO': 20, + 'DEBUG': 10, +} + +class LogLevelObserver(tw_log.FileLogObserver): + def __init__(self, f, log_level=levels['INFO']): + tw_log.FileLogObserver.__init__(self, f) + self.log_level = log_level + + def should_emit(self, eventDict): + if eventDict['isError']: + level = levels['ERROR'] + elif 'log_level' in eventDict: + level = eventDict['log_level'] + else: + level = levels['INFO'] + + # To support twisted > 15.2 log_level argument + if hasattr(level, 'name'): + level = levels[level.name.upper()] + + source = 'unknown' + if 'source' in eventDict: + source = eventDict['source'] + + # Don't log messages not coming from OONI unless the configured log + # level is debug and unless they are really important. + if (source != 'ooni' and + level <= levels['WARN'] and + self.log_level >= levels['DEBUG']): + return False + + if level >= self.log_level: + return True + return False + def emit(self, eventDict): - text = txlog.textFromEventDict(eventDict) - if text is None: + if not self.should_emit(eventDict): return + tw_log.FileLogObserver.emit(self, eventDict) - util.untilConcludes(self.write, "%s\n" % text) - util.untilConcludes(self.flush) # Hoorj! +class StdoutStderrObserver(LogLevelObserver): + stderr = sys.stderr + def emit(self, eventDict): + if not self.should_emit(eventDict): + return + + text = tw_log.textFromEventDict(eventDict) + if eventDict['isError']: + self.stderr.write(text + "\n") + self.stderr.flush() + else: + self.write(text + "\n") + self.flush() class OONILogger(object): + def msg(self, msg, *arg, **kw): + text = log_encode(msg) + tw_log.msg(text, log_level=levels['INFO'], source="ooni") + + def debug(self, msg, *arg, **kw): + text = log_encode(msg) + tw_log.msg(text, log_level=levels['DEBUG'], source="ooni") + + def err(self, msg, *arg, **kw): + if isinstance(msg, str) or isinstance(msg, unicode): + text = "[!] " + log_encode(msg) + tw_log.msg(text, log_level=levels['ERROR'], source="ooni") + else: + tw_log.err(msg, source="ooni") + + def warn(self, *arg, **kw): + text = log_encode(msg) + tw_log.msg(text, log_level=levels['WARNING'], source="ooni") + + def exception(self, error): + """ + Error can either be an error message to print to stdout and to the logfile + or it can be a twisted.python.failure.Failure instance. + """ + tw_log.err(error, source="ooni") + def start(self, logfile=None, application_name="ooniprobe"): from ooni.settings import config @@ -57,18 +133,25 @@ class OONILogger(object): log_folder = os.path.dirname(logfile) log_filename = os.path.basename(logfile) + file_log_level = levels.get(config.basic.loglevel, + levels['INFO']) + stdout_log_level = levels['INFO'] + if config.advanced.debug: + stdout_log_level = levels['DEBUG'] daily_logfile = DailyLogFile(log_filename, log_folder) - txlog.msg("Starting %s on %s (%s UTC)" % (application_name, - otime.prettyDateNow(), - otime.prettyDateNowUTC())) + tw_log.msg("Starting %s on %s (%s UTC)" % (application_name, + otime.prettyDateNow(), + otime.prettyDateNowUTC())) - self.fileObserver = txlog.FileLogObserver(daily_logfile) - self.stdoutObserver = LogWithNoPrefix(sys.stdout) + self.fileObserver = LogLevelObserver(daily_logfile, + log_level=file_log_level) + self.stdoutObserver = StdoutStderrObserver(sys.stdout, + log_level=stdout_log_level) - txlog.startLoggingWithObserver(self.stdoutObserver.emit) - txlog.addObserver(self.fileObserver.emit) + tw_log.startLoggingWithObserver(self.fileObserver.emit) + tw_log.addObserver(self.stdoutObserver.emit) def stop(self): self.stdoutObserver.stop() @@ -76,42 +159,9 @@ class OONILogger(object): oonilogger = OONILogger() +start = oonilogger.start +stop = oonilogger.stop -def start(logfile=None, application_name="ooniprobe"): - oonilogger.start(logfile, application_name) - - -def stop(): - oonilogger.stop() - - -def msg(msg, *arg, **kw): - from ooni.settings import config - if config.logging: - print "%s" % log_encode(msg) - - -def debug(msg, *arg, **kw): - from ooni.settings import config - if config.advanced.debug and config.logging: - print "[D] %s" % log_encode(msg) - - -def err(msg, *arg, **kw): - from ooni.settings import config - if config.logging: - if isinstance(msg, Exception): - msg = "%s: %s" % (msg.__class__.__name__, msg) - print "[!] %s" % log_encode(msg) - - -def exception(error): - """ - Error can either be an error message to print to stdout and to the logfile - or it can be a twisted.python.failure.Failure instance. - """ - if isinstance(error, Failure): - error.printTraceback(sys.stdout) - else: - exc_type, exc_value, exc_traceback = sys.exc_info() - traceback.print_exception(exc_type, exc_value, exc_traceback) +msg = oonilogger.msg +debug = oonilogger.debug +err = oonilogger.err

1 0

[ooni-probe/master] Edit ooniprobe configuration file
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 921a561062c6fc9cba375c61e4638ac33100acbe Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 29 23:56:20 2016 +0200 Edit ooniprobe configuration file --- data/ooniprobe.conf.sample | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index cde4c00..f4c0812 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -5,6 +5,7 @@ basic: # Where OONIProbe should be writing it's log file logfile: /var/log/ooniprobe.log + loglevel: WARNING privacy: # Should we include the IP address of the probe in the report? includeip: false @@ -44,10 +45,6 @@ advanced: reporting_retries: 5 # How many reports to perform concurrently reporting_concurrency: 7 - oonid_api_port: 8042 - report_log_file: null - inputs_dir: null - decks_dir: null # If we should support communicating to plaintext backends (via HTTP) insecure_backend: false # The preferred backend type, can be one of onion, https or cloudfront

1 0

[ooni-probe/master] Forward ports from the vagrant file
by art＠torproject.org 19 Sep '16

19 Sep '16

commit cd18ae75cb2e63e05649adc166dd5c91d6c3fde1 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Jul 30 00:36:10 2016 +0200 Forward ports from the vagrant file --- Vagrantfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Vagrantfile b/Vagrantfile index 59b2751..cf73244 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -54,6 +54,7 @@ Vagrant.configure("2") do |config| config.vm.box = "debian/contrib-jessie64" config.vm.define "probe" do |probe| + probe.vm.network "forwarded_port", guest: 8842, host: 8042 probe.vm.synced_folder ".", "/data/ooni-probe" probe.vm.provision :shell, :inline => $setup_ooniprobe end

1 0

[ooni-probe/master] Implement deck store
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 97ed0fcbda310db18b2138f70024f08de5b523b9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 29 22:50:45 2016 +0200 Implement deck store * Write all runtime files to /var/lib/ooni * Other various fixes to runtime paths * Include the default deck and data files in distribution --- MANIFEST.in | 2 ++ data/decks/web.yaml | 22 +++++++++++++ ooni/agent/agent.py | 8 ----- ooni/agent/scheduler.py | 22 ++++++++++++- ooni/deck/deck.py | 15 +++++++-- ooni/deck/store.py | 35 +++++++++++++++++--- ooni/scripts/ooniprobe_agent.py | 17 +++++++--- ooni/settings.py | 72 ++++++++++++++++++++++++++++------------- ooni/ui/web/server.py | 8 +---- ooni/utils/log.py | 2 ++ 10 files changed, 153 insertions(+), 50 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 692c73f..485e834 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ +recursive-include ooni/ui/web/client * include README.rst ChangeLog.rst requirements.txt LICENSE +recursive-include data/decks * include data/oonideckgen.1 include data/ooniprobe.1 include data/oonireport.1 diff --git a/data/decks/web.yaml b/data/decks/web.yaml new file mode 100644 index 0000000..a81b8f8 --- /dev/null +++ b/data/decks/web.yaml @@ -0,0 +1,22 @@ +--- +name: Web related ooniprobe tests +description: This deck runs HTTP Header Field Manipulation, HTTP Invalid + Request and the Web Connectivity test +tasks: +- name: Runs the HTTP Header Field Manipulation test + ooni: + test_name: http_header_field_manipulation + +- name: Runs the HTTP Invalid Request Line test + ooni: + test_name: http_invalid_request_line + +- name: Runs the Web Connectivity Test + ooni: + test_name: web_connectivity + file: $citizenlab_global_urls + +- name: Runs the Web Connectivity Test + ooni: + test_name: web_connectivity + file: $citizenlab_${probe_cc}_urls diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py index ff9a2dd..c2e7e26 100644 --- a/ooni/agent/agent.py +++ b/ooni/agent/agent.py @@ -1,7 +1,6 @@ from twisted.application import service from ooni.director import Director from ooni.settings import config -from ooni.utils import log from ooni.ui.web.web import WebUIService from ooni.agent.scheduler import SchedulerService @@ -11,9 +10,6 @@ class AgentService(service.MultiService): service.MultiService.__init__(self) director = Director() - config.set_paths() - config.initialize_ooni_home() - config.read_config_file() self.web_ui_service = WebUIService(director, web_ui_port) self.web_ui_service.setServiceParent(self) @@ -24,9 +20,5 @@ class AgentService(service.MultiService): def startService(self): service.MultiService.startService(self) - log.start() - def stopService(self): service.MultiService.stopService(self) - - log.stop() diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 167a14a..43715e8 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -7,7 +7,7 @@ from twisted.python.filepath import FilePath from ooni.scripts import oonireport from ooni import resources from ooni.utils import log, SHORT_DATE -from ooni.deck.store import input_store +from ooni.deck.store import input_store, deck_store from ooni.settings import config from ooni.contrib import croniter from ooni.geoip import probe_ip @@ -116,6 +116,25 @@ class DeleteOldReports(ScheduledTask): log.debug("Deleting old report {0}".format(measurement["id"])) measurement_path.child(measurement['id']).remove() + +class RunDecks(ScheduledTask): + """ + This will run the decks that have been configured on the system as the + decks to run by default. + """ + schedule = '@daily' + identifier = 'run-decks' + + def __init__(self, director, schedule=None): + super(RunDecks, self).__init__(schedule) + self.director = director + + @defer.inlineCallbacks + def task(self): + for deck_id, deck in deck_store.list(): + yield deck.setup() + yield deck.run(self.director) + class SendHeartBeat(ScheduledTask): """ This task is used to send a heartbeat that the probe is still alive and @@ -193,6 +212,7 @@ class SchedulerService(service.MultiService): self.schedule(UpdateInputsAndResources()) self.schedule(UploadReports()) self.schedule(DeleteOldReports()) + self.schedule(RunDecks(self.director)) self._looping_call.start(self.interval) diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index 868bfb8..1b2300a 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -1,5 +1,6 @@ import os from copy import deepcopy +from string import Template import yaml from twisted.internet import defer @@ -10,7 +11,6 @@ from ooni.backend_client import BouncerClient, CollectorClient from ooni.backend_client import get_preferred_bouncer from ooni.deck.backend import lookup_collector_and_test_helpers from ooni.deck.legacy import convert_legacy_deck -from ooni.deck.store import input_store from ooni.geoip import probe_ip from ooni.nettest import NetTestLoader, nettest_to_path from ooni.measurements import generate_summary @@ -19,6 +19,7 @@ from ooni.utils import log, generate_filename def resolve_file_path(v, prepath=None): + from ooni.deck.store import input_store if v.startswith("$"): # This raises InputNotFound and we let it carry onto the caller return input_store.get(v[1:])["filepath"] @@ -248,8 +249,13 @@ class NGDeck(object): """ This method needs to be called before you are able to run a deck. """ + from ooni.deck.store import InputNotFound for task in self._tasks: - yield task.setup() + try: + yield task.setup() + except InputNotFound: + log.msg("Skipping this task because the input cannot be found") + self._skip = True self._is_setup = True @defer.inlineCallbacks @@ -365,7 +371,10 @@ class DeckTask(object): def _setup_ooni(self): yield probe_ip.lookup() for input_file in self.ooni['net_test_loader'].inputFiles: - file_path = resolve_file_path(input_file['filename'], self.cwd) + filename = Template(input_file['filename']).safe_substitute( + probe_cc=probe_ip.geodata['countrycode'].lower() + ) + file_path = resolve_file_path(filename, self.cwd) input_file['test_options'][input_file['key']] = file_path self.ooni['test_details'] = self.ooni['net_test_loader'].getTestDetails() self.id = generate_filename(self.ooni['test_details']) diff --git a/ooni/deck/store.py b/ooni/deck/store.py index 05c0b95..7c90204 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -5,13 +5,18 @@ from copy import deepcopy from twisted.internet import defer from twisted.python.filepath import FilePath +from ooni.deck.deck import NGDeck from ooni.otime import timestampNowISO8601UTC from ooni.resources import check_for_update from ooni.settings import config +from ooni.utils import log class InputNotFound(Exception): pass +class DeckNotFound(Exception): + pass + class InputStore(object): def __init__(self): self.path = FilePath(config.inputs_directory) @@ -117,12 +122,34 @@ class InputStore(object): class DeckStore(object): def __init__(self): self.path = FilePath(config.decks_directory) + self._cache = {} + self._cache_stale = True - def update(self): - pass + def list(self): + decks = [] + if self._cache_stale: + self._update_cache() + for deck_id, deck in self._cache.iteritems(): + decks.append((deck_id, deck)) + return decks - def get(self): - pass + def _update_cache(self): + for deck_path in self.path.listdir(): + if not deck_path.endswith('.yaml'): + continue + deck_id = deck_path[:-1*len('.yaml')] + deck = NGDeck( + deck_path=self.path.child(deck_path).path + ) + self._cache[deck_id] = deck + def get(self, deck_id): + if self._cache_stale: + self._update_cache() + try: + return self._cache[deck_id] + except KeyError: + raise DeckNotFound(deck_id) +deck_store = DeckStore() input_store = InputStore() diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py index 3f8efc8..30df9b9 100644 --- a/ooni/scripts/ooniprobe_agent.py +++ b/ooni/scripts/ooniprobe_agent.py @@ -7,6 +7,7 @@ import signal from twisted.scripts import twistd from twisted.python import usage +from ooni.utils import log from ooni.settings import config from ooni.agent.agent import AgentService @@ -48,8 +49,15 @@ class AgentOptions(usage.Options): self.twistd_args = [] def start_agent(options=None): - os.chdir(config.ooni_home) - twistd_args = [] + config.set_paths() + config.initialize_ooni_home() + config.read_config_file() + + os.chdir(config.running_path) + + # Since we are starting the logger below ourselves we make twistd log to + # a null log observer + twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull'] twistd_config = OoniprobeTwistdConfig() if options is not None: twistd_args.extend(options.twistd_args) @@ -63,12 +71,13 @@ def start_agent(options=None): } print("Starting ooniprobe agent.") print("To view the GUI go to %s" % WEB_UI_URL) + log.start() twistd.runApp(twistd_config) return 0 def status_agent(): pidfile = os.path.join( - config.ooni_home, + config.running_path, 'twistd.pid' ) if not os.path.exists(pidfile): @@ -88,7 +97,7 @@ def status_agent(): def stop_agent(): # This function is borrowed from tahoe pidfile = os.path.join( - config.ooni_home, + config.running_path, 'twistd.pid' ) if not os.path.exists(pidfile): diff --git a/ooni/settings.py b/ooni/settings.py index 194729a..5762919 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -1,4 +1,5 @@ import os +import sys import yaml import getpass from ConfigParser import SafeConfigParser @@ -12,7 +13,6 @@ from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol from ooni.utils import Storage, log, get_ooni_root from ooni import errors - class OConfig(object): _custom_home = None @@ -41,18 +41,56 @@ class OConfig(object): @property def var_lib_path(self): + if hasattr(sys, 'real_prefix'): + # We are in a virtualenv use the /usr/share in the virtualenv + return os.path.join( + os.path.abspath(sys.prefix), + 'var', 'lib', 'ooni' + ) var_lib_path = self.embedded_settings("directories", "var_lib") if var_lib_path: return os.path.abspath(var_lib_path) return "/var/lib/ooni" @property + def running_path(self): + """ + This is the directory used to store state application data. + It defaults to /var/lib/ooni, but if that is not writeable we will + use the ooni_home. + """ + var_lib_path = self.var_lib_path + if os.access(var_lib_path, os.W_OK): + return var_lib_path + return self.ooni_home + + @property def usr_share_path(self): + if hasattr(sys, 'real_prefix'): + # We are in a virtualenv use the /usr/share in the virtualenv + return os.path.join( + os.path.abspath(sys.prefix), + 'usr', 'share', 'ooni' + ) usr_share_path = self.embedded_settings("directories", "usr_share") if usr_share_path: return os.path.abspath(usr_share_path) return "/usr/share/ooni" + + @property + def etc_path(self): + if hasattr(sys, 'real_prefix'): + # We are in a virtualenv use the /usr/share in the virtualenv + return os.path.join( + os.path.abspath(sys.prefix), + 'usr', 'share', 'ooni' + ) + etc_path = self.embedded_settings("directories", "etc") + if etc_path: + return os.path.abspath(etc_path) + return "/etc" + @property def data_directory_candidates(self): dirs = [ @@ -93,28 +131,15 @@ class OConfig(object): def set_paths(self): self.nettest_directory = os.path.join(get_ooni_root(), 'nettests') + self.web_ui_directory = os.path.join(get_ooni_root(), 'web', 'client') - if self.advanced.inputs_dir: - self.inputs_directory = self.advanced.inputs_dir - else: - self.inputs_directory = os.path.join(self.ooni_home, 'inputs') - - self.scheduler_directory = os.path.join(self.ooni_home, 'scheduler') + self.inputs_directory = os.path.join(self.running_path, 'inputs') + self.scheduler_directory = os.path.join(self.running_path, 'scheduler') + self.decks_directory = os.path.join(self.running_path, 'decks') + self.resources_directory = os.path.join(self.running_path, 'resources') - if self.advanced.decks_dir: - self.decks_directory = self.advanced.decks_dir - else: - self.decks_directory = os.path.join(self.ooni_home, 'decks') - - self.measurements_directory = os.path.join(self.ooni_home, + self.measurements_directory = os.path.join(self.running_path, 'measurements') - self.resources_directory = os.path.join(self.ooni_home, - "resources") - if self.advanced.report_log_file: - self.report_log_file = self.advanced.report_log_file - else: - self.report_log_file = os.path.join(self.ooni_home, - 'reporting.yml') if self.global_options.get('configfile'): config_file = self.global_options['configfile'] @@ -123,8 +148,9 @@ class OConfig(object): self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf') if 'logfile' in self.basic: - self.basic.logfile = expanduser(self.basic.logfile.replace( - '~', '~'+self.current_user)) + self.basic.logfile = expanduser( + self.basic.logfile.replace('~', '~'+self.current_user) + ) def initialize_ooni_home(self, custom_home=None): if custom_home: @@ -147,7 +173,7 @@ class OConfig(object): ] for path in sub_directories: try: - os.mkdir(path) + os.makedirs(path) except OSError as exc: if exc.errno != 17: raise diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index e1a6398..74cb235 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -26,11 +26,6 @@ from ooni.geoip import probe_ip config.advanced.debug = True -def rpath(*path): - context = os.path.abspath(os.path.dirname(__file__)) - return os.path.join(context, *path) - - class WebUIError(Exception): def __init__(self, code, message): self.code = code @@ -392,5 +387,4 @@ class WebUIAPI(object): @app.route('/client/', branch=True) @xsrf_protect(check=False) def static(self, request): - path = rpath("client") - return static.File(path) + return static.File(config.web_ui_directory) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 982a353..1aba8dd 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -158,6 +158,8 @@ class OONILogger(object): self.fileObserver.stop() oonilogger = OONILogger() +# This is a mock of a LoggerObserverFactory to be supplied to twistd. +ooniloggerNull = lambda: lambda eventDict: None start = oonilogger.start stop = oonilogger.stop

1 0

[ooni-probe/master] Add missing modules to setup.py
by art＠torproject.org 19 Sep '16

19 Sep '16

commit f5c02d1593b37d5fd54ab7671b0914b9ad3a78ed Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Jul 30 00:26:29 2016 +0200 Add missing modules to setup.py --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index c1ae189..6940f66 100644 --- a/setup.py +++ b/setup.py @@ -307,6 +307,10 @@ packages = [ 'ooni', 'ooni.agent', 'ooni.common', + 'ooni.contrib', + 'ooni.contrib.dateutil', + 'ooni.contrib.dateutil.tz', + 'ooni.deck', 'ooni.kit', 'ooni.nettests', 'ooni.nettests.manipulation',

1 0

[ooni-probe/master] Convert the director status into a property so the IP and ASN get's updated
by art＠torproject.org 19 Sep '16

19 Sep '16

commit c7970d24f0285e08a57a3b362889f308dbcdbf3c Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Aug 1 19:02:23 2016 +0200 Convert the director status into a property so the IP and ASN get's updated * Make another call to resolveGeodata after we fetch the inputs (this should fix the issue @agrabeli reported in erroneous IP lookups) --- ooni/agent/scheduler.py | 1 + ooni/ui/web/server.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 0d8689c..a6d689f 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -84,6 +84,7 @@ class UpdateInputsAndResources(ScheduledTask): yield probe_ip.lookup() yield resources.check_for_update(probe_ip.geodata['countrycode']) yield input_store.update(probe_ip.geodata['countrycode']) + yield probe_ip.resolveGeodata() class UploadReports(ScheduledTask): diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 50db3ad..a03daa4 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -127,13 +127,7 @@ class WebUIAPI(object): self._xsrf_token = ''.join([rng.choice(token_space) for _ in range(30)]) - self.status = { - "software_version": ooniprobe_version, - "software_name": "ooniprobe", - "asn": probe_ip.geodata['asn'], - "country_code": probe_ip.geodata['countrycode'], - "director_started": False - } + self._director_started = False self.status_poller = LongPoller( self._long_polling_timeout, _reactor) @@ -150,14 +144,22 @@ class WebUIAPI(object): d.addCallback(self.director_started) d.addBoth(lambda _: self.status_poller.notify()) + @property + def status(self): + return { + "software_version": ooniprobe_version, + "software_name": "ooniprobe", + "asn": probe_ip.geodata['asn'], + "country_code": probe_ip.geodata['countrycode'], + "director_started": self._director_started + } + def handle_director_event(self, event): log.msg("Handling event {0}".format(event.type)) self.director_event_poller.notify(event) def director_started(self, _): - self.status['director_started'] = True - self.status["asn"] = probe_ip.geodata['asn'] - self.status["country_code"] = probe_ip.geodata['countrycode'] + self._director_started = True @app.handle_errors(NotFound) @xsrf_protect(check=False)

1 0

[ooni-probe/master] Run the check_for_update function when installation is complete
by art＠torproject.org 19 Sep '16

19 Sep '16

commit f7fec4b6d0aca457cfafdb0617218f2b4c0f0345 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Jul 30 17:53:32 2016 +0200 Run the check_for_update function when installation is complete --- Vagrantfile | 2 +- ooni/agent/scheduler.py | 6 ++- setup.py | 114 +++--------------------------------------------- 3 files changed, 10 insertions(+), 112 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index cf73244..0e75d55 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,7 +13,7 @@ apt-get update apt-get install -y tor deb.torproject.org-keyring # Setup for sniffer subsystem -apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip +apt-get install -y build-essential libdumbnet-dev libpcap-dev libgeoip-dev libffi-dev python-dev python-pip libssl-dev cd /data/ooni-probe python setup.py install diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 7a77afb..0d8689c 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -1,7 +1,8 @@ from datetime import datetime from twisted.application import service -from twisted.internet import task, defer +from twisted.internet import defer +from twisted.internet.task import LoopingCall from twisted.python.filepath import FilePath from ooni.scripts import oonireport @@ -72,6 +73,7 @@ class ScheduledTask(object): finally: self._last_run_lock.unlock() + class UpdateInputsAndResources(ScheduledTask): identifier = "update-inputs" schedule = "@daily" @@ -177,7 +179,7 @@ class SchedulerService(service.MultiService): service.MultiService.__init__(self) self.director = director self.interval = interval - self._looping_call = task.LoopingCall(self._should_run) + self._looping_call = LoopingCall(self._should_run) self._scheduled_tasks = [] def schedule(self, task): diff --git a/setup.py b/setup.py index 6940f66..da48323 100644 --- a/setup.py +++ b/setup.py @@ -89,16 +89,14 @@ Have fun! from __future__ import print_function import os -import sys import shutil import tempfile import subprocess from ConfigParser import SafeConfigParser from os.path import join as pj -from setuptools import setup, Command +from setuptools import setup from setuptools.command.install import install -from distutils.spawn import find_executable from ooni import __version__, __author__ @@ -177,11 +175,9 @@ class OoniInstall(install): pass def ooniresources(self): - ooniresources = find_executable("ooniresources") - process = subprocess.Popen([ooniresources], - stdout=sys.stdout.fileno(), - stderr=sys.stderr.fileno()) - process.wait() + from ooni.resources import check_for_update + from twisted.internet import task + task.react(lambda _: check_for_update()) def update_lepidopter_config(self): try: @@ -200,105 +196,6 @@ class OoniInstall(install): if is_lepidopter(): self.update_lepidopter_config() -class ExecutableNotFound(Exception): - pass - -class CreateOoniResources(Command): - description = ("Create ooni-resources.tar.gz containing test-lists and " - "GeoIP data files") - user_options = [] - - def initialize_options(self): - pass - def finalize_options(self): - pass - def download(self, url, directory, filename): - dst_path = pj(directory, filename) - args = [ - self.wget, - "-O", - dst_path, - url - ] - out = run_command(args) - if out is None: - raise Exception("Failed to download {0}".format(url)) - return dst_path - - def find_executables(self): - self.wget = find_executable("wget") - if not self.wget: - raise ExecutableNotFound("wget") - self.tar = find_executable("tar") - if not self.tar: - raise ExecutableNotFound("tar") - self.unzip = find_executable("unzip") - if not self.unzip: - raise ExecutableNotFound("unzip") - self.gunzip = find_executable("gunzip") - if not self.gunzip: - raise ExecutableNotFound("gunzip") - - def run(self): - dst_path = "dist/ooni-resources.tar.gz" - - try: - self.find_executables() - except ExecutableNotFound as enf: - print("ERR: Could not find '{0}'".format(enf.message)) - return - - tmp_dir = tempfile.mkdtemp() - pkg_dir = tempfile.mkdtemp() - - os.mkdir(pj(pkg_dir, "resources")) - os.mkdir(pj(pkg_dir, "GeoIP")) - - try: - geoip_asn_path = self.download(GEOIP_ASN_URL, tmp_dir, "GeoIPASNum.dat.gz") - except Exception as exc: - print(exc.message) - return - try: - geoip_path = self.download(GEOIP_URL, tmp_dir, "GeoIP.dat.gz") - except Exception as exc: - print(exc.message) - return - try: - test_lists_path = self.download(TEST_LISTS_URL, tmp_dir, "master.zip") - except Exception as exc: - print(exc.message) - return - - run_command([self.gunzip, geoip_asn_path]) - run_command([self.gunzip, geoip_path]) - run_command([self.unzip, "-d", tmp_dir, test_lists_path]) - - shutil.move(pj(tmp_dir, "GeoIP.dat"), - pj(pkg_dir, "GeoIP", "GeoIP.dat")) - shutil.move(pj(tmp_dir, "GeoIPASNum.dat"), - pj(pkg_dir, "GeoIP", "GeoIPASNum.dat")) - shutil.move(pj(tmp_dir, "test-lists-master", "lists"), - pj(pkg_dir, "resources", "citizenlab-test-lists")) - # Don't include services and official lists - shutil.rmtree( - pj(pkg_dir, - "resources", - "citizenlab-test-lists", - "services"), - ignore_errors=True) - shutil.rmtree( - pj(pkg_dir, - "resources", - "citizenlab-test-lists", - "official"), - ignore_errors=True) - run_command([self.tar, "cvzf", dst_path, "-C", pkg_dir, "."]) - - # Cleanup - shutil.rmtree(pkg_dir, ignore_errors=True) - shutil.rmtree(tmp_dir, ignore_errors=True) - print("Written ooniresources to {0}".format(dst_path)) install_requires = [] dependency_links = [] @@ -363,8 +260,7 @@ setup( ] }, cmdclass={ - "install": OoniInstall, - "create_ooniresources": CreateOoniResources + "install": OoniInstall }, classifiers=( "Development Status :: 5 - Production/Stable",

1 0

[ooni-probe/master] Bump to version a0
by art＠torproject.org 19 Sep '16

19 Sep '16

commit d1765a80349cc5eba46b342056cc08cc29ae6ae0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Jul 30 17:23:46 2016 +0200 Bump to version a0 --- ooni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index c5c3c0a..653a636 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "2.0.0.dev1" +__version__ = "2.0.0a0" __all__ = [ 'agent',

1 0