[tor-commits] [ooni-probe/master] Move oonireport, ooniresources and oonideckgen into the scripts/ directory.

art at torproject.org art at torproject.org
Mon Sep 19 12:14:24 UTC 2016


commit d707df8b536d96e70567085608a0fa780794af25
Author: Arturo Filastò <arturo at filasto.net>
Date:   Mon Jul 25 16:05:22 2016 +0200

    Move oonireport, ooniresources and oonideckgen into the scripts/ directory.
    
    * Delete unused oonid command
    
    * Create ooniprobe-agent script
---
 bin/oonid                                        |   7 -
 bin/oonideckgen                                  |  37 ----
 bin/oonireport                                   |  38 ----
 bin/ooniresources                                |  35 ----
 ooni/deckgen/__init__.py                         |   1 -
 ooni/deckgen/cli.py                              | 190 ------------------
 ooni/deckgen/processors/__init__.py              |   0
 ooni/deckgen/processors/citizenlab_test_lists.py |  55 ------
 ooni/deckgen/processors/namebench_dns_servers.py |  51 -----
 ooni/report/__init__.py                          |   1 -
 ooni/report/cli.py                               |  90 ---------
 ooni/report/parser.py                            |  35 ----
 ooni/report/tool.py                              | 117 -----------
 ooni/resources.py                                | 152 +++++++++++++++
 ooni/resources/__init__.py                       |  22 ---
 ooni/resources/cli.py                            |  44 -----
 ooni/resources/update.py                         | 187 ------------------
 ooni/scripts/__init__.py                         |   0
 ooni/scripts/oonideckgen.py                      | 151 ++++++++++++++
 ooni/scripts/ooniprobe_agent.py                  |  32 +++
 ooni/scripts/oonireport.py                       | 238 +++++++++++++++++++++++
 ooni/scripts/ooniresources.py                    |  34 ++++
 setup.py                                         |  14 +-
 23 files changed, 620 insertions(+), 911 deletions(-)

diff --git a/bin/oonid b/bin/oonid
deleted file mode 100755
index dd59eb9..0000000
--- a/bin/oonid
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-# XXX This is very Ghetto. The proper way to do this is to make a twisted
-# plugin. The plugin will then be installed and can then be run directly with
-# `twistd oonid`
-
-OONID_PATH=`python -c 'import ooni;import os;print os.path.join(os.path.dirname(os.path.abspath(ooni.__file__)), "oonid.py")'`
-twistd -y $OONID_PATH
diff --git a/bin/oonideckgen b/bin/oonideckgen
deleted file mode 100755
index f4dd392..0000000
--- a/bin/oonideckgen
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-import sys
-import exceptions
-
-from twisted.internet import defer, reactor
-
-from ooni.utils import log
-from ooni.deckgen import cli
-
-exitCode = 128
-def failed(failure):
-    global exitCode
-
-    r = failure.trap(exceptions.SystemExit,
-                     Exception)
-    if r != exceptions.SystemExit:
-        log.err("Failed to run oonideckgen")
-        log.exception(failure)
-        exitCode = 127
-    else:
-        exitCode = failure.value.code
-    reactor.stop()
-
-def done(result):
-    global exitCode
-
-    exitCode = 0
-    reactor.stop()
-
-def start():
-    d = defer.maybeDeferred(cli.run)
-    d.addCallback(done)
-    d.addErrback(failed)
-
-reactor.callWhenRunning(start)
-reactor.run()
-sys.exit(exitCode)
diff --git a/bin/oonireport b/bin/oonireport
deleted file mode 100755
index 9f59683..0000000
--- a/bin/oonireport
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-import sys
-import exceptions
-from twisted.internet import defer, reactor
-
-from ooni.utils import log
-from ooni.report import cli
-
-exitCode = 128
-
-def failed(failure):
-    global exitCode
-
-    r = failure.trap(exceptions.SystemExit,
-                     Exception)
-    if r != exceptions.SystemExit:
-        log.exception(failure)
-        log.err("Failed to run oonireport")
-        exitCode = 127
-    else:
-        exitCode = failure.value.code
-    reactor.stop()
-
-
-def done(result):
-    global exitCode
-    exitCode = 0
-
-    reactor.stop()
-
-def start():
-    d = defer.maybeDeferred(cli.run)
-    d.addCallback(done)
-    d.addErrback(failed)
-
-reactor.callWhenRunning(start)
-reactor.run()
-sys.exit(exitCode)
diff --git a/bin/ooniresources b/bin/ooniresources
deleted file mode 100755
index 6913bb4..0000000
--- a/bin/ooniresources
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env python
-import sys
-from twisted.internet import defer, reactor
-
-from ooni.utils import log
-from ooni.resources import cli
-
-exitCode = 128
-def failed(failure):
-    global exitCode
-
-    r = failure.trap(exceptions.SystemExit,
-                     Exception)
-    if r != exceptions.SystemExit:
-        log.err("Failed to run ooniresources")
-        log.exception(failure)
-        exitCode = 127
-    else:
-        exitCode = failure.value.code
-    reactor.stop()
-
-def done(result):
-    global exitCode
-
-    exitCode = 0
-    reactor.stop()
-
-def start():
-    d = defer.maybeDeferred(cli.run)
-    d.addCallback(done)
-    d.addErrback(done)
-
-reactor.callWhenRunning(start)
-reactor.run()
-sys.exit(exitCode)
diff --git a/ooni/deckgen/__init__.py b/ooni/deckgen/__init__.py
deleted file mode 100644
index d3ec452..0000000
--- a/ooni/deckgen/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__version__ = "0.2.0"
diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py
deleted file mode 100644
index 9f2cc4c..0000000
--- a/ooni/deckgen/cli.py
+++ /dev/null
@@ -1,190 +0,0 @@
-from __future__ import print_function
-
-import os
-import sys
-import copy
-import errno
-import shutil
-
-import yaml
-
-from twisted.internet import defer
-from twisted.python import usage
-
-from ooni import errors
-from ooni.geoip import ProbeIP
-from ooni.settings import config
-
-from ooni.deckgen import __version__
-from ooni.deckgen.processors import citizenlab_test_lists
-from ooni.resources.update import download_resources
-
-class Options(usage.Options):
-    synopsis = """%s [options]
-    """ % sys.argv[0]
-
-    optParameters = [
-        ["country-code", "c", None,
-         "Specify the two letter country code for which we should "
-         "generate the deck."],
-        ["collector", None, None, "Specify a custom collector to use when "
-                                  "submitting reports"],
-        ["bouncer", None, None, "Specify a custom bouncer to use"],
-        ["output", "o", None,
-         "Specify the directory where to write output."]
-    ]
-
-    def opt_version(self):
-        print("oonideckgen version: %s" % __version__)
-        sys.exit(0)
-
-
-class Deck(object):
-    _base_entry = {
-        "options": {
-            "test_file": None,
-            "subargs": [],
-            "annotations": None,
-
-            "collector": None,
-            "bouncer": None,
-
-            "reportfile": None,
-
-            "no-collector": 0,
-            "no-geoip": 0,
-            "no-yamloo": 0,
-            "verbose": 0
-        }
-    }
-
-    def __init__(self, collector=None, bouncer=None):
-        self.deck_entries = []
-        self.collector = collector
-        self.bouncer = bouncer
-
-    def add_test(self, test_file, subargs=[]):
-        deck_entry = copy.deepcopy(self._base_entry)
-        deck_entry['options']['collector'] = self.collector
-        deck_entry['options']['bouncer'] = self.bouncer
-        deck_entry['options']['test_file'] = test_file
-        deck_entry['options']['subargs'] = subargs
-        self.deck_entries.append(deck_entry)
-
-    def pprint(self):
-        print(yaml.safe_dump(self.deck_entries))
-
-    def write_to_file(self, filename):
-        with open(filename, "w+") as f:
-            f.write(yaml.safe_dump(self.deck_entries))
-
-
-def generate_deck(options):
-    url_list_country = None
-    try:
-        url_list_country = citizenlab_test_lists.generate_country_input(
-            options['country-code'],
-            options['output']
-        )
-    except Exception:
-        print("Could not generate country specific url list")
-        print("We will just use the global one.")
-
-    url_list_global = citizenlab_test_lists.generate_global_input(
-        options['output']
-    )
-
-    deck = Deck(collector=options['collector'], bouncer=options['bouncer'])
-    deck.add_test('manipulation/http_invalid_request_line')
-    deck.add_test('manipulation/http_header_field_manipulation')
-
-    if url_list_country is not None:
-        deck.add_test('blocking/web_connectivity', ['-f', url_list_country])
-    deck.add_test('blocking/web_connectivity', ['-f', url_list_global])
-
-    if config.advanced.debug:
-        deck.pprint()
-    deck_filename = os.path.join(options['output'],
-                                 "default-user.deck")
-    deck.write_to_file(deck_filename)
-    print("Deck written to %s" % deck_filename)
-    print("Run ooniprobe like so:")
-    print("ooniprobe -i %s" % deck_filename)
-
-
- at defer.inlineCallbacks
-def get_user_country_code():
-    config.privacy.includecountry = True
-    probe_ip = ProbeIP()
-    yield probe_ip.lookup()
-    defer.returnValue(probe_ip.geodata['countrycode'])
-
-def resources_up_to_date():
-    if config.get_data_file_path("GeoIP/GeoIP.dat") is None:
-        return False
-
-    if config.get_data_file_path("resources/"
-                                 "namebench-dns-servers.csv") is None:
-        return False
-
-    if config.get_data_file_path("resources/"
-                                 "citizenlab-test-lists/"
-                                 "global.csv") is None:
-        return False
-
-    return True
-
- at defer.inlineCallbacks
-def run():
-    options = Options()
-    try:
-        options.parseOptions()
-    except usage.UsageError as error_message:
-        print("%s: %s" % (sys.argv[0], error_message))
-        print(options)
-        sys.exit(1)
-
-    if not resources_up_to_date():
-        print("Resources for running ooniprobe are not up to date.")
-        print("Will update them now.")
-        yield download_resources()
-
-    if not options['output']:
-        options['output'] = os.getcwd()
-
-    if not options['country-code']:
-        try:
-            options['country-code'] = yield get_user_country_code()
-        except errors.ProbeIPUnknown:
-            print("Could not determine your IP address.")
-            print("Check your internet connection or specify a country code "
-                  "with -c.")
-            sys.exit(4)
-
-    if len(options['country-code']) != 2:
-        print("%s: --country-code must be 2 characters" % sys.argv[0])
-        sys.exit(2)
-
-    if not os.path.isdir(options['output']):
-        print("%s: %s is not a directory" % (sys.argv[0],
-                                             options['output']))
-        sys.exit(3)
-
-    options['country-code'] = options['country-code'].lower()
-
-    output_dir = os.path.abspath(options['output'])
-    output_dir = os.path.join(output_dir, "deck")
-
-    if os.path.isdir(output_dir):
-        print("Found previous deck deleting content of it")
-        shutil.rmtree(output_dir)
-
-    options['output'] = output_dir
-
-    try:
-        os.makedirs(options['output'])
-    except OSError as exception:
-        if exception.errno != errno.EEXIST:
-            raise
-
-    generate_deck(options)
diff --git a/ooni/deckgen/processors/__init__.py b/ooni/deckgen/processors/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py
deleted file mode 100644
index 8a660cf..0000000
--- a/ooni/deckgen/processors/citizenlab_test_lists.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import os
-import csv
-from ooni.settings import config
-
-
-def load_input(file_input, file_output):
-    fw = open(file_output, "w+")
-    with open(file_input) as f:
-        csvreader = csv.reader(f)
-        csvreader.next()
-        for row in csvreader:
-            fw.write("%s\n" % row[0])
-    fw.close()
-
-
-def generate_country_input(country_code, dst):
-    """
-    Write to dst/citizenlab-urls-{country_code}.txt
-    the list for the given country code.
-
-    Returns:
-
-        the path to the generated input
-    """
-
-    country_code = country_code.lower()
-    filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code)
-
-    input_list = config.get_data_file_path("resources/"
-                                           "citizenlab-test-lists/"
-                                           + country_code + ".csv")
-
-    if not input_list:
-        raise Exception("Could not find list for country %s" % country_code)
-
-    load_input(input_list, filename)
-
-    return filename
-
-
-def generate_global_input(dst):
-    filename = os.path.join(dst, "citizenlab-urls-global.txt")
-
-    input_list = config.get_data_file_path("resources/"
-                                           "citizenlab-test-lists/"
-                                           "global.csv")
-
-    if not input_list:
-        print("Could not find the global input list")
-        print("Perhaps you should run ooniresources")
-        raise Exception("Could not find the global input list")
-
-    load_input(input_list, filename)
-
-    return filename
diff --git a/ooni/deckgen/processors/namebench_dns_servers.py b/ooni/deckgen/processors/namebench_dns_servers.py
deleted file mode 100644
index 9855611..0000000
--- a/ooni/deckgen/processors/namebench_dns_servers.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import os
-import csv
-import GeoIP
-
-from ooni.settings import config
-
-
-class GeoIPDB(object):
-    _borg = {}
-    country = None
-
-    def __init__(self):
-        self.__dict__ = self._borg
-        if not self.country:
-            try:
-                country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
-                self.country = GeoIP.open(country_file,
-                                          GeoIP.GEOIP_STANDARD)
-            except:
-                raise Exception("Edit the geoip_data_dir line in your config"
-                                " file to point to your geoip files")
-
-
-def generate_country_input(country_code, dst):
-
-    csv_file = config.get_data_file_path("resources/"
-                                         "namebench-dns-servers.csv")
-
-    filename = os.path.join(dst, "dns-server-%s.txt" % country_code)
-    fw = open(filename, "w")
-    geoip_db = GeoIPDB()
-    reader = csv.reader(open(csv_file))
-    for row in reader:
-        if row[2] == 'X-Internal-IP':
-            continue
-        elif row[2] == 'X-Unroutable':
-            continue
-        elif row[2] == 'X-Link_local':
-            continue
-        ipaddr = row[0]
-        cc = geoip_db.country.country_code_by_addr(ipaddr)
-        if not cc:
-            continue
-        if cc.lower() == country_code.lower():
-            fw.write(ipaddr + "\n")
-    fw.close()
-    return filename
-
-
-def generate_global_input(dst):
-    pass
diff --git a/ooni/report/__init__.py b/ooni/report/__init__.py
deleted file mode 100644
index 3dc1f76..0000000
--- a/ooni/report/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__version__ = "0.1.0"
diff --git a/ooni/report/cli.py b/ooni/report/cli.py
deleted file mode 100644
index 485ab52..0000000
--- a/ooni/report/cli.py
+++ /dev/null
@@ -1,90 +0,0 @@
-from __future__ import print_function
-
-import os
-import sys
-
-from ooni.constants import CANONICAL_BOUNCER_ONION
-from ooni.report import __version__
-from ooni.report import tool
-from ooni.settings import config
-
-from twisted.python import usage
-
-
-class Options(usage.Options):
-
-    synopsis = """%s [options] upload | status
-""" % (os.path.basename(sys.argv[0]),)
-
-    optFlags = [
-        ["default-collector", "d", "Upload the reports to the default "
-                                   "collector that is looked up with the "
-                                   "canonical bouncer."]
-    ]
-
-    optParameters = [
-        ["configfile", "f", None,
-         "Specify the configuration file to use."],
-        ["collector", "c", None,
-         "Specify the collector to upload the result to."],
-        ["bouncer", "b", None,
-         "Specify the bouncer to query for a collector."]
-    ]
-
-    def opt_version(self):
-        print("oonireport version: %s" % __version__)
-        sys.exit(0)
-
-    def parseArgs(self, *args):
-        if len(args) == 0:
-            raise usage.UsageError(
-                "Must specify at least one command"
-            )
-            return
-        self['command'] = args[0]
-        if self['command'] not in ("upload", "status"):
-            raise usage.UsageError(
-                "Must specify either command upload or status"
-            )
-        if self['command'] == "upload":
-            try:
-                self['report_file'] = args[1]
-            except IndexError:
-                self['report_file'] = None
-
-
-def tor_check():
-    if not config.tor.socks_port:
-        print("Currently oonireport requires that you start Tor yourself "
-              "and set the socks_port inside of ooniprobe.conf")
-        sys.exit(1)
-
-
-def run(args=sys.argv[1:]):
-    options = Options()
-    try:
-        options.parseOptions(args)
-    except Exception as exc:
-        print("Error: %s" % exc)
-        print(options)
-        sys.exit(2)
-    config.global_options = dict(options)
-    config.set_paths()
-    config.read_config_file()
-
-    if options['default-collector']:
-        options['bouncer'] = CANONICAL_BOUNCER_ONION
-
-    if options['command'] == "upload" and options['report_file']:
-        tor_check()
-        return tool.upload(options['report_file'],
-                           options['collector'],
-                           options['bouncer'])
-    elif options['command'] == "upload":
-        tor_check()
-        return tool.upload_all(options['collector'],
-                               options['bouncer'])
-    elif options['command'] == "status":
-        return tool.status()
-    else:
-        print(options)
diff --git a/ooni/report/parser.py b/ooni/report/parser.py
deleted file mode 100644
index bef948c..0000000
--- a/ooni/report/parser.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import yaml
-
-
-class ReportLoader(object):
-    _header_keys = (
-        'probe_asn',
-        'probe_cc',
-        'probe_ip',
-        'start_time',
-        'test_name',
-        'test_version',
-        'options',
-        'input_hashes',
-        'software_name',
-        'software_version'
-    )
-
-    def __init__(self, report_filename):
-        self._fp = open(report_filename)
-        self._yfp = yaml.safe_load_all(self._fp)
-
-        self.header = self._yfp.next()
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        try:
-            return self._yfp.next()
-        except StopIteration:
-            self.close()
-            raise StopIteration
-
-    def close(self):
-        self._fp.close()
diff --git a/ooni/report/tool.py b/ooni/report/tool.py
deleted file mode 100644
index f8af132..0000000
--- a/ooni/report/tool.py
+++ /dev/null
@@ -1,117 +0,0 @@
-from __future__ import print_function
-import yaml
-import sys
-
-from twisted.internet import defer
-
-from ooni.constants import CANONICAL_BOUNCER_ONION
-from ooni.reporter import OONIBReporter, OONIBReportLog
-
-from ooni.utils import log
-from ooni.report import parser
-from ooni.settings import config
-from ooni.backend_client import BouncerClient, CollectorClient
-
- at defer.inlineCallbacks
-def lookup_collector_client(report_header, bouncer):
-    oonib_client = BouncerClient(bouncer)
-    net_tests = [{
-        'test-helpers': [],
-        'input-hashes': report_header['input_hashes'],
-        'name': report_header['test_name'],
-        'version': report_header['test_version'],
-    }]
-    result = yield oonib_client.lookupTestCollector(
-        net_tests
-    )
-    collector_client = CollectorClient(
-        address=result['net-tests'][0]['collector']
-    )
-    defer.returnValue(collector_client)
-
- at defer.inlineCallbacks
-def upload(report_file, collector=None, bouncer=None):
-    oonib_report_log = OONIBReportLog()
-    collector_client = None
-    if collector:
-        collector_client = CollectorClient(address=collector)
-
-    log.msg("Attempting to upload %s" % report_file)
-
-    with open(config.report_log_file) as f:
-        report_log = yaml.safe_load(f)
-
-    report = parser.ReportLoader(report_file)
-    if bouncer and collector_client is None:
-        collector_client = yield lookup_collector_client(report.header,
-                                                         bouncer)
-
-    if collector_client is None:
-        try:
-            collector_settings = report_log[report_file]['collector']
-            if collector_settings is None:
-                log.msg("Skipping uploading of %s since this measurement "
-                        "was run by specifying no collector." %
-                        report_file)
-                defer.returnValue(None)
-            elif isinstance(collector_settings, dict):
-                collector_client = CollectorClient(settings=collector_settings)
-            elif isinstance(collector_settings, str):
-                collector_client = CollectorClient(address=collector_settings)
-        except KeyError:
-            log.msg("Could not find %s in reporting.yaml. Looking up "
-                    "collector with canonical bouncer." % report_file)
-            collector_client = yield lookup_collector_client(report.header,
-                                                             CANONICAL_BOUNCER_ONION)
-
-    oonib_reporter = OONIBReporter(report.header, collector_client)
-    log.msg("Creating report for %s with %s" % (report_file,
-                                                collector_client.settings))
-    report_id = yield oonib_reporter.createReport()
-    report.header['report_id'] = report_id
-    yield oonib_report_log.created(report_file,
-                                   collector_client.settings,
-                                   report_id)
-    log.msg("Writing report entries")
-    for entry in report:
-        yield oonib_reporter.writeReportEntry(entry)
-        sys.stdout.write('.')
-        sys.stdout.flush()
-    log.msg("Closing report")
-    yield oonib_reporter.finish()
-    yield oonib_report_log.closed(report_file)
-
-
- at defer.inlineCallbacks
-def upload_all(collector=None, bouncer=None):
-    oonib_report_log = OONIBReportLog()
-
-    for report_file, value in oonib_report_log.reports_to_upload:
-        try:
-            yield upload(report_file, collector, bouncer)
-        except Exception as exc:
-            log.exception(exc)
-
-
-def print_report(report_file, value):
-    print("* %s" % report_file)
-    print("  %s" % value['created_at'])
-
-
-def status():
-    oonib_report_log = OONIBReportLog()
-
-    print("Reports to be uploaded")
-    print("----------------------")
-    for report_file, value in oonib_report_log.reports_to_upload:
-        print_report(report_file, value)
-
-    print("Reports in progress")
-    print("-------------------")
-    for report_file, value in oonib_report_log.reports_in_progress:
-        print_report(report_file, value)
-
-    print("Incomplete reports")
-    print("------------------")
-    for report_file, value in oonib_report_log.reports_incomplete:
-        print_report(report_file, value)
diff --git a/ooni/resources.py b/ooni/resources.py
new file mode 100644
index 0000000..d49e679
--- /dev/null
+++ b/ooni/resources.py
@@ -0,0 +1,152 @@
+import json
+
+from twisted.python.filepath import FilePath
+from twisted.internet import defer
+from twisted.web.client import downloadPage, getPage
+
+from ooni.utils import log
+from ooni.settings import config
+
+class UpdateFailure(Exception):
+    pass
+
+def get_download_url(tag_name, filename):
+    return ("https://github.com/OpenObservatory/ooni-resources/releases"
+            "/download/{0}/{1}".format(tag_name, filename))
+
+def get_current_version():
+    manifest = FilePath(config.resources_directory).child("manifest.json")
+    if not manifest.exists():
+        return 0
+    with manifest.open("r") as f:
+        manifest = json.load(f)
+    return int(manifest["version"])
+
+ at defer.inlineCallbacks
+def get_latest_version():
+    """
+    Fetches the latest version of the resources package.
+    :return: (int) the latest version number
+    """
+    try:
+        version = yield getPage(get_download_url("latest", "version"))
+    except Exception as exc:
+        raise exc
+    defer.returnValue(int(version.strip()))
+
+
+def get_out_of_date_resources(current_manifest, new_manifest,
+                              country_code=None):
+    current_res = {}
+    new_res = {}
+    for r in current_manifest["resources"]:
+        current_res[r["path"]] = r
+
+    for r in new_manifest["resources"]:
+        new_res[r["path"]] = r
+
+    paths_to_delete = [
+        current_res[path] for path in list(set(current_res.keys()) -
+                                           set(new_res.keys()))
+        ]
+    paths_to_update = []
+    _resources = FilePath(config.resources_directory)
+    for path, info in new_res.items():
+        if (country_code is not None and
+                info["country_code"] != "ALL" and
+                info["country_code"] != country_code):
+            continue
+        if current_res.get(path, None) is None:
+            paths_to_update.append(info)
+        elif current_res[path]["version"] < info["version"]:
+            paths_to_update.append(info)
+        else:
+            pre_path, filename = info["path"].split("/")
+            # Also perform an update when it doesn't exist on disk, although
+            #  the manifest claims we have a more up to date version.
+            # This happens if an update by country_code happened and a new
+            # country code is now required.
+            if not _resources.child(pre_path).child(filename).exists():
+                paths_to_update.append(info)
+
+    return paths_to_update, paths_to_delete
+
+ at defer.inlineCallbacks
+def check_for_update(country_code=None):
+    """
+    Checks if we need to update the resources.
+    If the country_code is specified then only the resources for that
+    country will be updated/downloaded.
+    :return: the latest version.
+    """
+    temporary_files = []
+    def cleanup():
+        # If we fail we need to delete all the temporary files
+        for _, src_file_path in temporary_files:
+            src_file_path.remove()
+
+    current_version = get_current_version()
+    latest_version = yield get_latest_version()
+
+    # We are already at the latest version
+    if current_version == latest_version:
+        defer.returnValue(latest_version)
+
+    resources_dir = FilePath(config.resources_directory)
+    resources_dir.makedirs(ignoreExistingDirectory=True)
+    current_manifest = resources_dir.child("manifest.json")
+
+    new_manifest = current_manifest.temporarySibling()
+    new_manifest.alwaysCreate = 0
+
+    temporary_files.append((current_manifest, new_manifest))
+
+    try:
+        yield downloadPage(
+            get_download_url(latest_version, "manifest.json"),
+            new_manifest.path
+        )
+    except:
+        cleanup()
+        raise UpdateFailure("Failed to download manifest")
+
+    new_manifest_data = json.loads(new_manifest.getContent())
+
+    if current_manifest.exists():
+        with current_manifest.open("r") as f:
+            current_manifest_data = json.loads(f)
+    else:
+        current_manifest_data = {
+            "resources": []
+        }
+
+    to_update, to_delete = get_out_of_date_resources(
+            current_manifest_data, new_manifest_data, country_code)
+
+    try:
+        for resource in to_update:
+            pre_path, filename = resource["path"].split("/")
+            dst_file = resources_dir.child(pre_path).child(filename)
+            dst_file.parent().makedirs(ignoreExistingDirectory=True)
+            src_file = dst_file.temporarySibling()
+            src_file.alwaysCreate = 0
+
+            temporary_files.append((dst_file, src_file))
+            # The paths for the download require replacing "/" with "."
+            download_url = get_download_url(latest_version,
+                                            resource["path"].replace("/", "."))
+            print("Downloading {0}".format(download_url))
+            yield downloadPage(download_url, src_file.path)
+    except Exception as exc:
+        cleanup()
+        log.exception(exc)
+        raise UpdateFailure("Failed to download resource {0}".format(resource["path"]))
+
+    for dst_file, src_file in temporary_files:
+        log.msg("Moving {0} to {1}".format(src_file.path,
+                                           dst_file.path))
+        src_file.moveTo(dst_file)
+
+    for resource in to_delete:
+        log.msg("Deleting old resources")
+        resources_dir.child(resource["path"]).remove()
diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py
deleted file mode 100644
index 6550887..0000000
--- a/ooni/resources/__init__.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import json
-
-from twisted.python.filepath import FilePath
-
-from ooni import __resources_version__ as resources_version
-from ooni.settings import config
-
-ooni_resources_url = ("https://github.com/TheTorProject/ooni-probe/releases"
-                      "/download/v{}/"
-                      "ooni-resources.tar.gz").format(resources_version)
-
-def get_download_url(tag_name, filename):
-    return ("https://github.com/OpenObservatory/ooni-resources/releases"
-            "/download/{0}/{1}".format(tag_name, filename))
-
-def get_current_version():
-    manifest = FilePath(config.resources_directory).child("manifest.json")
-    if not manifest.exists():
-        return 0
-    with manifest.open("r") as f:
-        manifest = json.load(f)
-    return int(manifest["version"])
diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py
deleted file mode 100644
index 0d7832a..0000000
--- a/ooni/resources/cli.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import sys
-
-from twisted.internet import defer
-from twisted.python import usage
-
-from ooni.resources import __version__
-from ooni.resources import update
-
-
-class Options(usage.Options):
-    synopsis = """%s
-    This is used to update the resources required to run oonideckgen and
-    ooniprobe.
-    You just run this script with no arguments and it will update the
-    resources.
-    """ % sys.argv[0]
-
-    optFlags = [
-        ["update-inputs", None, "(deprecated) update the resources needed for "
-                                "inputs."],
-        ["update-geoip", None, "(deprecated) Update the geoip related "
-                               "resources."]
-    ]
-    optParameters = []
-
-    def opt_version(self):
-        print("ooniresources version: %s" % __version__)
-        sys.exit(0)
-
-
- at defer.inlineCallbacks
-def run():
-    options = Options()
-    try:
-        options.parseOptions()
-    except usage.UsageError as error_message:
-        print "%s: %s" % (sys.argv[0], error_message)
-        print "%s: Try --help for usage details." % (sys.argv[0])
-        sys.exit(1)
-
-    if options['update-inputs'] or options['update-geoip']:
-        print("WARNING: Passing command line arguments is deprecated")
-
-    yield update.download_resources()
diff --git a/ooni/resources/update.py b/ooni/resources/update.py
deleted file mode 100644
index b464920..0000000
--- a/ooni/resources/update.py
+++ /dev/null
@@ -1,187 +0,0 @@
-import os
-import json
-import tarfile
-import tempfile
-
-from twisted.python.filepath import FilePath
-from twisted.internet import defer
-from twisted.web.client import downloadPage, getPage
-
-from ooni.utils import log
-from ooni.settings import config
-from ooni.resources import ooni_resources_url, get_download_url
-from ooni.resources import get_current_version
-
-class UpdateFailure(Exception):
-    pass
-
- at defer.inlineCallbacks
-def get_latest_version():
-    """
-    Fetches the latest version of the resources package.
-    :return: (int) the latest version number
-    """
-    try:
-        version = yield getPage(get_download_url("latest", "version"))
-    except Exception as exc:
-        raise exc
-    defer.returnValue(int(version.strip()))
-
-
-def get_out_of_date_resources(current_manifest, new_manifest,
-                              country_code=None):
-    current_res = {}
-    new_res = {}
-    for r in current_manifest["resources"]:
-        current_res[r["path"]] = r
-
-    for r in new_manifest["resources"]:
-        new_res[r["path"]] = r
-
-    paths_to_delete = [
-        current_res[path] for path in list(set(current_res.keys()) -
-                                           set(new_res.keys()))
-        ]
-    paths_to_update = []
-    _resources = FilePath(config.resources_directory)
-    for path, info in new_res.items():
-        if (country_code is not None and
-                info["country_code"] != "ALL" and
-                info["country_code"] != country_code):
-            continue
-        if current_res[path]["version"] < info["version"]:
-            paths_to_update.append(info)
-        else:
-            pre_path, filename = info["path"].split("/")
-            # Also perform an update when it doesn't exist on disk, although
-            #  the manifest claims we have a more up to date version.
-            # This happens if an update by country_code happened and a new
-            # country code is now required.
-            if not _resources.child(pre_path).child(filename).exists():
-                paths_to_update.append(info)
-
-    return paths_to_update, paths_to_delete
-
- at defer.inlineCallbacks
-def check_for_update(country_code=None):
-    """
-    Checks if we need to update the resources.
-    If the country_code is specified then only the resources for that
-    country will be updated/downloaded.
-    :return: the latest version.
-    """
-    temporary_files = []
-    def cleanup():
-        # If we fail we need to delete all the temporary files
-        for _, src_file_path in temporary_files:
-            src_file_path.remove()
-
-    current_version = get_current_version()
-    latest_version = yield get_latest_version()
-
-    # We are already at the latest version
-    if current_version == latest_version:
-        defer.returnValue(latest_version)
-
-    resources_dir = FilePath(config.resources_directory)
-    resources_dir.makedirs(ignoreExistingDirectory=True)
-    current_manifest = resources_dir.child("manifest.json")
-
-    new_manifest = current_manifest.temporarySibling()
-    new_manifest.alwaysCreate = 0
-
-    temporary_files.append((current_manifest, new_manifest))
-
-    try:
-        yield downloadPage(
-            get_download_url(latest_version, "manifest.json"),
-            new_manifest.path
-        )
-    except:
-        cleanup()
-        raise UpdateFailure("Failed to download manifest")
-
-    new_manifest_data = json.loads(new_manifest.getContent())
-
-    to_update = new_manifest_data["resources"]
-    to_delete = []
-    if current_manifest.exists():
-        with current_manifest.open("r") as f:
-            current_manifest_data = json.loads(f)
-        to_update, to_delete = get_out_of_date_resources(
-            current_manifest_data, new_manifest_data, country_code)
-
-    try:
-        for resource in to_update:
-            pre_path, filename = resource["path"].split("/")
-            dst_file = resources_dir.child(pre_path).child(filename)
-            dst_file.parent().makedirs(ignoreExistingDirectory=True)
-            src_file = dst_file.temporarySibling()
-            src_file.alwaysCreate = 0
-
-            temporary_files.append((dst_file, src_file))
-            # The paths for the download require replacing "/" with "."
-            download_url = get_download_url(latest_version,
-                                            resource["path"].replace("/", "."))
-            print("Downloading {0}".format(download_url))
-            yield downloadPage(download_url, src_file.path)
-    except Exception as exc:
-        cleanup()
-        log.exception(exc)
-        raise UpdateFailure("Failed to download resource {0}".format(resource["path"]))
-
-    for dst_file, src_file in temporary_files:
-        log.msg("Moving {0} to {1}".format(src_file.path,
-                                           dst_file.path))
-        src_file.moveTo(dst_file)
-
-    for resource in to_delete:
-        log.msg("Deleting old resources")
-        resources_dir.child(resource["path"]).remove()
-
- at defer.inlineCallbacks
-def download_resources():
-    if os.access(config.var_lib_path, os.W_OK):
-        dst_directory = FilePath(config.var_lib_path)
-    else:
-        dst_directory = FilePath(config.ooni_home)
-
-    print("Downloading {} to {}".format(ooni_resources_url,
-                                        dst_directory.path))
-    tmp_download_directory = FilePath(tempfile.mkdtemp())
-    tmp_download_filename = tmp_download_directory.temporarySibling()
-
-
-    try:
-        yield downloadPage(ooni_resources_url, tmp_download_filename.path)
-        ooni_resources_tar_gz = tarfile.open(tmp_download_filename.path)
-        ooni_resources_tar_gz.extractall(tmp_download_directory.path)
-
-        if not tmp_download_directory.child('GeoIP').exists():
-            raise Exception("Could not find GeoIP data files in downloaded "
-                            "tar.")
-
-        if not tmp_download_directory.child('resources').exists():
-            raise Exception("Could not find resources data files in "
-                            "downloaded tar.")
-
-        geoip_dir = dst_directory.child('GeoIP')
-        resources_dir = dst_directory.child('resources')
-
-        if geoip_dir.exists():
-            geoip_dir.remove()
-        tmp_download_directory.child('GeoIP').moveTo(geoip_dir)
-
-        if resources_dir.exists():
-            resources_dir.remove()
-        tmp_download_directory.child('resources').moveTo(resources_dir)
-
-        print("Written GeoIP files to {}".format(geoip_dir.path))
-        print("Written resources files to {}".format(resources_dir.path))
-
-    except Exception as exc:
-        print("Failed to download resources!")
-        raise exc
-
-    finally:
-        tmp_download_directory.remove()
diff --git a/ooni/scripts/__init__.py b/ooni/scripts/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
new file mode 100644
index 0000000..fa675f9
--- /dev/null
+++ b/ooni/scripts/oonideckgen.py
@@ -0,0 +1,151 @@
+from __future__ import print_function
+
+import errno
+import os
+import shutil
+import sys
+
+from twisted.internet import defer, task
+from twisted.python import usage
+
+from ooni.otime import prettyDateNowUTC
+from ooni import errors
+from ooni.geoip import ProbeIP
+from ooni.resources import check_for_update
+from ooni.settings import config
+from ooni.deck import NGDeck
+
+__version__ = "1.0.0"
+
+class Options(usage.Options):
+    synopsis = """%s [options]
+    """ % sys.argv[0]
+
+    optParameters = [
+        ["country-code", "c", None,
+         "Specify the two letter country code for which we should "
+         "generate the deck."],
+        ["collector", None, None, "Specify a custom collector to use when "
+                                  "submitting reports"],
+        ["bouncer", None, None, "Specify a custom bouncer to use"],
+        ["output", "o", None,
+         "Specify the directory where to write output."]
+    ]
+
+    def opt_version(self):
+        print("oonideckgen version: " % __version__)
+        sys.exit(0)
+
+def generate_deck(options):
+
+    deck_data = {
+        "name": "Default ooniprobe deck",
+        "description": "Default ooniprobe deck generated on {0}".format(
+                       prettyDateNowUTC()),
+        "schedule": "@daily",
+        "tasks": [
+            {
+                "ooni": {
+                    "test_name": "http_invalid_request_line"
+                },
+            },
+            {
+                "ooni": {
+                    "test_name": "http_header_field_manipulation"
+                },
+            },
+            {
+                "ooni": {
+                    "test_name": "web_connectivity",
+                    "file": "$citizenlab_${probe_cc}_urls"
+                },
+            },
+            {
+                "ooni": {
+                    "test_name": "web_connectivity",
+                    "file": "$citizenlab_global_urls"
+                }
+            }
+        ]
+    }
+    if options["collector"]:
+        deck_data["collector"] = options['collector']
+
+    if options["bouncer"]:
+        deck_data["bouncer"] = options['bouncer']
+
+    deck = NGDeck(deck_data=deck_data)
+    with open(options['output']) as fw:
+        deck.write(fw)
+
+    print("Deck written to {0}".format(options['output']))
+    print("Run ooniprobe like so:")
+    print("ooniprobe -i {0}".format(options['output']))
+
+
+ at defer.inlineCallbacks
+def get_user_country_code():
+    config.privacy.includecountry = True
+    probe_ip = ProbeIP()
+    yield probe_ip.lookup()
+    defer.returnValue(probe_ip.geodata['countrycode'])
+
+
+ at defer.inlineCallbacks
+def oonideckgen():
+    options = Options()
+    try:
+        options.parseOptions()
+    except usage.UsageError as error_message:
+        print("%s: %s" % (sys.argv[0], error_message))
+        print(options)
+        sys.exit(1)
+
+    print("Checking for update of resources")
+    yield check_for_update()
+
+    if not options['output']:
+        options['output'] = os.getcwd()
+
+    if not options['country-code']:
+        try:
+            options['country-code'] = yield get_user_country_code()
+        except errors.ProbeIPUnknown:
+            print("Could not determine your IP address.")
+            print("Check your internet connection or specify a country code "
+                  "with -c.")
+            sys.exit(4)
+
+    if len(options['country-code']) != 2:
+        print("%s: --country-code must be 2 characters" % sys.argv[0])
+        sys.exit(2)
+
+    if not os.path.isdir(options['output']):
+        print("%s: %s is not a directory" % (sys.argv[0],
+                                             options['output']))
+        sys.exit(3)
+
+    options['country-code'] = options['country-code'].lower()
+
+    output_dir = os.path.abspath(options['output'])
+    output_dir = os.path.join(output_dir, "deck")
+
+    if os.path.isdir(output_dir):
+        print("Found previous deck deleting content of it")
+        shutil.rmtree(output_dir)
+
+    options['output'] = output_dir
+
+    try:
+        os.makedirs(options['output'])
+    except OSError as exception:
+        if exception.errno != errno.EEXIST:
+            raise
+
+    generate_deck(options)
+
+def run():
+    task.react(oonideckgen)
+
+if __name__ == "__main__":
+    run()
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
new file mode 100644
index 0000000..7833308
--- /dev/null
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -0,0 +1,32 @@
+from twisted.scripts import twistd
+from twisted.python import usage
+
+from ooni.agent.agent import AgentService
+
+class StartOoniprobeAgentPlugin:
+    tapname = "ooniprobe"
+
+    def makeService(self, so):
+        return AgentService()
+
+class OoniprobeTwistdConfig(twistd.ServerOptions):
+    subCommands = [
+        ("StartOoniprobeAgent", None, usage.Options, "ooniprobe agent")
+    ]
+
+def run():
+    twistd_args = ["--nodaemon"]
+    twistd_config = OoniprobeTwistdConfig()
+    twistd_args.append("StartOoniprobeAgent")
+    try:
+        twistd_config.parseOptions(twistd_args)
+    except usage.error, ue:
+        print("ooniprobe: usage error from twistd: {}\n".format(ue))
+    twistd_config.loadedPlugins = {
+        "StartOoniprobeAgent": StartOoniprobeAgentPlugin()
+    }
+    twistd.runApp(twistd_config)
+    return 0
+
+if __name__ == "__main__":
+    run()
diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py
new file mode 100644
index 0000000..6facadf
--- /dev/null
+++ b/ooni/scripts/oonireport.py
@@ -0,0 +1,238 @@
+from __future__ import print_function
+
+import os
+import sys
+import yaml
+
+from twisted.python import usage
+from twisted.internet import defer, task
+
+from ooni.constants import CANONICAL_BOUNCER_ONION
+from ooni.reporter import OONIBReporter, OONIBReportLog
+
+from ooni.utils import log
+from ooni.settings import config
+from ooni.backend_client import BouncerClient, CollectorClient
+
+__version__ = "0.1.0"
+
+ at defer.inlineCallbacks
+def lookup_collector_client(report_header, bouncer):
+    oonib_client = BouncerClient(bouncer)
+    net_tests = [{
+        'test-helpers': [],
+        'input-hashes': report_header['input_hashes'],
+        'name': report_header['test_name'],
+        'version': report_header['test_version'],
+    }]
+    result = yield oonib_client.lookupTestCollector(
+        net_tests
+    )
+    collector_client = CollectorClient(
+        address=result['net-tests'][0]['collector']
+    )
+    defer.returnValue(collector_client)
+
+ at defer.inlineCallbacks
+def upload(report_file, collector=None, bouncer=None):
+    oonib_report_log = OONIBReportLog()
+    collector_client = None
+    if collector:
+        collector_client = CollectorClient(address=collector)
+
+    log.msg("Attempting to upload %s" % report_file)
+
+    with open(config.report_log_file) as f:
+        report_log = yaml.safe_load(f)
+
+    report = ReportLoader(report_file)
+    if bouncer and collector_client is None:
+        collector_client = yield lookup_collector_client(report.header,
+                                                         bouncer)
+
+    if collector_client is None:
+        try:
+            collector_settings = report_log[report_file]['collector']
+            if collector_settings is None:
+                log.msg("Skipping uploading of %s since this measurement "
+                        "was run by specifying no collector." %
+                        report_file)
+                defer.returnValue(None)
+            elif isinstance(collector_settings, dict):
+                collector_client = CollectorClient(settings=collector_settings)
+            elif isinstance(collector_settings, str):
+                collector_client = CollectorClient(address=collector_settings)
+        except KeyError:
+            log.msg("Could not find %s in reporting.yaml. Looking up "
+                    "collector with canonical bouncer." % report_file)
+            collector_client = yield lookup_collector_client(report.header,
+                                                             CANONICAL_BOUNCER_ONION)
+
+    oonib_reporter = OONIBReporter(report.header, collector_client)
+    log.msg("Creating report for %s with %s" % (report_file,
+                                                collector_client.settings))
+    report_id = yield oonib_reporter.createReport()
+    report.header['report_id'] = report_id
+    yield oonib_report_log.created(report_file,
+                                   collector_client.settings,
+                                   report_id)
+    log.msg("Writing report entries")
+    for entry in report:
+        yield oonib_reporter.writeReportEntry(entry)
+        sys.stdout.write('.')
+        sys.stdout.flush()
+    log.msg("Closing report")
+    yield oonib_reporter.finish()
+    yield oonib_report_log.closed(report_file)
+
+
+ at defer.inlineCallbacks
+def upload_all(collector=None, bouncer=None):
+    oonib_report_log = OONIBReportLog()
+
+    for report_file, value in oonib_report_log.reports_to_upload:
+        try:
+            yield upload(report_file, collector, bouncer)
+        except Exception as exc:
+            log.exception(exc)
+
+
+def print_report(report_file, value):
+    print("* %s" % report_file)
+    print("  %s" % value['created_at'])
+
+
+def status():
+    oonib_report_log = OONIBReportLog()
+
+    print("Reports to be uploaded")
+    print("----------------------")
+    for report_file, value in oonib_report_log.reports_to_upload:
+        print_report(report_file, value)
+
+    print("Reports in progress")
+    print("-------------------")
+    for report_file, value in oonib_report_log.reports_in_progress:
+        print_report(report_file, value)
+
+    print("Incomplete reports")
+    print("------------------")
+    for report_file, value in oonib_report_log.reports_incomplete:
+        print_report(report_file, value)
+
+class ReportLoader(object):
+    _header_keys = (
+        'probe_asn',
+        'probe_cc',
+        'probe_ip',
+        'start_time',
+        'test_name',
+        'test_version',
+        'options',
+        'input_hashes',
+        'software_name',
+        'software_version'
+    )
+
+    def __init__(self, report_filename):
+        self._fp = open(report_filename)
+        self._yfp = yaml.safe_load_all(self._fp)
+
+        self.header = self._yfp.next()
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        try:
+            return self._yfp.next()
+        except StopIteration:
+            self.close()
+            raise StopIteration
+
+    def close(self):
+        self._fp.close()
+
+class Options(usage.Options):
+
+    synopsis = """%s [options] upload | status
+""" % (os.path.basename(sys.argv[0]),)
+
+    optFlags = [
+        ["default-collector", "d", "Upload the reports to the default "
+                                   "collector that is looked up with the "
+                                   "canonical bouncer."]
+    ]
+
+    optParameters = [
+        ["configfile", "f", None,
+         "Specify the configuration file to use."],
+        ["collector", "c", None,
+         "Specify the collector to upload the result to."],
+        ["bouncer", "b", None,
+         "Specify the bouncer to query for a collector."]
+    ]
+
+    def opt_version(self):
+        print("oonireport version: %s" % __version__)
+        sys.exit(0)
+
+    def parseArgs(self, *args):
+        if len(args) == 0:
+            raise usage.UsageError(
+                "Must specify at least one command"
+            )
+            return
+        self['command'] = args[0]
+        if self['command'] not in ("upload", "status"):
+            raise usage.UsageError(
+                "Must specify either command upload or status"
+            )
+        if self['command'] == "upload":
+            try:
+                self['report_file'] = args[1]
+            except IndexError:
+                self['report_file'] = None
+
+
+def tor_check():
+    if not config.tor.socks_port:
+        print("Currently oonireport requires that you start Tor yourself "
+              "and set the socks_port inside of ooniprobe.conf")
+        sys.exit(1)
+
+
+def oonireport(args=sys.argv[1:]):
+    options = Options()
+    try:
+        options.parseOptions(args)
+    except Exception as exc:
+        print("Error: %s" % exc)
+        print(options)
+        sys.exit(2)
+    config.global_options = dict(options)
+    config.set_paths()
+    config.read_config_file()
+
+    if options['default-collector']:
+        options['bouncer'] = CANONICAL_BOUNCER_ONION
+
+    if options['command'] == "upload" and options['report_file']:
+        tor_check()
+        return upload(options['report_file'],
+                      options['collector'],
+                      options['bouncer'])
+    elif options['command'] == "upload":
+        tor_check()
+        return upload_all(options['collector'],
+                          options['bouncer'])
+    elif options['command'] == "status":
+        return status()
+    else:
+        print(options)
+
+def run():
+    task.react(oonireport)
+
+if __name__ == "__main__":
+    run()
diff --git a/ooni/scripts/ooniresources.py b/ooni/scripts/ooniresources.py
new file mode 100644
index 0000000..5fdbbf0
--- /dev/null
+++ b/ooni/scripts/ooniresources.py
@@ -0,0 +1,34 @@
+import sys
+
+from twisted.python import usage
+
+class Options(usage.Options):
+    synopsis = """%s
+    [DEPRECATED] Usage of this script is deprecated and it will be deleted
+    in future versions of ooniprobe.
+    """ % sys.argv[0]
+
+    optFlags = [
+        ["update-inputs", None, "(deprecated) update the resources needed for "
+                                "inputs."],
+        ["update-geoip", None, "(deprecated) Update the geoip related "
+                               "resources."]
+    ]
+    optParameters = []
+
+    def opt_version(self):
+        print("ooniresources version: 0.2.0")
+        sys.exit(0)
+
+
+def run():
+    options = Options()
+    try:
+        options.parseOptions()
+    except usage.UsageError as error_message:
+        print "%s: %s" % (sys.argv[0], error_message)
+        print "%s: Try --help for usage details." % (sys.argv[0])
+        sys.exit(1)
+
+    print("WARNING: Usage of this script is deprecated.")
+    sys.exit(0)
diff --git a/setup.py b/setup.py
index 1a24460..d401a65 100644
--- a/setup.py
+++ b/setup.py
@@ -88,7 +88,6 @@ Have fun!
 
 from __future__ import print_function
 
-from ooni import __version__, __author__
 import os
 import sys
 import glob
@@ -102,6 +101,9 @@ from setuptools import setup, Command
 from setuptools.command.install import install
 from distutils.spawn import find_executable
 
+from ooni import __version__, __author__
+from ooni.scripts import ooniresources
+
 GEOIP_ASN_URL = "https://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz"
 GEOIP_URL = "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz"
 TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip"
@@ -393,6 +395,16 @@ setup(
     dependency_links=dependency_links,
     install_requires=install_requires,
     zip_safe=False,
+    entry_points={
+        'console_scripts': [
+            'ooniresources = ooni.scripts.ooniresources:run', # This is deprecated
+            'oonideckgen = ooni.scripts.oonideckgen:run', # This is deprecated
+
+            'ooniprobe = ooni.scripts.ooniprobe:run',
+            'oonireport = ooni.scripts.oonireport:run',
+            'ooniprobe-agent = ooni.scripts.ooniprobe_agent:run'
+        ]
+    },
     cmdclass={
         "install": OoniInstall,
         "create_ooniresources": CreateOoniResources,





More information about the tor-commits mailing list