commit d707df8b536d96e70567085608a0fa780794af25 Author: Arturo Filastò arturo@filasto.net Date: Mon Jul 25 16:05:22 2016 +0200
Move oonireport, ooniresources and oonideckgen into the scripts/ directory.
* Delete unused oonid command
* Create ooniprobe-agent script --- bin/oonid | 7 - bin/oonideckgen | 37 ---- bin/oonireport | 38 ---- bin/ooniresources | 35 ---- ooni/deckgen/__init__.py | 1 - ooni/deckgen/cli.py | 190 ------------------ ooni/deckgen/processors/__init__.py | 0 ooni/deckgen/processors/citizenlab_test_lists.py | 55 ------ ooni/deckgen/processors/namebench_dns_servers.py | 51 ----- ooni/report/__init__.py | 1 - ooni/report/cli.py | 90 --------- ooni/report/parser.py | 35 ---- ooni/report/tool.py | 117 ----------- ooni/resources.py | 152 +++++++++++++++ ooni/resources/__init__.py | 22 --- ooni/resources/cli.py | 44 ----- ooni/resources/update.py | 187 ------------------ ooni/scripts/__init__.py | 0 ooni/scripts/oonideckgen.py | 151 ++++++++++++++ ooni/scripts/ooniprobe_agent.py | 32 +++ ooni/scripts/oonireport.py | 238 +++++++++++++++++++++++ ooni/scripts/ooniresources.py | 34 ++++ setup.py | 14 +- 23 files changed, 620 insertions(+), 911 deletions(-)
diff --git a/bin/oonid b/bin/oonid deleted file mode 100755 index dd59eb9..0000000 --- a/bin/oonid +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# XXX This is very Ghetto. The proper way to do this is to make a twisted -# plugin. The plugin will then be installed and can then be run directly with -# `twistd oonid` - -OONID_PATH=`python -c 'import ooni;import os;print os.path.join(os.path.dirname(os.path.abspath(ooni.__file__)), "oonid.py")'` -twistd -y $OONID_PATH diff --git a/bin/oonideckgen b/bin/oonideckgen deleted file mode 100755 index f4dd392..0000000 --- a/bin/oonideckgen +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -import sys -import exceptions - -from twisted.internet import defer, reactor - -from ooni.utils import log -from ooni.deckgen import cli - -exitCode = 128 -def failed(failure): - global exitCode - - r = failure.trap(exceptions.SystemExit, - Exception) - if r != exceptions.SystemExit: - log.err("Failed to run oonideckgen") - log.exception(failure) - exitCode = 127 - else: - exitCode = failure.value.code - reactor.stop() - -def done(result): - global exitCode - - exitCode = 0 - reactor.stop() - -def start(): - d = defer.maybeDeferred(cli.run) - d.addCallback(done) - d.addErrback(failed) - -reactor.callWhenRunning(start) -reactor.run() -sys.exit(exitCode) diff --git a/bin/oonireport b/bin/oonireport deleted file mode 100755 index 9f59683..0000000 --- a/bin/oonireport +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -import sys -import exceptions -from twisted.internet import defer, reactor - -from ooni.utils import log -from ooni.report import cli - -exitCode = 128 - -def failed(failure): - global exitCode - - r = failure.trap(exceptions.SystemExit, - Exception) - if r != exceptions.SystemExit: - log.exception(failure) - log.err("Failed to run oonireport") - exitCode = 127 - else: - exitCode = failure.value.code - reactor.stop() - - -def done(result): - global exitCode - exitCode = 0 - - reactor.stop() - -def start(): - d = defer.maybeDeferred(cli.run) - d.addCallback(done) - d.addErrback(failed) - -reactor.callWhenRunning(start) -reactor.run() -sys.exit(exitCode) diff --git a/bin/ooniresources b/bin/ooniresources deleted file mode 100755 index 6913bb4..0000000 --- a/bin/ooniresources +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -import sys -from twisted.internet import defer, reactor - -from ooni.utils import log -from ooni.resources import cli - -exitCode = 128 -def failed(failure): - global exitCode - - r = failure.trap(exceptions.SystemExit, - Exception) - if r != exceptions.SystemExit: - log.err("Failed to run ooniresources") - log.exception(failure) - exitCode = 127 - else: - exitCode = failure.value.code - reactor.stop() - -def done(result): - global exitCode - - exitCode = 0 - reactor.stop() - -def start(): - d = defer.maybeDeferred(cli.run) - d.addCallback(done) - d.addErrback(done) - -reactor.callWhenRunning(start) -reactor.run() -sys.exit(exitCode) diff --git a/ooni/deckgen/__init__.py b/ooni/deckgen/__init__.py deleted file mode 100644 index d3ec452..0000000 --- a/ooni/deckgen/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.2.0" diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py deleted file mode 100644 index 9f2cc4c..0000000 --- a/ooni/deckgen/cli.py +++ /dev/null @@ -1,190 +0,0 @@ -from __future__ import print_function - -import os -import sys -import copy -import errno -import shutil - -import yaml - -from twisted.internet import defer -from twisted.python import usage - -from ooni import errors -from ooni.geoip import ProbeIP -from ooni.settings import config - -from ooni.deckgen import __version__ -from ooni.deckgen.processors import citizenlab_test_lists -from ooni.resources.update import download_resources - -class Options(usage.Options): - synopsis = """%s [options] - """ % sys.argv[0] - - optParameters = [ - ["country-code", "c", None, - "Specify the two letter country code for which we should " - "generate the deck."], - ["collector", None, None, "Specify a custom collector to use when " - "submitting reports"], - ["bouncer", None, None, "Specify a custom bouncer to use"], - ["output", "o", None, - "Specify the directory where to write output."] - ] - - def opt_version(self): - print("oonideckgen version: %s" % __version__) - sys.exit(0) - - -class Deck(object): - _base_entry = { - "options": { - "test_file": None, - "subargs": [], - "annotations": None, - - "collector": None, - "bouncer": None, - - "reportfile": None, - - "no-collector": 0, - "no-geoip": 0, - "no-yamloo": 0, - "verbose": 0 - } - } - - def __init__(self, collector=None, bouncer=None): - self.deck_entries = [] - self.collector = collector - self.bouncer = bouncer - - def add_test(self, test_file, subargs=[]): - deck_entry = copy.deepcopy(self._base_entry) - deck_entry['options']['collector'] = self.collector - deck_entry['options']['bouncer'] = self.bouncer - deck_entry['options']['test_file'] = test_file - deck_entry['options']['subargs'] = subargs - self.deck_entries.append(deck_entry) - - def pprint(self): - print(yaml.safe_dump(self.deck_entries)) - - def write_to_file(self, filename): - with open(filename, "w+") as f: - f.write(yaml.safe_dump(self.deck_entries)) - - -def generate_deck(options): - url_list_country = None - try: - url_list_country = citizenlab_test_lists.generate_country_input( - options['country-code'], - options['output'] - ) - except Exception: - print("Could not generate country specific url list") - print("We will just use the global one.") - - url_list_global = citizenlab_test_lists.generate_global_input( - options['output'] - ) - - deck = Deck(collector=options['collector'], bouncer=options['bouncer']) - deck.add_test('manipulation/http_invalid_request_line') - deck.add_test('manipulation/http_header_field_manipulation') - - if url_list_country is not None: - deck.add_test('blocking/web_connectivity', ['-f', url_list_country]) - deck.add_test('blocking/web_connectivity', ['-f', url_list_global]) - - if config.advanced.debug: - deck.pprint() - deck_filename = os.path.join(options['output'], - "default-user.deck") - deck.write_to_file(deck_filename) - print("Deck written to %s" % deck_filename) - print("Run ooniprobe like so:") - print("ooniprobe -i %s" % deck_filename) - - -@defer.inlineCallbacks -def get_user_country_code(): - config.privacy.includecountry = True - probe_ip = ProbeIP() - yield probe_ip.lookup() - defer.returnValue(probe_ip.geodata['countrycode']) - -def resources_up_to_date(): - if config.get_data_file_path("GeoIP/GeoIP.dat") is None: - return False - - if config.get_data_file_path("resources/" - "namebench-dns-servers.csv") is None: - return False - - if config.get_data_file_path("resources/" - "citizenlab-test-lists/" - "global.csv") is None: - return False - - return True - -@defer.inlineCallbacks -def run(): - options = Options() - try: - options.parseOptions() - except usage.UsageError as error_message: - print("%s: %s" % (sys.argv[0], error_message)) - print(options) - sys.exit(1) - - if not resources_up_to_date(): - print("Resources for running ooniprobe are not up to date.") - print("Will update them now.") - yield download_resources() - - if not options['output']: - options['output'] = os.getcwd() - - if not options['country-code']: - try: - options['country-code'] = yield get_user_country_code() - except errors.ProbeIPUnknown: - print("Could not determine your IP address.") - print("Check your internet connection or specify a country code " - "with -c.") - sys.exit(4) - - if len(options['country-code']) != 2: - print("%s: --country-code must be 2 characters" % sys.argv[0]) - sys.exit(2) - - if not os.path.isdir(options['output']): - print("%s: %s is not a directory" % (sys.argv[0], - options['output'])) - sys.exit(3) - - options['country-code'] = options['country-code'].lower() - - output_dir = os.path.abspath(options['output']) - output_dir = os.path.join(output_dir, "deck") - - if os.path.isdir(output_dir): - print("Found previous deck deleting content of it") - shutil.rmtree(output_dir) - - options['output'] = output_dir - - try: - os.makedirs(options['output']) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - - generate_deck(options) diff --git a/ooni/deckgen/processors/__init__.py b/ooni/deckgen/processors/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py deleted file mode 100644 index 8a660cf..0000000 --- a/ooni/deckgen/processors/citizenlab_test_lists.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import csv -from ooni.settings import config - - -def load_input(file_input, file_output): - fw = open(file_output, "w+") - with open(file_input) as f: - csvreader = csv.reader(f) - csvreader.next() - for row in csvreader: - fw.write("%s\n" % row[0]) - fw.close() - - -def generate_country_input(country_code, dst): - """ - Write to dst/citizenlab-urls-{country_code}.txt - the list for the given country code. - - Returns: - - the path to the generated input - """ - - country_code = country_code.lower() - filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code) - - input_list = config.get_data_file_path("resources/" - "citizenlab-test-lists/" - + country_code + ".csv") - - if not input_list: - raise Exception("Could not find list for country %s" % country_code) - - load_input(input_list, filename) - - return filename - - -def generate_global_input(dst): - filename = os.path.join(dst, "citizenlab-urls-global.txt") - - input_list = config.get_data_file_path("resources/" - "citizenlab-test-lists/" - "global.csv") - - if not input_list: - print("Could not find the global input list") - print("Perhaps you should run ooniresources") - raise Exception("Could not find the global input list") - - load_input(input_list, filename) - - return filename diff --git a/ooni/deckgen/processors/namebench_dns_servers.py b/ooni/deckgen/processors/namebench_dns_servers.py deleted file mode 100644 index 9855611..0000000 --- a/ooni/deckgen/processors/namebench_dns_servers.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -import csv -import GeoIP - -from ooni.settings import config - - -class GeoIPDB(object): - _borg = {} - country = None - - def __init__(self): - self.__dict__ = self._borg - if not self.country: - try: - country_file = config.get_data_file_path('GeoIP/GeoIP.dat') - self.country = GeoIP.open(country_file, - GeoIP.GEOIP_STANDARD) - except: - raise Exception("Edit the geoip_data_dir line in your config" - " file to point to your geoip files") - - -def generate_country_input(country_code, dst): - - csv_file = config.get_data_file_path("resources/" - "namebench-dns-servers.csv") - - filename = os.path.join(dst, "dns-server-%s.txt" % country_code) - fw = open(filename, "w") - geoip_db = GeoIPDB() - reader = csv.reader(open(csv_file)) - for row in reader: - if row[2] == 'X-Internal-IP': - continue - elif row[2] == 'X-Unroutable': - continue - elif row[2] == 'X-Link_local': - continue - ipaddr = row[0] - cc = geoip_db.country.country_code_by_addr(ipaddr) - if not cc: - continue - if cc.lower() == country_code.lower(): - fw.write(ipaddr + "\n") - fw.close() - return filename - - -def generate_global_input(dst): - pass diff --git a/ooni/report/__init__.py b/ooni/report/__init__.py deleted file mode 100644 index 3dc1f76..0000000 --- a/ooni/report/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.1.0" diff --git a/ooni/report/cli.py b/ooni/report/cli.py deleted file mode 100644 index 485ab52..0000000 --- a/ooni/report/cli.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import print_function - -import os -import sys - -from ooni.constants import CANONICAL_BOUNCER_ONION -from ooni.report import __version__ -from ooni.report import tool -from ooni.settings import config - -from twisted.python import usage - - -class Options(usage.Options): - - synopsis = """%s [options] upload | status -""" % (os.path.basename(sys.argv[0]),) - - optFlags = [ - ["default-collector", "d", "Upload the reports to the default " - "collector that is looked up with the " - "canonical bouncer."] - ] - - optParameters = [ - ["configfile", "f", None, - "Specify the configuration file to use."], - ["collector", "c", None, - "Specify the collector to upload the result to."], - ["bouncer", "b", None, - "Specify the bouncer to query for a collector."] - ] - - def opt_version(self): - print("oonireport version: %s" % __version__) - sys.exit(0) - - def parseArgs(self, *args): - if len(args) == 0: - raise usage.UsageError( - "Must specify at least one command" - ) - return - self['command'] = args[0] - if self['command'] not in ("upload", "status"): - raise usage.UsageError( - "Must specify either command upload or status" - ) - if self['command'] == "upload": - try: - self['report_file'] = args[1] - except IndexError: - self['report_file'] = None - - -def tor_check(): - if not config.tor.socks_port: - print("Currently oonireport requires that you start Tor yourself " - "and set the socks_port inside of ooniprobe.conf") - sys.exit(1) - - -def run(args=sys.argv[1:]): - options = Options() - try: - options.parseOptions(args) - except Exception as exc: - print("Error: %s" % exc) - print(options) - sys.exit(2) - config.global_options = dict(options) - config.set_paths() - config.read_config_file() - - if options['default-collector']: - options['bouncer'] = CANONICAL_BOUNCER_ONION - - if options['command'] == "upload" and options['report_file']: - tor_check() - return tool.upload(options['report_file'], - options['collector'], - options['bouncer']) - elif options['command'] == "upload": - tor_check() - return tool.upload_all(options['collector'], - options['bouncer']) - elif options['command'] == "status": - return tool.status() - else: - print(options) diff --git a/ooni/report/parser.py b/ooni/report/parser.py deleted file mode 100644 index bef948c..0000000 --- a/ooni/report/parser.py +++ /dev/null @@ -1,35 +0,0 @@ -import yaml - - -class ReportLoader(object): - _header_keys = ( - 'probe_asn', - 'probe_cc', - 'probe_ip', - 'start_time', - 'test_name', - 'test_version', - 'options', - 'input_hashes', - 'software_name', - 'software_version' - ) - - def __init__(self, report_filename): - self._fp = open(report_filename) - self._yfp = yaml.safe_load_all(self._fp) - - self.header = self._yfp.next() - - def __iter__(self): - return self - - def next(self): - try: - return self._yfp.next() - except StopIteration: - self.close() - raise StopIteration - - def close(self): - self._fp.close() diff --git a/ooni/report/tool.py b/ooni/report/tool.py deleted file mode 100644 index f8af132..0000000 --- a/ooni/report/tool.py +++ /dev/null @@ -1,117 +0,0 @@ -from __future__ import print_function -import yaml -import sys - -from twisted.internet import defer - -from ooni.constants import CANONICAL_BOUNCER_ONION -from ooni.reporter import OONIBReporter, OONIBReportLog - -from ooni.utils import log -from ooni.report import parser -from ooni.settings import config -from ooni.backend_client import BouncerClient, CollectorClient - -@defer.inlineCallbacks -def lookup_collector_client(report_header, bouncer): - oonib_client = BouncerClient(bouncer) - net_tests = [{ - 'test-helpers': [], - 'input-hashes': report_header['input_hashes'], - 'name': report_header['test_name'], - 'version': report_header['test_version'], - }] - result = yield oonib_client.lookupTestCollector( - net_tests - ) - collector_client = CollectorClient( - address=result['net-tests'][0]['collector'] - ) - defer.returnValue(collector_client) - -@defer.inlineCallbacks -def upload(report_file, collector=None, bouncer=None): - oonib_report_log = OONIBReportLog() - collector_client = None - if collector: - collector_client = CollectorClient(address=collector) - - log.msg("Attempting to upload %s" % report_file) - - with open(config.report_log_file) as f: - report_log = yaml.safe_load(f) - - report = parser.ReportLoader(report_file) - if bouncer and collector_client is None: - collector_client = yield lookup_collector_client(report.header, - bouncer) - - if collector_client is None: - try: - collector_settings = report_log[report_file]['collector'] - if collector_settings is None: - log.msg("Skipping uploading of %s since this measurement " - "was run by specifying no collector." % - report_file) - defer.returnValue(None) - elif isinstance(collector_settings, dict): - collector_client = CollectorClient(settings=collector_settings) - elif isinstance(collector_settings, str): - collector_client = CollectorClient(address=collector_settings) - except KeyError: - log.msg("Could not find %s in reporting.yaml. Looking up " - "collector with canonical bouncer." % report_file) - collector_client = yield lookup_collector_client(report.header, - CANONICAL_BOUNCER_ONION) - - oonib_reporter = OONIBReporter(report.header, collector_client) - log.msg("Creating report for %s with %s" % (report_file, - collector_client.settings)) - report_id = yield oonib_reporter.createReport() - report.header['report_id'] = report_id - yield oonib_report_log.created(report_file, - collector_client.settings, - report_id) - log.msg("Writing report entries") - for entry in report: - yield oonib_reporter.writeReportEntry(entry) - sys.stdout.write('.') - sys.stdout.flush() - log.msg("Closing report") - yield oonib_reporter.finish() - yield oonib_report_log.closed(report_file) - - -@defer.inlineCallbacks -def upload_all(collector=None, bouncer=None): - oonib_report_log = OONIBReportLog() - - for report_file, value in oonib_report_log.reports_to_upload: - try: - yield upload(report_file, collector, bouncer) - except Exception as exc: - log.exception(exc) - - -def print_report(report_file, value): - print("* %s" % report_file) - print(" %s" % value['created_at']) - - -def status(): - oonib_report_log = OONIBReportLog() - - print("Reports to be uploaded") - print("----------------------") - for report_file, value in oonib_report_log.reports_to_upload: - print_report(report_file, value) - - print("Reports in progress") - print("-------------------") - for report_file, value in oonib_report_log.reports_in_progress: - print_report(report_file, value) - - print("Incomplete reports") - print("------------------") - for report_file, value in oonib_report_log.reports_incomplete: - print_report(report_file, value) diff --git a/ooni/resources.py b/ooni/resources.py new file mode 100644 index 0000000..d49e679 --- /dev/null +++ b/ooni/resources.py @@ -0,0 +1,152 @@ +import json + +from twisted.python.filepath import FilePath +from twisted.internet import defer +from twisted.web.client import downloadPage, getPage + +from ooni.utils import log +from ooni.settings import config + +class UpdateFailure(Exception): + pass + +def get_download_url(tag_name, filename): + return ("https://github.com/OpenObservatory/ooni-resources/releases" + "/download/{0}/{1}".format(tag_name, filename)) + +def get_current_version(): + manifest = FilePath(config.resources_directory).child("manifest.json") + if not manifest.exists(): + return 0 + with manifest.open("r") as f: + manifest = json.load(f) + return int(manifest["version"]) + +@defer.inlineCallbacks +def get_latest_version(): + """ + Fetches the latest version of the resources package. + :return: (int) the latest version number + """ + try: + version = yield getPage(get_download_url("latest", "version")) + except Exception as exc: + raise exc + defer.returnValue(int(version.strip())) + + +def get_out_of_date_resources(current_manifest, new_manifest, + country_code=None): + current_res = {} + new_res = {} + for r in current_manifest["resources"]: + current_res[r["path"]] = r + + for r in new_manifest["resources"]: + new_res[r["path"]] = r + + paths_to_delete = [ + current_res[path] for path in list(set(current_res.keys()) - + set(new_res.keys())) + ] + paths_to_update = [] + _resources = FilePath(config.resources_directory) + for path, info in new_res.items(): + if (country_code is not None and + info["country_code"] != "ALL" and + info["country_code"] != country_code): + continue + if current_res.get(path, None) is None: + paths_to_update.append(info) + elif current_res[path]["version"] < info["version"]: + paths_to_update.append(info) + else: + pre_path, filename = info["path"].split("/") + # Also perform an update when it doesn't exist on disk, although + # the manifest claims we have a more up to date version. + # This happens if an update by country_code happened and a new + # country code is now required. + if not _resources.child(pre_path).child(filename).exists(): + paths_to_update.append(info) + + return paths_to_update, paths_to_delete + +@defer.inlineCallbacks +def check_for_update(country_code=None): + """ + Checks if we need to update the resources. + If the country_code is specified then only the resources for that + country will be updated/downloaded. + :return: the latest version. + """ + temporary_files = [] + def cleanup(): + # If we fail we need to delete all the temporary files + for _, src_file_path in temporary_files: + src_file_path.remove() + + current_version = get_current_version() + latest_version = yield get_latest_version() + + # We are already at the latest version + if current_version == latest_version: + defer.returnValue(latest_version) + + resources_dir = FilePath(config.resources_directory) + resources_dir.makedirs(ignoreExistingDirectory=True) + current_manifest = resources_dir.child("manifest.json") + + new_manifest = current_manifest.temporarySibling() + new_manifest.alwaysCreate = 0 + + temporary_files.append((current_manifest, new_manifest)) + + try: + yield downloadPage( + get_download_url(latest_version, "manifest.json"), + new_manifest.path + ) + except: + cleanup() + raise UpdateFailure("Failed to download manifest") + + new_manifest_data = json.loads(new_manifest.getContent()) + + if current_manifest.exists(): + with current_manifest.open("r") as f: + current_manifest_data = json.loads(f) + else: + current_manifest_data = { + "resources": [] + } + + to_update, to_delete = get_out_of_date_resources( + current_manifest_data, new_manifest_data, country_code) + + try: + for resource in to_update: + pre_path, filename = resource["path"].split("/") + dst_file = resources_dir.child(pre_path).child(filename) + dst_file.parent().makedirs(ignoreExistingDirectory=True) + src_file = dst_file.temporarySibling() + src_file.alwaysCreate = 0 + + temporary_files.append((dst_file, src_file)) + # The paths for the download require replacing "/" with "." + download_url = get_download_url(latest_version, + resource["path"].replace("/", ".")) + print("Downloading {0}".format(download_url)) + yield downloadPage(download_url, src_file.path) + except Exception as exc: + cleanup() + log.exception(exc) + raise UpdateFailure("Failed to download resource {0}".format(resource["path"])) + + for dst_file, src_file in temporary_files: + log.msg("Moving {0} to {1}".format(src_file.path, + dst_file.path)) + src_file.moveTo(dst_file) + + for resource in to_delete: + log.msg("Deleting old resources") + resources_dir.child(resource["path"]).remove() diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py deleted file mode 100644 index 6550887..0000000 --- a/ooni/resources/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -import json - -from twisted.python.filepath import FilePath - -from ooni import __resources_version__ as resources_version -from ooni.settings import config - -ooni_resources_url = ("https://github.com/TheTorProject/ooni-probe/releases" - "/download/v{}/" - "ooni-resources.tar.gz").format(resources_version) - -def get_download_url(tag_name, filename): - return ("https://github.com/OpenObservatory/ooni-resources/releases" - "/download/{0}/{1}".format(tag_name, filename)) - -def get_current_version(): - manifest = FilePath(config.resources_directory).child("manifest.json") - if not manifest.exists(): - return 0 - with manifest.open("r") as f: - manifest = json.load(f) - return int(manifest["version"]) diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py deleted file mode 100644 index 0d7832a..0000000 --- a/ooni/resources/cli.py +++ /dev/null @@ -1,44 +0,0 @@ -import sys - -from twisted.internet import defer -from twisted.python import usage - -from ooni.resources import __version__ -from ooni.resources import update - - -class Options(usage.Options): - synopsis = """%s - This is used to update the resources required to run oonideckgen and - ooniprobe. - You just run this script with no arguments and it will update the - resources. - """ % sys.argv[0] - - optFlags = [ - ["update-inputs", None, "(deprecated) update the resources needed for " - "inputs."], - ["update-geoip", None, "(deprecated) Update the geoip related " - "resources."] - ] - optParameters = [] - - def opt_version(self): - print("ooniresources version: %s" % __version__) - sys.exit(0) - - -@defer.inlineCallbacks -def run(): - options = Options() - try: - options.parseOptions() - except usage.UsageError as error_message: - print "%s: %s" % (sys.argv[0], error_message) - print "%s: Try --help for usage details." % (sys.argv[0]) - sys.exit(1) - - if options['update-inputs'] or options['update-geoip']: - print("WARNING: Passing command line arguments is deprecated") - - yield update.download_resources() diff --git a/ooni/resources/update.py b/ooni/resources/update.py deleted file mode 100644 index b464920..0000000 --- a/ooni/resources/update.py +++ /dev/null @@ -1,187 +0,0 @@ -import os -import json -import tarfile -import tempfile - -from twisted.python.filepath import FilePath -from twisted.internet import defer -from twisted.web.client import downloadPage, getPage - -from ooni.utils import log -from ooni.settings import config -from ooni.resources import ooni_resources_url, get_download_url -from ooni.resources import get_current_version - -class UpdateFailure(Exception): - pass - -@defer.inlineCallbacks -def get_latest_version(): - """ - Fetches the latest version of the resources package. - :return: (int) the latest version number - """ - try: - version = yield getPage(get_download_url("latest", "version")) - except Exception as exc: - raise exc - defer.returnValue(int(version.strip())) - - -def get_out_of_date_resources(current_manifest, new_manifest, - country_code=None): - current_res = {} - new_res = {} - for r in current_manifest["resources"]: - current_res[r["path"]] = r - - for r in new_manifest["resources"]: - new_res[r["path"]] = r - - paths_to_delete = [ - current_res[path] for path in list(set(current_res.keys()) - - set(new_res.keys())) - ] - paths_to_update = [] - _resources = FilePath(config.resources_directory) - for path, info in new_res.items(): - if (country_code is not None and - info["country_code"] != "ALL" and - info["country_code"] != country_code): - continue - if current_res[path]["version"] < info["version"]: - paths_to_update.append(info) - else: - pre_path, filename = info["path"].split("/") - # Also perform an update when it doesn't exist on disk, although - # the manifest claims we have a more up to date version. - # This happens if an update by country_code happened and a new - # country code is now required. - if not _resources.child(pre_path).child(filename).exists(): - paths_to_update.append(info) - - return paths_to_update, paths_to_delete - -@defer.inlineCallbacks -def check_for_update(country_code=None): - """ - Checks if we need to update the resources. - If the country_code is specified then only the resources for that - country will be updated/downloaded. - :return: the latest version. - """ - temporary_files = [] - def cleanup(): - # If we fail we need to delete all the temporary files - for _, src_file_path in temporary_files: - src_file_path.remove() - - current_version = get_current_version() - latest_version = yield get_latest_version() - - # We are already at the latest version - if current_version == latest_version: - defer.returnValue(latest_version) - - resources_dir = FilePath(config.resources_directory) - resources_dir.makedirs(ignoreExistingDirectory=True) - current_manifest = resources_dir.child("manifest.json") - - new_manifest = current_manifest.temporarySibling() - new_manifest.alwaysCreate = 0 - - temporary_files.append((current_manifest, new_manifest)) - - try: - yield downloadPage( - get_download_url(latest_version, "manifest.json"), - new_manifest.path - ) - except: - cleanup() - raise UpdateFailure("Failed to download manifest") - - new_manifest_data = json.loads(new_manifest.getContent()) - - to_update = new_manifest_data["resources"] - to_delete = [] - if current_manifest.exists(): - with current_manifest.open("r") as f: - current_manifest_data = json.loads(f) - to_update, to_delete = get_out_of_date_resources( - current_manifest_data, new_manifest_data, country_code) - - try: - for resource in to_update: - pre_path, filename = resource["path"].split("/") - dst_file = resources_dir.child(pre_path).child(filename) - dst_file.parent().makedirs(ignoreExistingDirectory=True) - src_file = dst_file.temporarySibling() - src_file.alwaysCreate = 0 - - temporary_files.append((dst_file, src_file)) - # The paths for the download require replacing "/" with "." - download_url = get_download_url(latest_version, - resource["path"].replace("/", ".")) - print("Downloading {0}".format(download_url)) - yield downloadPage(download_url, src_file.path) - except Exception as exc: - cleanup() - log.exception(exc) - raise UpdateFailure("Failed to download resource {0}".format(resource["path"])) - - for dst_file, src_file in temporary_files: - log.msg("Moving {0} to {1}".format(src_file.path, - dst_file.path)) - src_file.moveTo(dst_file) - - for resource in to_delete: - log.msg("Deleting old resources") - resources_dir.child(resource["path"]).remove() - -@defer.inlineCallbacks -def download_resources(): - if os.access(config.var_lib_path, os.W_OK): - dst_directory = FilePath(config.var_lib_path) - else: - dst_directory = FilePath(config.ooni_home) - - print("Downloading {} to {}".format(ooni_resources_url, - dst_directory.path)) - tmp_download_directory = FilePath(tempfile.mkdtemp()) - tmp_download_filename = tmp_download_directory.temporarySibling() - - - try: - yield downloadPage(ooni_resources_url, tmp_download_filename.path) - ooni_resources_tar_gz = tarfile.open(tmp_download_filename.path) - ooni_resources_tar_gz.extractall(tmp_download_directory.path) - - if not tmp_download_directory.child('GeoIP').exists(): - raise Exception("Could not find GeoIP data files in downloaded " - "tar.") - - if not tmp_download_directory.child('resources').exists(): - raise Exception("Could not find resources data files in " - "downloaded tar.") - - geoip_dir = dst_directory.child('GeoIP') - resources_dir = dst_directory.child('resources') - - if geoip_dir.exists(): - geoip_dir.remove() - tmp_download_directory.child('GeoIP').moveTo(geoip_dir) - - if resources_dir.exists(): - resources_dir.remove() - tmp_download_directory.child('resources').moveTo(resources_dir) - - print("Written GeoIP files to {}".format(geoip_dir.path)) - print("Written resources files to {}".format(resources_dir.path)) - - except Exception as exc: - print("Failed to download resources!") - raise exc - - finally: - tmp_download_directory.remove() diff --git a/ooni/scripts/__init__.py b/ooni/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py new file mode 100644 index 0000000..fa675f9 --- /dev/null +++ b/ooni/scripts/oonideckgen.py @@ -0,0 +1,151 @@ +from __future__ import print_function + +import errno +import os +import shutil +import sys + +from twisted.internet import defer, task +from twisted.python import usage + +from ooni.otime import prettyDateNowUTC +from ooni import errors +from ooni.geoip import ProbeIP +from ooni.resources import check_for_update +from ooni.settings import config +from ooni.deck import NGDeck + +__version__ = "1.0.0" + +class Options(usage.Options): + synopsis = """%s [options] + """ % sys.argv[0] + + optParameters = [ + ["country-code", "c", None, + "Specify the two letter country code for which we should " + "generate the deck."], + ["collector", None, None, "Specify a custom collector to use when " + "submitting reports"], + ["bouncer", None, None, "Specify a custom bouncer to use"], + ["output", "o", None, + "Specify the directory where to write output."] + ] + + def opt_version(self): + print("oonideckgen version: " % __version__) + sys.exit(0) + +def generate_deck(options): + + deck_data = { + "name": "Default ooniprobe deck", + "description": "Default ooniprobe deck generated on {0}".format( + prettyDateNowUTC()), + "schedule": "@daily", + "tasks": [ + { + "ooni": { + "test_name": "http_invalid_request_line" + }, + }, + { + "ooni": { + "test_name": "http_header_field_manipulation" + }, + }, + { + "ooni": { + "test_name": "web_connectivity", + "file": "$citizenlab_${probe_cc}_urls" + }, + }, + { + "ooni": { + "test_name": "web_connectivity", + "file": "$citizenlab_global_urls" + } + } + ] + } + if options["collector"]: + deck_data["collector"] = options['collector'] + + if options["bouncer"]: + deck_data["bouncer"] = options['bouncer'] + + deck = NGDeck(deck_data=deck_data) + with open(options['output']) as fw: + deck.write(fw) + + print("Deck written to {0}".format(options['output'])) + print("Run ooniprobe like so:") + print("ooniprobe -i {0}".format(options['output'])) + + +@defer.inlineCallbacks +def get_user_country_code(): + config.privacy.includecountry = True + probe_ip = ProbeIP() + yield probe_ip.lookup() + defer.returnValue(probe_ip.geodata['countrycode']) + + +@defer.inlineCallbacks +def oonideckgen(): + options = Options() + try: + options.parseOptions() + except usage.UsageError as error_message: + print("%s: %s" % (sys.argv[0], error_message)) + print(options) + sys.exit(1) + + print("Checking for update of resources") + yield check_for_update() + + if not options['output']: + options['output'] = os.getcwd() + + if not options['country-code']: + try: + options['country-code'] = yield get_user_country_code() + except errors.ProbeIPUnknown: + print("Could not determine your IP address.") + print("Check your internet connection or specify a country code " + "with -c.") + sys.exit(4) + + if len(options['country-code']) != 2: + print("%s: --country-code must be 2 characters" % sys.argv[0]) + sys.exit(2) + + if not os.path.isdir(options['output']): + print("%s: %s is not a directory" % (sys.argv[0], + options['output'])) + sys.exit(3) + + options['country-code'] = options['country-code'].lower() + + output_dir = os.path.abspath(options['output']) + output_dir = os.path.join(output_dir, "deck") + + if os.path.isdir(output_dir): + print("Found previous deck deleting content of it") + shutil.rmtree(output_dir) + + options['output'] = output_dir + + try: + os.makedirs(options['output']) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise + + generate_deck(options) + +def run(): + task.react(oonideckgen) + +if __name__ == "__main__": + run() diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py new file mode 100644 index 0000000..7833308 --- /dev/null +++ b/ooni/scripts/ooniprobe_agent.py @@ -0,0 +1,32 @@ +from twisted.scripts import twistd +from twisted.python import usage + +from ooni.agent.agent import AgentService + +class StartOoniprobeAgentPlugin: + tapname = "ooniprobe" + + def makeService(self, so): + return AgentService() + +class OoniprobeTwistdConfig(twistd.ServerOptions): + subCommands = [ + ("StartOoniprobeAgent", None, usage.Options, "ooniprobe agent") + ] + +def run(): + twistd_args = ["--nodaemon"] + twistd_config = OoniprobeTwistdConfig() + twistd_args.append("StartOoniprobeAgent") + try: + twistd_config.parseOptions(twistd_args) + except usage.error, ue: + print("ooniprobe: usage error from twistd: {}\n".format(ue)) + twistd_config.loadedPlugins = { + "StartOoniprobeAgent": StartOoniprobeAgentPlugin() + } + twistd.runApp(twistd_config) + return 0 + +if __name__ == "__main__": + run() diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py new file mode 100644 index 0000000..6facadf --- /dev/null +++ b/ooni/scripts/oonireport.py @@ -0,0 +1,238 @@ +from __future__ import print_function + +import os +import sys +import yaml + +from twisted.python import usage +from twisted.internet import defer, task + +from ooni.constants import CANONICAL_BOUNCER_ONION +from ooni.reporter import OONIBReporter, OONIBReportLog + +from ooni.utils import log +from ooni.settings import config +from ooni.backend_client import BouncerClient, CollectorClient + +__version__ = "0.1.0" + +@defer.inlineCallbacks +def lookup_collector_client(report_header, bouncer): + oonib_client = BouncerClient(bouncer) + net_tests = [{ + 'test-helpers': [], + 'input-hashes': report_header['input_hashes'], + 'name': report_header['test_name'], + 'version': report_header['test_version'], + }] + result = yield oonib_client.lookupTestCollector( + net_tests + ) + collector_client = CollectorClient( + address=result['net-tests'][0]['collector'] + ) + defer.returnValue(collector_client) + +@defer.inlineCallbacks +def upload(report_file, collector=None, bouncer=None): + oonib_report_log = OONIBReportLog() + collector_client = None + if collector: + collector_client = CollectorClient(address=collector) + + log.msg("Attempting to upload %s" % report_file) + + with open(config.report_log_file) as f: + report_log = yaml.safe_load(f) + + report = ReportLoader(report_file) + if bouncer and collector_client is None: + collector_client = yield lookup_collector_client(report.header, + bouncer) + + if collector_client is None: + try: + collector_settings = report_log[report_file]['collector'] + if collector_settings is None: + log.msg("Skipping uploading of %s since this measurement " + "was run by specifying no collector." % + report_file) + defer.returnValue(None) + elif isinstance(collector_settings, dict): + collector_client = CollectorClient(settings=collector_settings) + elif isinstance(collector_settings, str): + collector_client = CollectorClient(address=collector_settings) + except KeyError: + log.msg("Could not find %s in reporting.yaml. Looking up " + "collector with canonical bouncer." % report_file) + collector_client = yield lookup_collector_client(report.header, + CANONICAL_BOUNCER_ONION) + + oonib_reporter = OONIBReporter(report.header, collector_client) + log.msg("Creating report for %s with %s" % (report_file, + collector_client.settings)) + report_id = yield oonib_reporter.createReport() + report.header['report_id'] = report_id + yield oonib_report_log.created(report_file, + collector_client.settings, + report_id) + log.msg("Writing report entries") + for entry in report: + yield oonib_reporter.writeReportEntry(entry) + sys.stdout.write('.') + sys.stdout.flush() + log.msg("Closing report") + yield oonib_reporter.finish() + yield oonib_report_log.closed(report_file) + + +@defer.inlineCallbacks +def upload_all(collector=None, bouncer=None): + oonib_report_log = OONIBReportLog() + + for report_file, value in oonib_report_log.reports_to_upload: + try: + yield upload(report_file, collector, bouncer) + except Exception as exc: + log.exception(exc) + + +def print_report(report_file, value): + print("* %s" % report_file) + print(" %s" % value['created_at']) + + +def status(): + oonib_report_log = OONIBReportLog() + + print("Reports to be uploaded") + print("----------------------") + for report_file, value in oonib_report_log.reports_to_upload: + print_report(report_file, value) + + print("Reports in progress") + print("-------------------") + for report_file, value in oonib_report_log.reports_in_progress: + print_report(report_file, value) + + print("Incomplete reports") + print("------------------") + for report_file, value in oonib_report_log.reports_incomplete: + print_report(report_file, value) + +class ReportLoader(object): + _header_keys = ( + 'probe_asn', + 'probe_cc', + 'probe_ip', + 'start_time', + 'test_name', + 'test_version', + 'options', + 'input_hashes', + 'software_name', + 'software_version' + ) + + def __init__(self, report_filename): + self._fp = open(report_filename) + self._yfp = yaml.safe_load_all(self._fp) + + self.header = self._yfp.next() + + def __iter__(self): + return self + + def next(self): + try: + return self._yfp.next() + except StopIteration: + self.close() + raise StopIteration + + def close(self): + self._fp.close() + +class Options(usage.Options): + + synopsis = """%s [options] upload | status +""" % (os.path.basename(sys.argv[0]),) + + optFlags = [ + ["default-collector", "d", "Upload the reports to the default " + "collector that is looked up with the " + "canonical bouncer."] + ] + + optParameters = [ + ["configfile", "f", None, + "Specify the configuration file to use."], + ["collector", "c", None, + "Specify the collector to upload the result to."], + ["bouncer", "b", None, + "Specify the bouncer to query for a collector."] + ] + + def opt_version(self): + print("oonireport version: %s" % __version__) + sys.exit(0) + + def parseArgs(self, *args): + if len(args) == 0: + raise usage.UsageError( + "Must specify at least one command" + ) + return + self['command'] = args[0] + if self['command'] not in ("upload", "status"): + raise usage.UsageError( + "Must specify either command upload or status" + ) + if self['command'] == "upload": + try: + self['report_file'] = args[1] + except IndexError: + self['report_file'] = None + + +def tor_check(): + if not config.tor.socks_port: + print("Currently oonireport requires that you start Tor yourself " + "and set the socks_port inside of ooniprobe.conf") + sys.exit(1) + + +def oonireport(args=sys.argv[1:]): + options = Options() + try: + options.parseOptions(args) + except Exception as exc: + print("Error: %s" % exc) + print(options) + sys.exit(2) + config.global_options = dict(options) + config.set_paths() + config.read_config_file() + + if options['default-collector']: + options['bouncer'] = CANONICAL_BOUNCER_ONION + + if options['command'] == "upload" and options['report_file']: + tor_check() + return upload(options['report_file'], + options['collector'], + options['bouncer']) + elif options['command'] == "upload": + tor_check() + return upload_all(options['collector'], + options['bouncer']) + elif options['command'] == "status": + return status() + else: + print(options) + +def run(): + task.react(oonireport) + +if __name__ == "__main__": + run() diff --git a/ooni/scripts/ooniresources.py b/ooni/scripts/ooniresources.py new file mode 100644 index 0000000..5fdbbf0 --- /dev/null +++ b/ooni/scripts/ooniresources.py @@ -0,0 +1,34 @@ +import sys + +from twisted.python import usage + +class Options(usage.Options): + synopsis = """%s + [DEPRECATED] Usage of this script is deprecated and it will be deleted + in future versions of ooniprobe. + """ % sys.argv[0] + + optFlags = [ + ["update-inputs", None, "(deprecated) update the resources needed for " + "inputs."], + ["update-geoip", None, "(deprecated) Update the geoip related " + "resources."] + ] + optParameters = [] + + def opt_version(self): + print("ooniresources version: 0.2.0") + sys.exit(0) + + +def run(): + options = Options() + try: + options.parseOptions() + except usage.UsageError as error_message: + print "%s: %s" % (sys.argv[0], error_message) + print "%s: Try --help for usage details." % (sys.argv[0]) + sys.exit(1) + + print("WARNING: Usage of this script is deprecated.") + sys.exit(0) diff --git a/setup.py b/setup.py index 1a24460..d401a65 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,6 @@ Have fun!
from __future__ import print_function
-from ooni import __version__, __author__ import os import sys import glob @@ -102,6 +101,9 @@ from setuptools import setup, Command from setuptools.command.install import install from distutils.spawn import find_executable
+from ooni import __version__, __author__ +from ooni.scripts import ooniresources + GEOIP_ASN_URL = "https://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz" GEOIP_URL = "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat..." TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip" @@ -393,6 +395,16 @@ setup( dependency_links=dependency_links, install_requires=install_requires, zip_safe=False, + entry_points={ + 'console_scripts': [ + 'ooniresources = ooni.scripts.ooniresources:run', # This is deprecated + 'oonideckgen = ooni.scripts.oonideckgen:run', # This is deprecated + + 'ooniprobe = ooni.scripts.ooniprobe:run', + 'oonireport = ooni.scripts.oonireport:run', + 'ooniprobe-agent = ooni.scripts.ooniprobe_agent:run' + ] + }, cmdclass={ "install": OoniInstall, "create_ooniresources": CreateOoniResources,