commit 198681983304f77c0cc20a40b3c31ab66fc699d9 Author: kudrom <kudrom@riseup.net> Date: Wed Aug 27 17:40:28 2014 +0200 Review of deckgen branch --- ooni/deckgen/cli.py | 15 +++-------- ooni/deckgen/processors/citizenlab_test_lists.py | 29 +++++++++++----------- ooni/deckgen/processors/namebench_dns_servers.py | 1 - ooni/resources/cli.py | 2 -- ooni/resources/update.py | 9 +++++++ ooni/utils/__init__.py | 4 ++- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index 95e0c98..77f5933 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -5,7 +5,7 @@ import errno import yaml -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.python import usage from ooni.geoip import ProbeIP @@ -36,7 +36,7 @@ class Options(usage.Options): sys.exit(0) -class Deck(): +class Deck(object): _base_entry = { "options": { "collector": None, @@ -105,7 +105,8 @@ def generate_deck(options): deck.add_test('manipulation/http_invalid_request_line') deck.add_test('manipulation/http_header_field_manipulation') # deck.add_test('manipulation/traceroute') - deck.pprint() + if config.advanced.debug: + deck.pprint() deck_filename = os.path.join(options['output'], "%s-%s-user.deck" % (__version__, options['country-code'])) @@ -163,11 +164,3 @@ def run(): raise generate_deck(options) - -if __name__ == "__main__": - d = run() - - @d.addBoth - def cb(_): - reactor.stop() - reactor.start() diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py index 645d714..b36e726 100644 --- a/ooni/deckgen/processors/citizenlab_test_lists.py +++ b/ooni/deckgen/processors/citizenlab_test_lists.py @@ -2,6 +2,17 @@ import os import csv from ooni.settings import config + +def load_input(file_input, file_output): + fw = open(file_output, "w+") + with open(file_input) as f: + csvreader = csv.reader(f) + csvreader.next() + for row in csvreader: + fw.write("%s\n" % row[0]) + fw.close() + + def generate_country_input(country_code, dst): """ Write to dst/citizenlab-urls-{country_code}.txt @@ -14,7 +25,6 @@ def generate_country_input(country_code, dst): country_code = country_code.lower() filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code) - fw = open(filename, "w+") input_list = os.path.join(config.resources_directory, "citizenlab-test-lists", @@ -24,30 +34,19 @@ def generate_country_input(country_code, dst): if not os.path.exists(input_list): raise Exception("Could not find list for country %s" % country_code) - with open(input_list) as f: - csvreader = csv.reader(f) - csvreader.next() - for row in csvreader: - fw.write("%s\n" % row[0]) + load_input(input_list, filename) - fw.close() return filename def generate_global_input(dst): - filename = os.path.join(dst, "citizenlab-urls-global.txt") - fw = open(filename, "w+") input_list = os.path.join(config.resources_directory, "citizenlab-test-lists", "test-lists-master", "csv", "global.csv") - with open(input_list) as f: - csvreader = csv.reader(f) - csvreader.next() - for row in csvreader: - fw.write("%s\n" % row[0]) - fw.close() + load_input(input_list, filename) + return filename diff --git a/ooni/deckgen/processors/namebench_dns_servers.py b/ooni/deckgen/processors/namebench_dns_servers.py index 1079335..e434f72 100644 --- a/ooni/deckgen/processors/namebench_dns_servers.py +++ b/ooni/deckgen/processors/namebench_dns_servers.py @@ -14,7 +14,6 @@ class GeoIPDB(object): self.__dict__ = self._borg if not self.country: try: - print config.advanced.geoip_data_dir country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') self.country = GeoIP.open(country_file, diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py index a140c8d..cab9700 100644 --- a/ooni/resources/cli.py +++ b/ooni/resources/cli.py @@ -4,7 +4,6 @@ from twisted.internet import defer from twisted.python import usage from ooni.utils import log -from ooni.settings import config from ooni.resources import __version__ from ooni.resources import update @@ -26,7 +25,6 @@ class Options(usage.Options): @defer.inlineCallbacks def run(): - config.read_config_file() options = Options() try: options.parseOptions() diff --git a/ooni/resources/update.py b/ooni/resources/update.py index a2919fc..6479aa9 100644 --- a/ooni/resources/update.py +++ b/ooni/resources/update.py @@ -13,6 +13,15 @@ def download_resource(resources): print "Downloading %s" % filename filename = os.path.join(config.resources_directory, filename) + if not os.path.exists(filename): + directory = os.path.dirname(filename) + if not os.path.isdir(directory): + os.makedirs(directory) + f = open(filename, 'w') + f.close() + elif not os.path.isfile(filename): + print "[!] %s must be a file." % filename + defer.returnValue(False) yield downloadPage(resource['url'], filename) if resource['action'] is not None: diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 30a93d5..180b5b3 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -135,6 +135,7 @@ def generate_filename(testDetails, prefix=None, extension=None, filename=None): return final_filename + def sanitize_options(options): """ Strips all possible user identifying information from the ooniprobe test @@ -147,8 +148,8 @@ def sanitize_options(options): sanitized_options.append(option) return sanitized_options -def unzip(filename, dst): +def unzip(filename, dst): assert filename.endswith('.zip') dst_path = os.path.join( dst, @@ -159,6 +160,7 @@ def unzip(filename, dst): zip_file.extractall(dst_path) return dst_path + def gunzip(filename, dst): assert filename.endswith(".gz") dst_path = os.path.join(