commit 2db26224cc0e79a8b144bd590287cd688c7891e0 Author: Arturo Filastò art@fuffa.org Date: Sat Jan 3 17:48:18 2015 +0100

Restructure directories where ooni software writes/reads from

This implements: https://trac.torproject.org/projects/tor/ticket/14086 --- data/ooniprobe.conf.sample | 2 - ooni/deckgen/cli.py | 4 +- ooni/deckgen/processors/citizenlab_test_lists.py | 16 ++-- ooni/deckgen/processors/namebench_dns_servers.py | 8 +- ooni/geoip.py | 6 +- ooni/resources/__init__.py | 24 ++++-- ooni/resources/update.py | 5 +- ooni/settings.py | 88 +++++++++++++++------- setup.py | 25 +++++- 9 files changed, 118 insertions(+), 60 deletions(-)

diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index 831525a..3c9cea9 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -45,8 +45,6 @@ advanced: reporting_retries: 3 # How many reports to perform concurrently reporting_concurrency: 15 - # Specify here a custom data_dir path - data_dir: /usr/share/ooni oonid_api_port: 8042 tor: #socks_port: 8801 diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index 216b9a7..3f4342a 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -118,6 +118,7 @@ def generate_deck(options):

@defer.inlineCallbacks def get_user_country_code(): + config.privacy.includecountry = True probe_ip = ProbeIP() yield probe_ip.lookup() defer.returnValue(probe_ip.geodata['countrycode']) @@ -155,9 +156,6 @@ def run(): "deck-%s" % options['country-code']) options['output'] = output_dir

- config.initialize_ooni_home() - config.read_config_file() - try: os.makedirs(options['output']) except OSError as exception: diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py index b36e726..b455008 100644 --- a/ooni/deckgen/processors/citizenlab_test_lists.py +++ b/ooni/deckgen/processors/citizenlab_test_lists.py @@ -26,10 +26,10 @@ def generate_country_input(country_code, dst): country_code = country_code.lower() filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code)

- input_list = os.path.join(config.resources_directory, - "citizenlab-test-lists", - "test-lists-master", - "csv", country_code + ".csv") + input_list = config.get_data_file_path("resources/" + "citizenlab-test-lists/" + "test-lists-master/csv/" + + country_code + ".csv")

if not os.path.exists(input_list): raise Exception("Could not find list for country %s" % country_code) @@ -42,10 +42,10 @@ def generate_country_input(country_code, dst): def generate_global_input(dst): filename = os.path.join(dst, "citizenlab-urls-global.txt")

- input_list = os.path.join(config.resources_directory, - "citizenlab-test-lists", - "test-lists-master", - "csv", "global.csv") + input_list = config.get_data_file_path("resources/" + "citizenlab-test-lists/" + "test-lists-master/csv/" + "global.csv")

load_input(input_list, filename)

diff --git a/ooni/deckgen/processors/namebench_dns_servers.py b/ooni/deckgen/processors/namebench_dns_servers.py index e434f72..f35ca93 100644 --- a/ooni/deckgen/processors/namebench_dns_servers.py +++ b/ooni/deckgen/processors/namebench_dns_servers.py @@ -14,8 +14,7 @@ class GeoIPDB(object): self.__dict__ = self._borg if not self.country: try: - country_file = os.path.join(config.advanced.geoip_data_dir, - 'GeoIP.dat') + country_file = config.get_data_file_path('GeoIP/GeoIP.dat') self.country = GeoIP.open(country_file, GeoIP.GEOIP_STANDARD) except: @@ -25,8 +24,9 @@ class GeoIPDB(object):

def generate_country_input(country_code, dst):

- csv_file = os.path.join(config.resources_directory, - "namebench-dns-servers.csv") + csv_file = config.get_data_file_path("resources/" + "namebench-dns-servers.csv") + filename = os.path.join(dst, "dns-server-%s.txt" % country_code) fw = open(filename, "w") geoip_db = GeoIPDB() diff --git a/ooni/geoip.py b/ooni/geoip.py index 48e1134..9c3ee31 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -30,9 +30,9 @@ class GeoIPDataFilesNotFound(Exception): def IPToLocation(ipaddr): from ooni.settings import config

- city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') - country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') - asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') + city_file = config.get_data_file_path('GeoIP/GeoLiteCity.dat') + country_file = config.get_data_file_path('GeoIP/GeoIP.dat') + asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')

location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}

diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py index 71e0a95..46fad05 100644 --- a/ooni/resources/__init__.py +++ b/ooni/resources/__init__.py @@ -1,14 +1,24 @@ +import os + from ooni.settings import config from ooni.utils import unzip, gunzip

from ooni.deckgen.processors import citizenlab_test_lists from ooni.deckgen.processors import namebench_dns_servers

-config.initialize_ooni_home() -config.read_config_file() - __version__ = "0.0.1"

+if os.access(config.var_lib_path, os.W_OK): + resources_directory = os.path.join(config.var_lib_path, + "resources") + geoip_directory = os.path.join(config.var_lib_path, + "GeoIP") +else: + resources_directory = os.path.join(config.ooni_home, + "resources") + geoip_directory = os.path.join(config.ooni_home, + "GeoIP") + inputs = { "namebench-dns-servers.csv": { "url": "https://namebench.googlecode.com/svn/trunk/config/servers.csv", @@ -19,7 +29,7 @@ inputs = { "citizenlab-test-lists.zip": { "url": "https://github.com/citizenlab/test-lists/archive/master.zip", "action": unzip, - "action_args": [config.resources_directory], + "action_args": [resources_directory], "processor": citizenlab_test_lists } } @@ -29,21 +39,21 @@ geoip = { "url": "https://geolite.maxmind.com/download/" "geoip/database/GeoLiteCity.dat.gz", "action": gunzip, - "action_args": [config.advanced.geoip_data_dir], + "action_args": [geoip_directory], "processor": None }, "GeoIPASNum.dat.gz": { "url": "https://geolite.maxmind.com/download/" "geoip/database/asnum/GeoIPASNum.dat.gz", "action": gunzip, - "action_args": [config.advanced.geoip_data_dir], + "action_args": [geoip_directory], "processor": None }, "GeoIP.dat.gz": { "url": "https://geolite.maxmind.com/" "download/geoip/database/GeoLiteCountry/GeoIP.dat.gz", "action": gunzip, - "action_args": [config.advanced.geoip_data_dir], + "action_args": [geoip_directory], "processor": None } } diff --git a/ooni/resources/update.py b/ooni/resources/update.py index 4017965..7a949c9 100644 --- a/ooni/resources/update.py +++ b/ooni/resources/update.py @@ -3,8 +3,7 @@ import os from twisted.internet import defer from twisted.web.client import downloadPage

-from ooni.settings import config -from ooni.resources import inputs, geoip +from ooni.resources import inputs, geoip, resources_directory from ooni.utils import unzip, gunzip

@@ -17,7 +16,7 @@ def download_resource(resources): dirname = resource["action_args"][0] filename = os.path.join(dirname, filename) else: - filename = os.path.join(config.resources_directory, filename) + filename = os.path.join(resources_directory, filename) if not os.path.exists(filename): directory = os.path.dirname(filename) if not os.path.isdir(directory): diff --git a/ooni/settings.py b/ooni/settings.py index 5ad622d..71b449e 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -32,21 +32,63 @@ class OConfig(object): self.privacy = Storage() self.set_paths()

- @property - def data_directory(self): + def embedded_settings(self, category, option): embedded_settings = os.path.join(get_ooni_root(), 'settings.ini') - if os.getenv("OONI_DATA_DIR"): - return os.getenv("OONI_DATA_DIR") - elif self.global_options.get('datadir'): - return abspath(expanduser(self.global_options['datadir'])) - elif self.advanced.get('data_dir'): - return self.advanced['data_dir'] - elif os.path.isfile(embedded_settings): + if os.path.isfile(embedded_settings): settings = SafeConfigParser() with open(embedded_settings) as fp: settings.readfp(fp) - return os.path.abspath(settings.get("directories", "data_dir")) - return abspath(os.path.join(get_ooni_root(), '..', 'data')) + return settings.get(category, option) + return None + + @property + def var_lib_path(self): + var_lib_path = self.embedded_settings("directories", "var_lib") + if var_lib_path: + return os.path.abspath(var_lib_path) + return "/var/lib/ooni" + + @property + def usr_share_path(self): + usr_share_path = self.embedded_settings("directories", "usr_share") + if usr_share_path: + return os.path.abspath(usr_share_path) + return "/usr/share/ooni" + + @property + def data_directory_candidates(self): + dirs = [ + os.path.join(expanduser('~'+self.current_user), '.ooni'), + self.var_lib_path, + self.usr_share_path, + os.path.join(get_ooni_root(), '..', 'data'), + '/usr/share/' + ] + if os.getenv("OONI_DATA_DIR"): + dirs.insert(0, os.getenv("OONI_DATA_DIR")) + if self.global_options.get('datadir'): + dirs.insert(0, abspath(expanduser(self.global_options['datadir']))) + return dirs + + @property + def data_directory(self): + for target_dir in self.data_directory_candidates: + if os.path.isdir(target_dir): + return target_dir + return self.var_lib_path + + @property + def ooni_home(self): + if self._custom_home: + return self._custom_home + return os.path.join(expanduser('~'+self.current_user), + '.ooni') + + def get_data_file_path(self, file_name): + for target_dir in self.data_directory_candidates: + file_path = os.path.join(target_dir, file_name) + if os.path.isfile(file_path): + return file_path

def set_paths(self, ooni_home=None): if ooni_home: @@ -54,15 +96,12 @@ class OConfig(object):

self.nettest_directory = os.path.join(get_ooni_root(), 'nettests')

- self.ooni_home = os.path.join(expanduser('~'+self.current_user), - '.ooni') - if self._custom_home: - self.ooni_home = self._custom_home self.inputs_directory = os.path.join(self.ooni_home, 'inputs') self.decks_directory = os.path.join(self.ooni_home, 'decks') self.reports_directory = os.path.join(self.ooni_home, 'reports') self.report_log_file = os.path.join(self.ooni_home, 'reporting.yml') - self.resources_directory = os.path.join(self.data_directory, "resources") + self.resources_directory = os.path.join(self.data_directory, + 'resources')

if self.global_options.get('configfile'): config_file = self.global_options['configfile'] @@ -71,12 +110,8 @@ class OConfig(object): self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf')

if 'logfile' in self.basic: - self.basic.logfile = expanduser(self.basic.logfile.replace('~','~'+self.current_user)) - - if not os.path.exists(self.data_directory): - log.err("Data directory %s does not exists" % self.data_directory) - log.err("Edit data_dir inside of %s" % self.config_file) - + self.basic.logfile = expanduser(self.basic.logfile.replace( + '~', '~'+self.current_user))

def initialize_ooni_home(self, ooni_home=None): if ooni_home: @@ -94,14 +129,15 @@ class OConfig(object): def _create_config_file(self): target_config_file = self.config_file print "Creating it for you in '%s'." % target_config_file - sample_config_file = os.path.join(self.data_directory, - 'ooniprobe.conf.sample') + sample_config_file = self.get_data_file_path('ooniprobe.conf.sample')

with open(sample_config_file) as f: with open(target_config_file, 'w+') as w: for line in f: if line.startswith(' logfile: '): - w.write(' logfile: %s\n' % os.path.join(self.ooni_home, 'ooniprobe.log')) + w.write(' logfile: %s\n' % ( + os.path.join(self.ooni_home, 'ooniprobe.log')) + ) else: w.write(line)

@@ -176,4 +212,4 @@ config = OConfig() if not os.path.isfile(config.config_file) \ and os.path.isfile('/etc/ooni/ooniprobe.conf'): config.global_options['configfile'] = '/etc/ooniprobe.conf' - config.set_paths(ooni_home=config.advanced.data_dir) + config.set_paths() diff --git a/setup.py b/setup.py index 9b66169..62335d9 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,13 @@ class install(_st_install): o.close() return config_file

- def set_data_files(self, share_path): + def set_data_files(self, prefix): + share_path = pj(prefix, 'share') + if prefix.startswith("/usr"): + var_path = "/var/lib/" + else: + var_path = pj(prefix, 'var', 'lib') + for root, dirs, file_names in os.walk('data/'): files = [] for file_name in file_names: @@ -45,14 +51,25 @@ class install(_st_install): ) settings = SafeConfigParser() settings.add_section("directories") - settings.set("directories", "data_dir", + settings.set("directories", "usr_share", os.path.join(share_path, "ooni")) + settings.set("directories", "var_lib", + os.path.join(var_path, "ooni")) with open("ooni/settings.ini", "w+") as fp: settings.write(fp)

+ try: + os.makedirs(pj(var_path, 'ooni')) + except OSError: + pass + try: + os.makedirs(pj(share_path, 'ooni')) + except OSError: + pass + def run(self): - share_path = os.path.abspath(pj(self.prefix, 'share')) - self.set_data_files(share_path) + prefix = os.path.abspath(self.prefix) + self.set_data_files(prefix) self.do_egg_install()