[tor-commits] [ooni-probe/master] Restructure directories where ooni software writes/reads from

art at torproject.org art at torproject.org
Wed Feb 4 13:27:15 UTC 2015


commit 2db26224cc0e79a8b144bd590287cd688c7891e0
Author: Arturo Filastò <art at fuffa.org>
Date:   Sat Jan 3 17:48:18 2015 +0100

    Restructure directories where ooni software writes/reads from
    
    This implements: https://trac.torproject.org/projects/tor/ticket/14086
---
 data/ooniprobe.conf.sample                       |    2 -
 ooni/deckgen/cli.py                              |    4 +-
 ooni/deckgen/processors/citizenlab_test_lists.py |   16 ++--
 ooni/deckgen/processors/namebench_dns_servers.py |    8 +-
 ooni/geoip.py                                    |    6 +-
 ooni/resources/__init__.py                       |   24 ++++--
 ooni/resources/update.py                         |    5 +-
 ooni/settings.py                                 |   88 +++++++++++++++-------
 setup.py                                         |   25 +++++-
 9 files changed, 118 insertions(+), 60 deletions(-)

diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index 831525a..3c9cea9 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -45,8 +45,6 @@ advanced:
     reporting_retries: 3
     # How many reports to perform concurrently
     reporting_concurrency: 15
-    # Specify here a custom data_dir path
-    data_dir: /usr/share/ooni
     oonid_api_port: 8042
 tor:
     #socks_port: 8801
diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py
index 216b9a7..3f4342a 100644
--- a/ooni/deckgen/cli.py
+++ b/ooni/deckgen/cli.py
@@ -118,6 +118,7 @@ def generate_deck(options):
 
 @defer.inlineCallbacks
 def get_user_country_code():
+    config.privacy.includecountry = True
     probe_ip = ProbeIP()
     yield probe_ip.lookup()
     defer.returnValue(probe_ip.geodata['countrycode'])
@@ -155,9 +156,6 @@ def run():
                               "deck-%s" % options['country-code'])
     options['output'] = output_dir
 
-    config.initialize_ooni_home()
-    config.read_config_file()
-
     try:
         os.makedirs(options['output'])
     except OSError as exception:
diff --git a/ooni/deckgen/processors/citizenlab_test_lists.py b/ooni/deckgen/processors/citizenlab_test_lists.py
index b36e726..b455008 100644
--- a/ooni/deckgen/processors/citizenlab_test_lists.py
+++ b/ooni/deckgen/processors/citizenlab_test_lists.py
@@ -26,10 +26,10 @@ def generate_country_input(country_code, dst):
     country_code = country_code.lower()
     filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code)
 
-    input_list = os.path.join(config.resources_directory,
-                              "citizenlab-test-lists",
-                              "test-lists-master",
-                              "csv", country_code + ".csv")
+    input_list = config.get_data_file_path("resources/"
+                                           "citizenlab-test-lists/"
+                                           "test-lists-master/csv/"
+                                           + country_code + ".csv")
 
     if not os.path.exists(input_list):
         raise Exception("Could not find list for country %s" % country_code)
@@ -42,10 +42,10 @@ def generate_country_input(country_code, dst):
 def generate_global_input(dst):
     filename = os.path.join(dst, "citizenlab-urls-global.txt")
 
-    input_list = os.path.join(config.resources_directory,
-                              "citizenlab-test-lists",
-                              "test-lists-master",
-                              "csv", "global.csv")
+    input_list = config.get_data_file_path("resources/"
+                                           "citizenlab-test-lists/"
+                                           "test-lists-master/csv/"
+                                           "global.csv")
 
     load_input(input_list, filename)
 
diff --git a/ooni/deckgen/processors/namebench_dns_servers.py b/ooni/deckgen/processors/namebench_dns_servers.py
index e434f72..f35ca93 100644
--- a/ooni/deckgen/processors/namebench_dns_servers.py
+++ b/ooni/deckgen/processors/namebench_dns_servers.py
@@ -14,8 +14,7 @@ class GeoIPDB(object):
         self.__dict__ = self._borg
         if not self.country:
             try:
-                country_file = os.path.join(config.advanced.geoip_data_dir,
-                                            'GeoIP.dat')
+                country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
                 self.country = GeoIP.open(country_file,
                                           GeoIP.GEOIP_STANDARD)
             except:
@@ -25,8 +24,9 @@ class GeoIPDB(object):
 
 def generate_country_input(country_code, dst):
 
-    csv_file = os.path.join(config.resources_directory,
-                            "namebench-dns-servers.csv")
+    csv_file = config.get_data_file_path("resources/"
+                                           "namebench-dns-servers.csv")
+
     filename = os.path.join(dst, "dns-server-%s.txt" % country_code)
     fw = open(filename, "w")
     geoip_db = GeoIPDB()
diff --git a/ooni/geoip.py b/ooni/geoip.py
index 48e1134..9c3ee31 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -30,9 +30,9 @@ class GeoIPDataFilesNotFound(Exception):
 def IPToLocation(ipaddr):
     from ooni.settings import config
 
-    city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
-    country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
-    asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
+    city_file = config.get_data_file_path('GeoIP/GeoLiteCity.dat')
+    country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
+    asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')
 
     location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
 
diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py
index 71e0a95..46fad05 100644
--- a/ooni/resources/__init__.py
+++ b/ooni/resources/__init__.py
@@ -1,14 +1,24 @@
+import os
+
 from ooni.settings import config
 from ooni.utils import unzip, gunzip
 
 from ooni.deckgen.processors import citizenlab_test_lists
 from ooni.deckgen.processors import namebench_dns_servers
 
-config.initialize_ooni_home()
-config.read_config_file()
-
 __version__ = "0.0.1"
 
+if os.access(config.var_lib_path, os.W_OK):
+    resources_directory = os.path.join(config.var_lib_path,
+                                       "resources")
+    geoip_directory = os.path.join(config.var_lib_path,
+                                   "GeoIP")
+else:
+    resources_directory = os.path.join(config.ooni_home,
+                                       "resources")
+    geoip_directory = os.path.join(config.ooni_home,
+                                   "GeoIP")
+
 inputs = {
     "namebench-dns-servers.csv": {
         "url": "https://namebench.googlecode.com/svn/trunk/config/servers.csv",
@@ -19,7 +29,7 @@ inputs = {
     "citizenlab-test-lists.zip": {
         "url": "https://github.com/citizenlab/test-lists/archive/master.zip",
         "action": unzip,
-        "action_args": [config.resources_directory],
+        "action_args": [resources_directory],
         "processor": citizenlab_test_lists
     }
 }
@@ -29,21 +39,21 @@ geoip = {
         "url": "https://geolite.maxmind.com/download/"
                "geoip/database/GeoLiteCity.dat.gz",
         "action": gunzip,
-        "action_args": [config.advanced.geoip_data_dir],
+        "action_args": [geoip_directory],
         "processor": None
     },
     "GeoIPASNum.dat.gz": {
         "url": "https://geolite.maxmind.com/download/"
                "geoip/database/asnum/GeoIPASNum.dat.gz",
         "action": gunzip,
-        "action_args": [config.advanced.geoip_data_dir],
+        "action_args": [geoip_directory],
         "processor": None
     },
     "GeoIP.dat.gz": {
         "url": "https://geolite.maxmind.com/"
                "download/geoip/database/GeoLiteCountry/GeoIP.dat.gz",
         "action": gunzip,
-        "action_args": [config.advanced.geoip_data_dir],
+        "action_args": [geoip_directory],
         "processor": None
     }
 }
diff --git a/ooni/resources/update.py b/ooni/resources/update.py
index 4017965..7a949c9 100644
--- a/ooni/resources/update.py
+++ b/ooni/resources/update.py
@@ -3,8 +3,7 @@ import os
 from twisted.internet import defer
 from twisted.web.client import downloadPage
 
-from ooni.settings import config
-from ooni.resources import inputs, geoip
+from ooni.resources import inputs, geoip, resources_directory
 from ooni.utils import unzip, gunzip
 
 
@@ -17,7 +16,7 @@ def download_resource(resources):
                     dirname = resource["action_args"][0]
                     filename = os.path.join(dirname, filename)
         else:
-            filename = os.path.join(config.resources_directory, filename)
+            filename = os.path.join(resources_directory, filename)
         if not os.path.exists(filename):
             directory = os.path.dirname(filename)
             if not os.path.isdir(directory):
diff --git a/ooni/settings.py b/ooni/settings.py
index 5ad622d..71b449e 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -32,21 +32,63 @@ class OConfig(object):
         self.privacy = Storage()
         self.set_paths()
 
-    @property
-    def data_directory(self):
+    def embedded_settings(self, category, option):
         embedded_settings = os.path.join(get_ooni_root(), 'settings.ini')
-        if os.getenv("OONI_DATA_DIR"):
-            return os.getenv("OONI_DATA_DIR")
-        elif self.global_options.get('datadir'):
-            return abspath(expanduser(self.global_options['datadir']))
-        elif self.advanced.get('data_dir'):
-            return self.advanced['data_dir']
-        elif os.path.isfile(embedded_settings):
+        if os.path.isfile(embedded_settings):
             settings = SafeConfigParser()
             with open(embedded_settings) as fp:
                 settings.readfp(fp)
-            return os.path.abspath(settings.get("directories", "data_dir"))
-        return abspath(os.path.join(get_ooni_root(), '..', 'data'))
+            return settings.get(category, option)
+        return None
+
+    @property
+    def var_lib_path(self):
+        var_lib_path = self.embedded_settings("directories", "var_lib")
+        if var_lib_path:
+            return os.path.abspath(var_lib_path)
+        return "/var/lib/ooni"
+
+    @property
+    def usr_share_path(self):
+        usr_share_path = self.embedded_settings("directories", "usr_share")
+        if usr_share_path:
+            return os.path.abspath(usr_share_path)
+        return "/usr/share/ooni"
+
+    @property
+    def data_directory_candidates(self):
+        dirs = [
+            os.path.join(expanduser('~'+self.current_user), '.ooni'),
+            self.var_lib_path,
+            self.usr_share_path,
+            os.path.join(get_ooni_root(), '..', 'data'),
+            '/usr/share/'
+        ]
+        if os.getenv("OONI_DATA_DIR"):
+            dirs.insert(0, os.getenv("OONI_DATA_DIR"))
+        if self.global_options.get('datadir'):
+            dirs.insert(0, abspath(expanduser(self.global_options['datadir'])))
+        return dirs
+
+    @property
+    def data_directory(self):
+        for target_dir in self.data_directory_candidates:
+            if os.path.isdir(target_dir):
+                return target_dir
+        return self.var_lib_path
+
+    @property
+    def ooni_home(self):
+        if self._custom_home:
+            return self._custom_home
+        return os.path.join(expanduser('~'+self.current_user),
+                                       '.ooni')
+
+    def get_data_file_path(self, file_name):
+        for target_dir in self.data_directory_candidates:
+            file_path = os.path.join(target_dir, file_name)
+            if os.path.isfile(file_path):
+                return file_path
 
     def set_paths(self, ooni_home=None):
         if ooni_home:
@@ -54,15 +96,12 @@ class OConfig(object):
 
         self.nettest_directory = os.path.join(get_ooni_root(), 'nettests')
 
-        self.ooni_home = os.path.join(expanduser('~'+self.current_user),
-                                      '.ooni')
-        if self._custom_home:
-            self.ooni_home = self._custom_home
         self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
         self.decks_directory = os.path.join(self.ooni_home, 'decks')
         self.reports_directory = os.path.join(self.ooni_home, 'reports')
         self.report_log_file = os.path.join(self.ooni_home, 'reporting.yml')
-        self.resources_directory = os.path.join(self.data_directory, "resources")
+        self.resources_directory = os.path.join(self.data_directory,
+                                                'resources')
 
         if self.global_options.get('configfile'):
             config_file = self.global_options['configfile']
@@ -71,12 +110,8 @@ class OConfig(object):
             self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf')
 
         if 'logfile' in self.basic:
-            self.basic.logfile = expanduser(self.basic.logfile.replace('~','~'+self.current_user))
-
-        if not os.path.exists(self.data_directory):
-            log.err("Data directory %s does not exists" % self.data_directory)
-            log.err("Edit data_dir inside of %s" % self.config_file)
-
+            self.basic.logfile = expanduser(self.basic.logfile.replace(
+                '~', '~'+self.current_user))
 
     def initialize_ooni_home(self, ooni_home=None):
         if ooni_home:
@@ -94,14 +129,15 @@ class OConfig(object):
     def _create_config_file(self):
         target_config_file = self.config_file
         print "Creating it for you in '%s'." % target_config_file
-        sample_config_file = os.path.join(self.data_directory,
-                                          'ooniprobe.conf.sample')
+        sample_config_file = self.get_data_file_path('ooniprobe.conf.sample')
 
         with open(sample_config_file) as f:
             with open(target_config_file, 'w+') as w:
                 for line in f:
                     if line.startswith('    logfile: '):
-                        w.write('    logfile: %s\n' % os.path.join(self.ooni_home, 'ooniprobe.log'))
+                        w.write('    logfile: %s\n' % (
+                            os.path.join(self.ooni_home, 'ooniprobe.log'))
+                        )
                     else:
                         w.write(line)
 
@@ -176,4 +212,4 @@ config = OConfig()
 if not os.path.isfile(config.config_file) \
        and os.path.isfile('/etc/ooni/ooniprobe.conf'):
     config.global_options['configfile'] = '/etc/ooniprobe.conf'
-    config.set_paths(ooni_home=config.advanced.data_dir)
+    config.set_paths()
diff --git a/setup.py b/setup.py
index 9b66169..62335d9 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,13 @@ class install(_st_install):
         o.close()
         return config_file
 
-    def set_data_files(self, share_path):
+    def set_data_files(self, prefix):
+        share_path = pj(prefix, 'share')
+        if prefix.startswith("/usr"):
+            var_path = "/var/lib/"
+        else:
+            var_path = pj(prefix, 'var', 'lib')
+
         for root, dirs, file_names in os.walk('data/'):
             files = []
             for file_name in file_names:
@@ -45,14 +51,25 @@ class install(_st_install):
             )
         settings = SafeConfigParser()
         settings.add_section("directories")
-        settings.set("directories", "data_dir",
+        settings.set("directories", "usr_share",
                      os.path.join(share_path, "ooni"))
+        settings.set("directories", "var_lib",
+                     os.path.join(var_path, "ooni"))
         with open("ooni/settings.ini", "w+") as fp:
             settings.write(fp)
 
+        try:
+            os.makedirs(pj(var_path, 'ooni'))
+        except OSError:
+            pass
+        try:
+            os.makedirs(pj(share_path, 'ooni'))
+        except OSError:
+            pass
+
     def run(self):
-        share_path = os.path.abspath(pj(self.prefix, 'share'))
-        self.set_data_files(share_path)
+        prefix = os.path.abspath(self.prefix)
+        self.set_data_files(prefix)
         self.do_egg_install()
 
 





More information about the tor-commits mailing list