January 2015 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge branch 'fix/ooniresources' into feature/fix_setup.py
by art＠torproject.org 03 Jan '15

03 Jan '15

commit 6bc339ce8ef7b6f701a2ab6c560b3728a9a4bdde Merge: da91932 add9474 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 13 12:26:49 2014 +0100 Merge branch 'fix/ooniresources' into feature/fix_setup.py * fix/ooniresources: By default read the datadirectory from a ini file Set ooni systemwide config to /etc/ooni/ooniprobe.conf Leave GeoIP data in /usr/share/GeoIP Add GeoLiteCity dataset to ooniresources Remove comment Check for system-config globally, use /var/lib/ooni Display correct download path Update MaxMind URL Set user's logfile Set system-wide defaults for logfile, use /var/lib/ooni instead of /usr/share/ooni Check for system-wide configuration Use HTTPS URLs for MaxMind resources data/ooniprobe.conf.sample | 4 ++-- ooni/resources/__init__.py | 13 +++++++++++-- ooni/resources/update.py | 7 ++++++- ooni/settings.ini | 3 +++ ooni/settings.py | 37 +++++++++++++++++-------------------- ooni/utils/__init__.py | 7 ++++++- 6 files changed, 45 insertions(+), 26 deletions(-)

1 0

[ooni-probe/master] By default read the datadirectory from a ini file
by art＠torproject.org 03 Jan '15

03 Jan '15

commit add94741b2cd235de5379a695885ef7654acf8a0 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 13 12:26:24 2014 +0100 By default read the datadirectory from a ini file --- ooni/settings.ini | 3 +++ ooni/settings.py | 28 ++++++++++++---------------- ooni/utils/__init__.py | 7 ++++++- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/ooni/settings.ini b/ooni/settings.ini new file mode 100644 index 0000000..52fb901 --- /dev/null +++ b/ooni/settings.ini @@ -0,0 +1,3 @@ +[directories] +data_dir = /var/lib/ooni/ + diff --git a/ooni/settings.py b/ooni/settings.py index 51eef07..fbf85c2 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -1,7 +1,7 @@ import os -import sys import yaml import getpass +from ConfigParser import SafeConfigParser from twisted.internet import defer, reactor from twisted.internet.endpoints import TCP4ClientEndpoint @@ -10,7 +10,7 @@ from os.path import abspath, expanduser from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol from ooni import geoip -from ooni.utils import Storage, log +from ooni.utils import Storage, log, get_ooni_root from ooni import errors @@ -34,20 +34,18 @@ class OConfig(object): @property def data_directory(self): - data_directory = abspath(os.path.join(__file__, '..', '..', 'data')) - if os.getenv("OONI_DATA_DIR"): - data_directory = os.getenv("OONI_DATA_DIR") + return os.getenv("OONI_DATA_DIR") elif self.global_options.get('datadir'): - data_directory = abspath(expanduser(self.global_options['datadir'])) + return abspath(expanduser(self.global_options['datadir'])) elif self.advanced.get('data_dir'): - data_directory = self.advanced['data_dir'] - elif hasattr(sys, 'real_prefix'): - data_directory = os.path.abspath(os.path.join(sys.prefix, 'share', 'ooni')) - elif not os.path.exists(data_directory): - data_directory = '/var/lib/ooni/' - - return data_directory + return self.advanced['data_dir'] + else: + embedded_settings = os.path.join(get_ooni_root(), 'settings.ini') + settings = SafeConfigParser() + settings.readfp(open(embedded_settings)) + return os.path.abspath(settings.get("directories", "data_dir")) + #return abspath(os.path.join(__file__, '..', '..', 'data')) def set_paths(self, ooni_home=None): if ooni_home: @@ -100,9 +98,7 @@ class OConfig(object): with open(sample_config_file) as f: with open(target_config_file, 'w+') as w: for line in f: - if line.startswith(' data_dir: '): - w.write(' data_dir: %s\n' % self.data_directory) - elif line.startswith(' logfile: '): + if line.startswith(' logfile: '): w.write(' logfile: %s\n' % os.path.join(self.ooni_home, 'ooniprobe.log')) else: w.write(line) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 180b5b3..02904cc 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -42,7 +42,7 @@ class Storage(dict): try: del self[key] except KeyError, k: - raise AttributeError, k + raise AttributeError(k) def __repr__(self): return '<Storage ' + dict.__repr__(self) + '>' @@ -171,3 +171,8 @@ def gunzip(filename, dst): gzip_file = gzip.open(filename) shutil.copyfileobj(gzip_file, fw) gzip_file.close() + + +def get_ooni_root(): + script = os.path.join(__file__, '..') + return os.path.dirname(os.path.realpath(script))

1 0

[ooni-probe/master] Write the settings.ini file when installing ooniprobe.
by art＠torproject.org 03 Jan '15

03 Jan '15

commit 7e41f7e3b739b4777035a29bacc4c07279cc6e0c Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 13 12:34:50 2014 +0100 Write the settings.ini file when installing ooniprobe. --- MANIFEST.in | 1 + ooni/settings.py | 9 +++++---- setup.py | 9 +++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index da56999..7fd57f3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,3 +9,4 @@ include data/oonideckgen.1 include data/ooniprobe.1 include data/oonireport.1 include data/ooniresources.1 +include ooni/settings.ini diff --git a/ooni/settings.py b/ooni/settings.py index fbf85c2..8496c16 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -43,17 +43,18 @@ class OConfig(object): else: embedded_settings = os.path.join(get_ooni_root(), 'settings.ini') settings = SafeConfigParser() - settings.readfp(open(embedded_settings)) + with open(embedded_settings) as fp: + settings.readfp(fp) return os.path.abspath(settings.get("directories", "data_dir")) - #return abspath(os.path.join(__file__, '..', '..', 'data')) def set_paths(self, ooni_home=None): if ooni_home: self._custom_home = ooni_home - self.nettest_directory = abspath(os.path.join(__file__, '..', 'nettests')) + self.nettest_directory = os.path.join(get_ooni_root(), 'nettests') - self.ooni_home = os.path.join(expanduser('~'+self.current_user), '.ooni') + self.ooni_home = os.path.join(expanduser('~'+self.current_user), + '.ooni') if self._custom_home: self.ooni_home = self._custom_home self.inputs_directory = os.path.join(self.ooni_home, 'inputs') diff --git a/setup.py b/setup.py index 444bcd8..d06540b 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ from ooni import __version__, __author__ import os import sys import tempfile +from ConfigParser import SafeConfigParser from os.path import join as pj from setuptools import setup @@ -42,6 +43,13 @@ class install(_st_install): files ] ) + settings = SafeConfigParser() + with open("ooni/settings.ini") as fp: + settings.readfp(fp) + settings.set("directories", "data_dir", + os.path.join(share_path, "ooni")) + with open("ooni/settings.ini", "w+") as fp: + settings.write(fp) def run(self): share_path = os.path.abspath(pj(self.prefix, 'share')) @@ -98,6 +106,7 @@ setup( package_dir={'ooni': 'ooni'}, data_files=data_files, packages=packages, + include_package_data = True, scripts=["bin/oonideckgen", "bin/ooniprobe", "bin/oonireport", "bin/ooniresources"], dependency_links=dependency_links,

1 0

[ooni-probe/master] Fix path to ooni data dir
by art＠torproject.org 03 Jan '15

03 Jan '15

commit 35e034c07fd051946dd4a1675d0b01ad6161dc66 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 13 13:10:00 2014 +0100 Fix path to ooni data dir --- data/ooniprobe.conf.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index df92fbd..831525a 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -46,7 +46,7 @@ advanced: # How many reports to perform concurrently reporting_concurrency: 15 # Specify here a custom data_dir path - data_dir: /var/lib/ooni + data_dir: /usr/share/ooni oonid_api_port: 8042 tor: #socks_port: 8801

1 0

[ooni-probe/master] Initialise ooni_home before running deckgen and resources
by art＠torproject.org 03 Jan '15

03 Jan '15

commit 65e2bbf115c2588c23743d88cdbb139e9e5fbb94 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 13 13:21:10 2014 +0100 Initialise ooni_home before running deckgen and resources --- ooni/deckgen/cli.py | 1 + ooni/resources/cli.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index 77f5933..216b9a7 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -155,6 +155,7 @@ def run(): "deck-%s" % options['country-code']) options['output'] = output_dir + config.initialize_ooni_home() config.read_config_file() try: diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py index cab9700..b836b18 100644 --- a/ooni/resources/cli.py +++ b/ooni/resources/cli.py @@ -3,6 +3,7 @@ import sys from twisted.internet import defer from twisted.python import usage +from ooni.settings import config from ooni.utils import log from ooni.resources import __version__ @@ -38,6 +39,8 @@ def run(): print options sys.exit(1) + config.initialize_ooni_home() + config.read_config_file() if options['update-inputs']: print "Downloading inputs" try:

1 0

[ooni-probe/master] Merge pull request #364 from kudrom/fix/utils
by art＠torproject.org 03 Jan '15

03 Jan '15

commit 5c9bb4de0724d4ff2eaae6d8ad87f50a922097b4 Merge: 30fa387 fb4bfce Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Nov 6 17:49:51 2014 +0100 Merge pull request #364 from kudrom/fix/utils refactoring of net and txscapy utils ooni/errors.py | 16 +++++ ooni/geoip.py | 3 +- ooni/nettest.py | 2 +- .../experimental/bridge_reachability/echo.py | 62 ++++++++--------- ooni/oonicli.py | 2 +- ooni/templates/scapyt.py | 2 +- ooni/tests/test_oonicli.py | 2 +- ooni/tests/test_txscapy.py | 13 ---- ooni/tests/test_utils.py | 6 +- ooni/utils/net.py | 34 +++++++++ ooni/utils/txscapy.py | 73 ++------------------ 11 files changed, 96 insertions(+), 119 deletions(-)

1 0

[ooni-probe/master] Delete settings.ini and only create it when building
by art＠torproject.org 03 Jan '15

03 Jan '15

commit b66e5581519e4b231459cb2157fb73fd0340f7a5 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 13 12:46:46 2014 +0100 Delete settings.ini and only create it when building --- ooni/settings.ini | 3 --- ooni/settings.py | 5 +++-- setup.py | 4 +--- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ooni/settings.ini b/ooni/settings.ini deleted file mode 100644 index 52fb901..0000000 --- a/ooni/settings.ini +++ /dev/null @@ -1,3 +0,0 @@ -[directories] -data_dir = /var/lib/ooni/ - diff --git a/ooni/settings.py b/ooni/settings.py index 8496c16..5ad622d 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -34,18 +34,19 @@ class OConfig(object): @property def data_directory(self): + embedded_settings = os.path.join(get_ooni_root(), 'settings.ini') if os.getenv("OONI_DATA_DIR"): return os.getenv("OONI_DATA_DIR") elif self.global_options.get('datadir'): return abspath(expanduser(self.global_options['datadir'])) elif self.advanced.get('data_dir'): return self.advanced['data_dir'] - else: - embedded_settings = os.path.join(get_ooni_root(), 'settings.ini') + elif os.path.isfile(embedded_settings): settings = SafeConfigParser() with open(embedded_settings) as fp: settings.readfp(fp) return os.path.abspath(settings.get("directories", "data_dir")) + return abspath(os.path.join(get_ooni_root(), '..', 'data')) def set_paths(self, ooni_home=None): if ooni_home: diff --git a/setup.py b/setup.py index d06540b..1e67c1d 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,6 @@ class install(_st_install): ] ) settings = SafeConfigParser() - with open("ooni/settings.ini") as fp: - settings.readfp(fp) settings.set("directories", "data_dir", os.path.join(share_path, "ooni")) with open("ooni/settings.ini", "w+") as fp: @@ -106,7 +104,7 @@ setup( package_dir={'ooni': 'ooni'}, data_files=data_files, packages=packages, - include_package_data = True, + include_package_data=True, scripts=["bin/oonideckgen", "bin/ooniprobe", "bin/oonireport", "bin/ooniresources"], dependency_links=dependency_links,

1 0

[ooni-probe/master] Directly map process stdout and stderr to stdout and sterr fds
by art＠torproject.org 03 Jan '15

03 Jan '15

commit ab46fd21aa14be2904ebb37162edc3bcfc8eb2d4 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 13 13:34:52 2014 +0100 Directly map process stdout and stderr to stdout and sterr fds --- ooni/resources/__init__.py | 3 +-- ooni/resources/cli.py | 3 --- setup.py | 7 ++----- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py index d0f0cb9..71e0a95 100644 --- a/ooni/resources/__init__.py +++ b/ooni/resources/__init__.py @@ -1,11 +1,10 @@ -import os - from ooni.settings import config from ooni.utils import unzip, gunzip from ooni.deckgen.processors import citizenlab_test_lists from ooni.deckgen.processors import namebench_dns_servers +config.initialize_ooni_home() config.read_config_file() __version__ = "0.0.1" diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py index b836b18..cab9700 100644 --- a/ooni/resources/cli.py +++ b/ooni/resources/cli.py @@ -3,7 +3,6 @@ import sys from twisted.internet import defer from twisted.python import usage -from ooni.settings import config from ooni.utils import log from ooni.resources import __version__ @@ -39,8 +38,6 @@ def run(): print options sys.exit(1) - config.initialize_ooni_home() - config.read_config_file() if options['update-inputs']: print "Downloading inputs" try: diff --git a/setup.py b/setup.py index 7b46764..bd9255e 100644 --- a/setup.py +++ b/setup.py @@ -149,8 +149,5 @@ setup( from subprocess import Popen, PIPE process = Popen(['ooniresources', '--update-inputs', '--update-geoip'], - stdout=PIPE, stderr=PIPE) -while process.poll() is None: - out = process.stdout.read() - sys.stdout.write(out) - sys.stdout.flush() + stdout=sys.stdout.fileno(), stderr=sys.stderr.fileno()) +process.wait()

1 0

[ooni-probe/master] Fix email address of ooni-probe author
by art＠torproject.org 03 Jan '15

03 Jan '15

commit bdb7896b40f0e9ae45e2c092f27c1c45acf7fa7d Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Nov 18 12:12:05 2014 +0100 Fix email address of ooni-probe author --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bd9255e..9b66169 100644 --- a/setup.py +++ b/setup.py @@ -97,7 +97,7 @@ setup( name="ooniprobe", version=__version__, author=__author__, - author_email="ooni-dev(a)torproject.org", + author_email="ooni-dev(a)lists.torproject.org", description="Network Interference detection tool.", long_description=readme + '\n\n' + changelog, license='BSD 2 clause',

1 0

[ooni-probe/master] Remove unneeded files from MANIFEST
by art＠torproject.org 03 Jan '15

03 Jan '15

commit f0a489e02eb4a5779a53f41879ee77bab6ed4fa0 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 23 12:19:19 2014 +0100 Remove unneeded files from MANIFEST --- MANIFEST.in | 7 ------- 1 file changed, 7 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7fd57f3..9b70d09 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,12 +1,5 @@ include README.rst ChangeLog.rst requirements.txt LICENSE recursive-include data/decks * include data/inputs/ -include data/GeoIP.dat -include data/GeoIPASNum.dat -include data/GeoLiteCity.dat include data/ooniprobe.conf.sample -include data/oonideckgen.1 -include data/ooniprobe.1 -include data/oonireport.1 -include data/ooniresources.1 include ooni/settings.ini

1 0