commit 6c3accabfe4c7f465afc02593d46bab4b29026a2 Author: Arturo Filastò <art@fuffa.org> Date: Fri Jan 3 16:05:19 2014 +0100 Include the GeoIP data files as part of the source distribution * Install the geoip data files to /usr/share/geoip --- MANIFEST.in | 7 ++-- data/ooniprobe.conf.sample | 4 +-- requirements.txt | 4 +-- setup.py | 77 ++++++++++++++++++++++++++++++++++++++------ 4 files changed, 76 insertions(+), 16 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index adcc182..92586f8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include README.md ChangeLog.md requirements.txt -recursive-include data/ui/app * recursive-include data/decks * -recursive-include data/inputs * -include data/Makefile +include data/inputs/ +include data/GeoIP.dat +include data/GeoIPASNum.dat +include data/GeoLiteCity.dat include data/ooniprobe.conf.sample diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample index 83c1ebc..8a4e0f7 100644 --- a/data/ooniprobe.conf.sample +++ b/data/ooniprobe.conf.sample @@ -21,7 +21,7 @@ reports: pcap: null collector: 'httpo://nkvphnp3p6agi5qq.onion' advanced: - geoip_data_dir: /usr/share/ooni/ + geoip_data_dir: /usr/share/geoip debug: false # enable if auto detection fails #tor_binary: /usr/sbin/tor @@ -46,7 +46,7 @@ advanced: # How many reports to perform concurrently reporting_concurrency: 15 # Specify here a custom data_dir path - data_dir: /usr/share/ooni/ + data_dir: /usr/share/ooni oonid_api_port: 8042 tor: #socks_port: 8801 diff --git a/requirements.txt b/requirements.txt index 5fb1ff9..638b33b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,8 @@ docutils>=0.9.1 ipaddr>=2.1.10 pyOpenSSL>=0.13 # XXX fix this once the following issue is resolved: https://github.com/appliedsec/pygeoip/issues/56 -https://github.com/appliedsec/pygeoip/archive/master.zip -#pygeoip>=0.3.1 +https://github.com/appliedsec/pygeoip/archive/master.zip#egg=pygeoip-0.3.1 +pygeoip>=0.3.1 txtorcon>=0.7 txsocksx>=0.0.2 Pyrex>=0.9.8.6 diff --git a/setup.py b/setup.py index 8eccd4e..f761f30 100644 --- a/setup.py +++ b/setup.py @@ -2,41 +2,101 @@ #-*- coding: utf-8 -*- from ooni import __version__ +import urllib2 import os +import gzip from os.path import join as pj import sys from setuptools import setup -dependency_links = [ - 'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe' -] +def download_geoip_files(): + urls = [ + 'http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz', + 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat....', + 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz' + ] + for url in urls: + target_gz_file = pj('data', os.path.basename(url)) + target_file = target_gz_file.replace('.gz', '') + + if os.path.isfile(target_file): + print "%s already exists. Skipping." % target_file + continue + + print "Downloading %s" % url + response = urllib2.urlopen(url) + + with open(target_gz_file, 'w+') as f: + f.write(response.read()) + + with open(target_file, 'w+') as f: + gf = gzip.open(target_gz_file, 'rb') + f.write(gf.read()) + gf.close() + + os.unlink(target_gz_file) + +download_geoip_files() usr_share_path = '/usr/share/ooni' # If this is true then it means we are in a virtualenv +# therefore we should not place our data files inside /usr/share/ooni, but +# place them inside the virtual env system prefix. if hasattr(sys, 'real_prefix'): - usr_share_path = pj(sys.prefix, 'share', 'ooni') + usr_share_path = os.path.abspath(pj(sys.prefix, 'share', 'ooni')) + if not os.path.isdir(usr_share_path): + os.makedirs(usr_share_path) with open(pj('data', 'ooniprobe.conf.sample.new'), 'w+') as w: with open(pj('data', 'ooniprobe.conf.sample')) as f: for line in f: if line.startswith(' data_dir: /usr/share/ooni'): w.write(' data_dir: %s\n' % usr_share_path) + elif line.startswith(' geoip_data_dir: /usr/share/'): + w.write(' geoip_data_dir: %s\n' % usr_share_path) else: w.write(line) os.rename(pj('data', 'ooniprobe.conf.sample.new'), pj('data', 'ooniprobe.conf.sample')) -data_files = [] + data_files = [( + usr_share_path + '/', + [ + 'data/GeoIP.dat', + 'data/GeoIPASNum.dat', + 'data/GeoLiteCity.dat' + ] + )] +else: + data_files = [( + '/usr/share/geoip/', + [ + 'data/GeoIP.dat', + 'data/GeoIPASNum.dat', + 'data/GeoLiteCity.dat' + ] + )] + for root, dirs, file_names in os.walk('data/'): files = [] for file_name in file_names: - if not file_name.endswith('.pyc'): - files.append(pj(root, file_name)) + if file_name.endswith('.pyc'): + continue + elif file_name.endswith('.dat') and \ + file_name.startswith('Geo'): + continue + files.append(pj(root, file_name)) data_files.append([pj(usr_share_path, root.replace('data/', '')), files]) install_requires = [] +dependency_links = [ + 'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe' +] with open('requirements.txt') as f: for line in f: - if line.startswith("#") or line.startswith('http'): + if line.startswith("#"): + continue + if line.startswith('https'): + dependency_links.append(line) continue install_requires.append(line) @@ -54,7 +114,6 @@ setup( 'ooni.nettests.blocking', 'ooni.nettests.third_party', 'ooni.templates', 'ooni.tests', 'ooni.utils'], - scripts=["bin/ooniprobe"], dependency_links=dependency_links, install_requires=install_requires