[tor-commits] [ooni-probe/master] Include the GeoIP data files as part of the source distribution
art at torproject.org
art at torproject.org
Sat Jan 4 16:56:27 UTC 2014
commit 6c3accabfe4c7f465afc02593d46bab4b29026a2
Author: Arturo Filastò <art at fuffa.org>
Date: Fri Jan 3 16:05:19 2014 +0100
Include the GeoIP data files as part of the source distribution
* Install the geoip data files to /usr/share/geoip
---
MANIFEST.in | 7 ++--
data/ooniprobe.conf.sample | 4 +--
requirements.txt | 4 +--
setup.py | 77 ++++++++++++++++++++++++++++++++++++++------
4 files changed, 76 insertions(+), 16 deletions(-)
diff --git a/MANIFEST.in b/MANIFEST.in
index adcc182..92586f8 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,6 +1,7 @@
include README.md ChangeLog.md requirements.txt
-recursive-include data/ui/app *
recursive-include data/decks *
-recursive-include data/inputs *
-include data/Makefile
+include data/inputs/
+include data/GeoIP.dat
+include data/GeoIPASNum.dat
+include data/GeoLiteCity.dat
include data/ooniprobe.conf.sample
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index 83c1ebc..8a4e0f7 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -21,7 +21,7 @@ reports:
pcap: null
collector: 'httpo://nkvphnp3p6agi5qq.onion'
advanced:
- geoip_data_dir: /usr/share/ooni/
+ geoip_data_dir: /usr/share/geoip
debug: false
# enable if auto detection fails
#tor_binary: /usr/sbin/tor
@@ -46,7 +46,7 @@ advanced:
# How many reports to perform concurrently
reporting_concurrency: 15
# Specify here a custom data_dir path
- data_dir: /usr/share/ooni/
+ data_dir: /usr/share/ooni
oonid_api_port: 8042
tor:
#socks_port: 8801
diff --git a/requirements.txt b/requirements.txt
index 5fb1ff9..638b33b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,8 +5,8 @@ docutils>=0.9.1
ipaddr>=2.1.10
pyOpenSSL>=0.13
# XXX fix this once the following issue is resolved: https://github.com/appliedsec/pygeoip/issues/56
-https://github.com/appliedsec/pygeoip/archive/master.zip
-#pygeoip>=0.3.1
+https://github.com/appliedsec/pygeoip/archive/master.zip#egg=pygeoip-0.3.1
+pygeoip>=0.3.1
txtorcon>=0.7
txsocksx>=0.0.2
Pyrex>=0.9.8.6
diff --git a/setup.py b/setup.py
index 8eccd4e..f761f30 100644
--- a/setup.py
+++ b/setup.py
@@ -2,41 +2,101 @@
#-*- coding: utf-8 -*-
from ooni import __version__
+import urllib2
import os
+import gzip
from os.path import join as pj
import sys
from setuptools import setup
-dependency_links = [
- 'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe'
-]
+def download_geoip_files():
+ urls = [
+ 'http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz',
+ 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz',
+ 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz'
+ ]
+ for url in urls:
+ target_gz_file = pj('data', os.path.basename(url))
+ target_file = target_gz_file.replace('.gz', '')
+
+ if os.path.isfile(target_file):
+ print "%s already exists. Skipping." % target_file
+ continue
+
+ print "Downloading %s" % url
+ response = urllib2.urlopen(url)
+
+ with open(target_gz_file, 'w+') as f:
+ f.write(response.read())
+
+ with open(target_file, 'w+') as f:
+ gf = gzip.open(target_gz_file, 'rb')
+ f.write(gf.read())
+ gf.close()
+
+ os.unlink(target_gz_file)
+
+download_geoip_files()
usr_share_path = '/usr/share/ooni'
# If this is true then it means we are in a virtualenv
+# therefore we should not place our data files inside /usr/share/ooni, but
+# place them inside the virtual env system prefix.
if hasattr(sys, 'real_prefix'):
- usr_share_path = pj(sys.prefix, 'share', 'ooni')
+ usr_share_path = os.path.abspath(pj(sys.prefix, 'share', 'ooni'))
+ if not os.path.isdir(usr_share_path):
+ os.makedirs(usr_share_path)
with open(pj('data', 'ooniprobe.conf.sample.new'), 'w+') as w:
with open(pj('data', 'ooniprobe.conf.sample')) as f:
for line in f:
if line.startswith(' data_dir: /usr/share/ooni'):
w.write(' data_dir: %s\n' % usr_share_path)
+ elif line.startswith(' geoip_data_dir: /usr/share/'):
+ w.write(' geoip_data_dir: %s\n' % usr_share_path)
else:
w.write(line)
os.rename(pj('data', 'ooniprobe.conf.sample.new'),
pj('data', 'ooniprobe.conf.sample'))
-data_files = []
+ data_files = [(
+ usr_share_path + '/',
+ [
+ 'data/GeoIP.dat',
+ 'data/GeoIPASNum.dat',
+ 'data/GeoLiteCity.dat'
+ ]
+ )]
+else:
+ data_files = [(
+ '/usr/share/geoip/',
+ [
+ 'data/GeoIP.dat',
+ 'data/GeoIPASNum.dat',
+ 'data/GeoLiteCity.dat'
+ ]
+ )]
+
for root, dirs, file_names in os.walk('data/'):
files = []
for file_name in file_names:
- if not file_name.endswith('.pyc'):
- files.append(pj(root, file_name))
+ if file_name.endswith('.pyc'):
+ continue
+ elif file_name.endswith('.dat') and \
+ file_name.startswith('Geo'):
+ continue
+ files.append(pj(root, file_name))
data_files.append([pj(usr_share_path, root.replace('data/', '')), files])
install_requires = []
+dependency_links = [
+ 'https://people.torproject.org/~ioerror/src/mirrors/ooniprobe'
+]
with open('requirements.txt') as f:
for line in f:
- if line.startswith("#") or line.startswith('http'):
+ if line.startswith("#"):
+ continue
+ if line.startswith('https'):
+ dependency_links.append(line)
continue
install_requires.append(line)
@@ -54,7 +114,6 @@ setup(
'ooni.nettests.blocking',
'ooni.nettests.third_party',
'ooni.templates', 'ooni.tests', 'ooni.utils'],
-
scripts=["bin/ooniprobe"],
dependency_links=dependency_links,
install_requires=install_requires
More information about the tor-commits
mailing list