commit da919320ca780656e0369983d437d8e08470cf08 Author: Arturo Filastò <art@fuffa.org> Date: Thu Nov 13 11:42:06 2014 +0100 Remove geoip downloading from setup.py script --- requirements.txt | 1 + setup.py | 115 +++++++++++++++++++++++++----------------------------- 2 files changed, 54 insertions(+), 62 deletions(-) diff --git a/requirements.txt b/requirements.txt index 50259ff..e28a08d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ parsley>=1.1 scapy-real>=2.2.0-dev pypcap>=1.1 service-identity +pydumbnet diff --git a/setup.py b/setup.py index aa17f85..444bcd8 100644 --- a/setup.py +++ b/setup.py @@ -1,76 +1,58 @@ #!/usr/bin/env python -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- from ooni import __version__, __author__ -import urllib2 import os -import gzip -from os.path import join as pj import sys -from setuptools import setup - -def download_geoip_files(dst): - urls = [ - 'http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz', - 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat....', - 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz' - ] - if not os.path.exists(dst): - os.makedirs(dst) - for url in urls: - target_gz_file = pj(dst, os.path.basename(url)) - target_file = target_gz_file.replace('.gz', '') - - if os.path.isfile(target_file): - print "%s already exists. Skipping." % target_file - continue - - print "Downloading %s" % url - response = urllib2.urlopen(url) - - CHUNK = 4 * 1024 - with open(target_gz_file, 'w+') as f: - while True: - chunk = response.read(CHUNK) - if not chunk: break - f.write(chunk) - - with open(target_file, 'w+') as f: - gf = gzip.open(target_gz_file, 'rb') - while True: - chunk = gf.read(CHUNK) - if not chunk: break - f.write(chunk) - gf.close() - - os.unlink(target_gz_file) +import tempfile -usr_share_path = '/usr/share' -# If this is true then it means we are in a virtualenv -# therefore we should not place our data files inside /usr/share/ooni, but -# place them inside the virtual env system prefix. -if hasattr(sys, 'real_prefix'): - usr_share_path = os.path.abspath(pj(sys.prefix, 'share')) +from os.path import join as pj +from setuptools import setup +from setuptools.command.install import install as _st_install + + +class install(_st_install): + def gen_config(self, share_path): + config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample") + o = open(config_file, "w+") + with open("data/ooniprobe.conf.sample") as f: + for line in f: + if "/usr/share" in line: + line = line.replace("/usr/share", share_path) + o.write(line) + o.close() + return config_file + + def set_data_files(self, share_path): + for root, dirs, file_names in os.walk('data/'): + files = [] + for file_name in file_names: + if file_name.endswith('.pyc'): + continue + elif file_name.endswith('.dat') and \ + file_name.startswith('Geo'): + continue + elif file_name == "ooniprobe.conf.sample": + files.append(self.gen_config(share_path)) + continue + files.append(pj(root, file_name)) + self.distribution.data_files.append( + [ + pj(share_path, 'ooni', root.replace('data/', '')), + files + ] + ) + + def run(self): + share_path = os.path.abspath(pj(self.prefix, 'share')) + self.set_data_files(share_path) + self.do_egg_install() -download_geoip_files(pj(usr_share_path, 'GeoIP')) install_requires = [] dependency_links = [] data_files = [] - -for root, dirs, file_names in os.walk('data/'): - files = [] - for file_name in file_names: - if file_name.endswith('.pyc'): - continue - elif file_name.endswith('.dat') and \ - file_name.startswith('Geo'): - continue - files.append(pj(root, file_name)) - data_files.append([pj(usr_share_path, 'ooni', root.replace('data/', '')), files]) - - -packages=[ +packages = [ 'ooni', 'ooni.api', 'ooni.deckgen', @@ -120,6 +102,7 @@ setup( "bin/oonireport", "bin/ooniresources"], dependency_links=dependency_links, install_requires=install_requires, + cmdclass={"install": install}, classifiers=( "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -155,3 +138,11 @@ setup( "Topic :: System :: Networking :: Monitoring", ) ) + +from subprocess import Popen, PIPE +process = Popen(['ooniresources', '--update-inputs', '--update-geoip'], + stdout=PIPE, stderr=PIPE) +while process.poll() is None: + out = process.stdout.read() + sys.stdout.write(out) + sys.stdout.flush()