commit a61e3361a0bd3d931ab539301963940154b4277a Author: Arturo Filastò art@fuffa.org Date: Tue Aug 12 21:43:46 2014 +0200
Add support for downloading GeoIP files via the ooniresource tool.
* Add better exception handling to ooniresource. * Add gunzip to utils. --- ooni/resources/__init__.py | 21 ++++++++++++++++++++- ooni/resources/cli.py | 34 ++++++++++++++++++++++++++-------- ooni/resources/update.py | 15 +++++++++++---- ooni/utils/__init__.py | 13 +++++++++++++ 4 files changed, 70 insertions(+), 13 deletions(-)
diff --git a/ooni/resources/__init__.py b/ooni/resources/__init__.py index 0afc0af..e60784f 100644 --- a/ooni/resources/__init__.py +++ b/ooni/resources/__init__.py @@ -1,9 +1,11 @@ from ooni.settings import config -from ooni.utils import unzip +from ooni.utils import unzip, gunzip
from ooni.deckgen.processors import citizenlab_test_lists from ooni.deckgen.processors import namebench_dns_servers
+config.read_config_file() + __version__ = "0.0.1"
inputs = { @@ -20,3 +22,20 @@ inputs = { "processor": citizenlab_test_lists } } + +geoip = { + "GeoIPASNum.dat.gz": { + "url": "http://www.maxmind.com/download/" + "geoip/database/asnum/GeoIPASNum.dat.gz", + "action": gunzip, + "action_args": [config.advanced.geoip_data_dir], + "processor": None + }, + "GeoIP.dat.gz": { + "url": "http://geolite.maxmind.com/" + "download/geoip/database/GeoLiteCountry/GeoIP.dat.gz", + "action": gunzip, + "action_args": [config.advanced.geoip_data_dir], + "processor": None + } +} diff --git a/ooni/resources/cli.py b/ooni/resources/cli.py index 3ee7945..a140c8d 100644 --- a/ooni/resources/cli.py +++ b/ooni/resources/cli.py @@ -1,7 +1,9 @@ import sys
+from twisted.internet import defer from twisted.python import usage
+from ooni.utils import log from ooni.settings import config
from ooni.resources import __version__ @@ -9,10 +11,11 @@ from ooni.resources import update
class Options(usage.Options): - synopsis = """%s""" + synopsis = """%s""" % sys.argv[0]
optFlags = [ - ["update-inputs", None, "Update the resources needed for inputs"] + ["update-inputs", None, "Update the resources needed for inputs."], + ["update-geoip", None, "Update the geoip related resources."] ] optParameters = []
@@ -21,9 +24,10 @@ class Options(usage.Options): sys.exit(0)
+@defer.inlineCallbacks def run(): - options = Options() config.read_config_file() + options = Options() try: options.parseOptions() except usage.UsageError as error_message: @@ -31,9 +35,23 @@ def run(): print "%s: Try --help for usage details." % (sys.argv[0]) sys.exit(1)
- if options['update-inputs']: - return update.download_inputs() + if not any(options.values()): + print("%s: no command specified" % sys.argv[0]) + print options + sys.exit(1)
- print "%s: no command specified" % sys.argv[0] - print "%s: Try --help for usage details." % (sys.argv[0]) - sys.exit(1) + if options['update-inputs']: + print "Downloading inputs" + try: + yield update.download_inputs() + except Exception as exc: + log.err("failed to download geoip files") + log.exception(exc) + + if options['update-geoip']: + print "Downloading geoip files" + try: + yield update.download_geoip() + except Exception as exc: + log.err("failed to download geoip files") + log.exception(exc) diff --git a/ooni/resources/update.py b/ooni/resources/update.py index 28a6ec7..4cb0f49 100644 --- a/ooni/resources/update.py +++ b/ooni/resources/update.py @@ -4,7 +4,7 @@ from twisted.internet import reactor, defer, protocol from twisted.web.client import RedirectAgent, Agent
from ooni.settings import config -from ooni.resources import inputs +from ooni.resources import inputs, geoip
agent = RedirectAgent(Agent(reactor))
@@ -29,12 +29,11 @@ class SaveToFile(protocol.Protocol):
@defer.inlineCallbacks -def download_inputs(): - for filename, resource in inputs.items(): +def download_resource(resources): + for filename, resource in resources.items(): print "Downloading %s" % filename
filename = os.path.join(config.resources_directory, filename) - response = yield agent.request("GET", resource['url']) finished = defer.Deferred() response.deliverBody(SaveToFile(finished, response.length, filename)) @@ -45,3 +44,11 @@ def download_inputs(): filename, *resource['action_args']) print "%s written." % filename + + +def download_inputs(): + return download_resource(inputs) + + +def download_geoip(): + return download_resource(geoip) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 5050e14..30a93d5 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -1,8 +1,10 @@ +import shutil import string import random import glob import os
+import gzip from zipfile import ZipFile
from ooni import otime @@ -156,3 +158,14 @@ def unzip(filename, dst): zip_file = ZipFile(zfp) zip_file.extractall(dst_path) return dst_path + +def gunzip(filename, dst): + assert filename.endswith(".gz") + dst_path = os.path.join( + dst, + os.path.basename(filename).replace(".gz", "") + ) + with open(dst_path, "w+") as fw: + gzip_file = gzip.open(filename) + shutil.copyfileobj(gzip_file, fw) + gzip_file.close()