commit 72c123125ac19d0a62f7eb5d3864efd01525d48a Author: Arturo Filastò arturo@filasto.net Date: Wed Jul 27 00:44:04 2016 +0200
Move gunzip and rename to utils --- ooni/agent/scheduler.py | 4 ++-- ooni/resources.py | 10 +--------- ooni/ui/cli.py | 1 - ooni/utils/__init__.py | 33 ++++++++++++++++++++------------- 4 files changed, 23 insertions(+), 25 deletions(-)
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index ace3bc5..3a9da20 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -137,11 +137,11 @@ class SchedulerService(service.MultiService): self._scheduled_tasks.append(task)
def _task_failed(self, failure, task): - log.msg("Failed to run {0}".format(task.identifier)) + log.debug("Failed to run {0}".format(task.identifier)) log.exception(failure)
def _task_success(self, result, task): - log.msg("Ran {0}".format(task.identifier)) + log.debug("Ran {0}".format(task.identifier))
def _should_run(self): """ diff --git a/ooni/resources.py b/ooni/resources.py index d67908c..8a0a57f 100644 --- a/ooni/resources.py +++ b/ooni/resources.py @@ -1,14 +1,12 @@ -import os import gzip import json import shutil
-from twisted.python.runtime import platform from twisted.python.filepath import FilePath from twisted.internet import defer from twisted.web.client import downloadPage, getPage
-from ooni.utils import log +from ooni.utils import log, gunzip, rename from ooni.settings import config
class UpdateFailure(Exception): @@ -85,12 +83,6 @@ def gunzip(file_path): in_file.close() rename(tmp_location.path, file_path)
-def rename(src, dst): - # Best effort atomic renaming - if platform.isWindows() and os.path.exists(dst): - os.unlink(dst) - os.rename(src, dst) - @defer.inlineCallbacks def check_for_update(country_code=None): """ diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index e8d747c..d8a4a8f 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -2,7 +2,6 @@ import sys
import os import json -import yaml import random import textwrap import urlparse diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index d202d97..f8ee953 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -1,12 +1,15 @@ import shutil import string import random +import gzip import os -from datetime import datetime
-import gzip +from datetime import datetime from zipfile import ZipFile
+from twisted.python.filepath import FilePath +from twisted.python.runtime import platform + from ooni import errors
@@ -128,6 +131,11 @@ def sanitize_options(options): sanitized_options.append(option) return sanitized_options
+def rename(src, dst): + # Best effort atomic renaming + if platform.isWindows() and os.path.exists(dst): + os.unlink(dst) + os.rename(src, dst)
def unzip(filename, dst): assert filename.endswith('.zip') @@ -141,17 +149,16 @@ def unzip(filename, dst): return dst_path
-def gunzip(filename, dst): - assert filename.endswith(".gz") - dst_path = os.path.join( - dst, - os.path.basename(filename).replace(".gz", "") - ) - with open(dst_path, "w+") as fw: - gzip_file = gzip.open(filename) - shutil.copyfileobj(gzip_file, fw) - gzip_file.close() - +def gunzip(file_path): + """ + gunzip a file in place. + """ + tmp_location = FilePath(file_path).temporarySibling() + in_file = gzip.open(file_path) + with tmp_location.open('w') as out_file: + shutil.copyfileobj(in_file, out_file) + in_file.close() + rename(tmp_location.path, file_path)
def get_ooni_root(): script = os.path.join(__file__, '..')