[tor-commits] [ooni-probe/master] Cleanup some of the log outputs.
art at torproject.org
art at torproject.org
Mon Sep 19 12:14:24 UTC 2016
commit cf333afdf5dc08e17d02516a0af79552aa6f0275
Author: Arturo Filastò <arturo at filasto.net>
Date: Thu Jul 28 01:38:41 2016 +0200
Cleanup some of the log outputs.
* Fix more unittests.
Will you be green green green?
---
ooni/agent/scheduler.py | 2 +-
ooni/director.py | 6 +++---
ooni/reporter.py | 7 +++++--
ooni/resources.py | 30 ++++++++++++++----------------
ooni/tests/test_geoip.py | 16 +++++++++++++++-
ooni/tests/test_oonicli.py | 5 ++---
ooni/tests/test_resources.py | 16 +++++++++++++++-
ooni/tests/test_utils.py | 8 ++++----
ooni/utils/onion.py | 1 -
9 files changed, 59 insertions(+), 32 deletions(-)
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 9784f8a..71a7edb 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -97,7 +97,7 @@ def run_system_tasks(no_input_store=False):
if no_input_store:
log.debug("Not updating the inputs")
- task_classes.pop(UpdateInputsAndResources)
+ task_classes.remove(UpdateInputsAndResources)
for task_class in task_classes:
task = task_class()
diff --git a/ooni/director.py b/ooni/director.py
index ad6d1e2..f6311ac 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -141,7 +141,7 @@ class Director(object):
self._tor_starting.addCallback(self._tor_startup_success)
def _tor_startup_failure(self, failure):
- log.msg("Failed to start tor")
+ log.err("Failed to start tor")
log.exception(failure)
self._reset_tor_state()
self.notify(DirectorEvent("error",
@@ -382,7 +382,7 @@ class Director(object):
yield self._tor_starting
defer.returnValue(self._tor_state)
- log.msg("Starting Tor...")
+ log.msg("Starting Tor")
self._tor_state = 'starting'
if check_incoherences:
try:
@@ -405,7 +405,7 @@ class Director(object):
data_dir = os.path.expanduser(config.tor.data_dir)
if not os.path.exists(data_dir):
- log.msg("%s does not exist. Creating it." % data_dir)
+ log.debug("%s does not exist. Creating it." % data_dir)
os.makedirs(data_dir)
tor_config.DataDirectory = data_dir
diff --git a/ooni/reporter.py b/ooni/reporter.py
index e2f155f..20a13f5 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -342,9 +342,12 @@ class OONIBReporter(OReporter):
log.msg("Try running a different test or try reporting to a "
"different collector.")
raise errors.OONIBReportCreationError
- except Exception, e:
+ except errors.OONIBError:
log.err("Failed to connect to reporter backend")
- log.exception(e)
+ raise errors.OONIBReportCreationError
+ except Exception as exc:
+ log.err("Failed to connect to reporter backend")
+ log.exception(exc)
raise errors.OONIBReportCreationError
self.reportId = response['report_id'].encode('ascii')
diff --git a/ooni/resources.py b/ooni/resources.py
index 47ebf86..aef0f13 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -1,24 +1,25 @@
-import gzip
import json
-import shutil
from twisted.python.filepath import FilePath
from twisted.internet import defer
from twisted.web.client import downloadPage, getPage, HTTPClientFactory
+from ooni.utils import log, gunzip, rename
+from ooni.settings import config
+
# Disable logs of HTTPClientFactory
HTTPClientFactory.noisy = False
-from ooni.utils import log, gunzip, rename
-from ooni.settings import config
class UpdateFailure(Exception):
pass
+
def get_download_url(tag_name, filename):
return ("https://github.com/OpenObservatory/ooni-resources/releases"
"/download/{0}/{1}".format(tag_name, filename))
+
def get_current_version():
manifest = FilePath(config.resources_directory).child("manifest.json")
if not manifest.exists():
@@ -27,6 +28,7 @@ def get_current_version():
manifest = json.load(f)
return int(manifest["version"])
+
@defer.inlineCallbacks
def get_latest_version():
"""
@@ -41,7 +43,8 @@ def get_latest_version():
def get_out_of_date_resources(current_manifest, new_manifest,
- country_code=None):
+ country_code=None,
+ resources_directory=config.resources_directory):
current_res = {}
new_res = {}
for r in current_manifest["resources"]:
@@ -51,11 +54,13 @@ def get_out_of_date_resources(current_manifest, new_manifest,
new_res[r["path"]] = r
paths_to_delete = [
- current_res[path] for path in list(set(current_res.keys()) -
- set(new_res.keys()))
- ]
+ current_res[path] for path in list(
+ set(current_res.keys()) -
+ set(new_res.keys())
+ )
+ ]
paths_to_update = []
- _resources = FilePath(config.resources_directory)
+ _resources = FilePath(resources_directory)
for path, info in new_res.items():
if (country_code is not None and
info["country_code"] != "ALL" and
@@ -78,13 +83,6 @@ def get_out_of_date_resources(current_manifest, new_manifest,
return paths_to_update, paths_to_delete
-def gunzip(file_path):
- tmp_location = FilePath(file_path).temporarySibling()
- in_file = gzip.open(file_path)
- with tmp_location.open('w') as out_file:
- shutil.copyfileobj(in_file, out_file)
- in_file.close()
- rename(tmp_location.path, file_path)
@defer.inlineCallbacks
def check_for_update(country_code=None):
diff --git a/ooni/tests/test_geoip.py b/ooni/tests/test_geoip.py
index 8eb964d..b5e76f3 100644
--- a/ooni/tests/test_geoip.py
+++ b/ooni/tests/test_geoip.py
@@ -1,4 +1,5 @@
-
+import os
+import shutil
from twisted.internet import defer
from ooni.tests import is_internet_connected, bases
@@ -23,6 +24,17 @@ class TestGeoIP(bases.ConfigTestCase):
assert len(res.split('.')) == 4
def test_geoip_database_version(self):
+ maxmind_dir = os.path.join(self.config.resources_directory,
+ 'maxmind-geoip')
+ try:
+ os.mkdir(maxmind_dir)
+ except OSError:
+ pass
+ with open(os.path.join(maxmind_dir, 'GeoIP.dat'), 'w+') as f:
+ f.write("XXX")
+ with open(os.path.join(maxmind_dir, 'GeoIPASNum.dat'), 'w+') as f:
+ f.write("XXX")
+
version = geoip.database_version()
assert 'GeoIP' in version.keys()
assert 'GeoIPASNum' in version.keys()
@@ -31,3 +43,5 @@ class TestGeoIP(bases.ConfigTestCase):
assert isinstance(version['GeoIP']['timestamp'], float)
assert len(version['GeoIPASNum']['sha256']) == 64
assert isinstance(version['GeoIPASNum']['timestamp'], float)
+
+ shutil.rmtree(maxmind_dir)
diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index c3facee..06a21ff 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -55,7 +55,6 @@ advanced:
oonid_api_port: 8042
tor:
socks_port: 9050
-
""" % config.data_directory
@@ -149,8 +148,8 @@ class TestRunDirector(ConfigTestCase):
yield self.run_helper('blocking/dns_consistency',
['-b', '8.8.8.8:53',
- '-t', '8.8.8.8',
- '-f', 'example-input.txt'],
+ '-t', '8.8.8.8',
+ '-f', 'example-input.txt'],
verify_function)
@defer.inlineCallbacks
diff --git a/ooni/tests/test_resources.py b/ooni/tests/test_resources.py
index 45473e9..6d1bb3c 100644
--- a/ooni/tests/test_resources.py
+++ b/ooni/tests/test_resources.py
@@ -1,3 +1,7 @@
+import os
+import shutil
+import tempfile
+
from ooni.resources import get_out_of_date_resources, check_for_update
from ooni.tests.bases import ConfigTestCase
@@ -36,7 +40,17 @@ class TestResourceUpdate(ConfigTestCase):
return check_for_update()
def test_resources_out_of_date(self):
+ tmp_dir = tempfile.mkdtemp()
+ os.mkdir(os.path.join(tmp_dir, 'some'))
+ original_paths = map(lambda r: r['path'],
+ SAMPLE_CURRENT_MANIFEST['resources'])
+ for path in original_paths:
+ with open(os.path.join(tmp_dir, path), 'w+'):
+ pass
paths_to_update, paths_to_delete = get_out_of_date_resources(
- SAMPLE_CURRENT_MANIFEST, SAMPLE_NEW_MANIFEST)
+ SAMPLE_CURRENT_MANIFEST, SAMPLE_NEW_MANIFEST,
+ resources_directory=tmp_dir)
self.assertEqual(paths_to_update[0]["path"], "some/file-to-update.txt")
self.assertEqual(paths_to_delete[0]["path"], "some/file-to-delete.txt")
+
+ shutil.rmtree(tmp_dir)
diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py
index bbaa26b..ea1d858 100644
--- a/ooni/tests/test_utils.py
+++ b/ooni/tests/test_utils.py
@@ -26,19 +26,19 @@ class TestUtils(unittest.TestCase):
def test_generate_filename(self):
filename = generate_filename(self.test_details)
- self.assertEqual(filename, 'foo-2016-01-01T012222Z')
+ self.assertEqual(filename, '20160101T012222Z-ZZ-AS0-foo')
def test_generate_filename_with_extension(self):
filename = generate_filename(self.test_details, extension=self.extension)
- self.assertEqual(filename, 'foo-2016-01-01T012222Z.ext')
+ self.assertEqual(filename, '20160101T012222Z-ZZ-AS0-foo.ext')
def test_generate_filename_with_prefix(self):
filename = generate_filename(self.test_details, prefix=self.prefix)
- self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z')
+ self.assertEqual(filename, 'prefix-20160101T012222Z-ZZ-AS0-foo')
def test_generate_filename_with_extension_and_prefix(self):
filename = generate_filename(self.test_details, prefix=self.prefix, extension=self.extension)
- self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z.ext')
+ self.assertEqual(filename, 'prefix-20160101T012222Z-ZZ-AS0-foo.ext')
def test_get_addresses(self):
addresses = net.getAddresses()
diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
index e18a6ee..df9dfec 100644
--- a/ooni/utils/onion.py
+++ b/ooni/utils/onion.py
@@ -244,7 +244,6 @@ class TorLauncherWithRetries(object):
@defer.inlineCallbacks
def _state_complete(self, state):
config.tor_state = state
- log.msg("Successfully bootstrapped Tor")
log.debug("We now have the following circuits: ")
for circuit in state.circuits.values():
log.debug(" * %s" % circuit)
More information about the tor-commits
mailing list