commit cf333afdf5dc08e17d02516a0af79552aa6f0275 Author: Arturo Filastò arturo@filasto.net Date: Thu Jul 28 01:38:41 2016 +0200
Cleanup some of the log outputs.
* Fix more unittests.
Will you be green green green? --- ooni/agent/scheduler.py | 2 +- ooni/director.py | 6 +++--- ooni/reporter.py | 7 +++++-- ooni/resources.py | 30 ++++++++++++++---------------- ooni/tests/test_geoip.py | 16 +++++++++++++++- ooni/tests/test_oonicli.py | 5 ++--- ooni/tests/test_resources.py | 16 +++++++++++++++- ooni/tests/test_utils.py | 8 ++++---- ooni/utils/onion.py | 1 - 9 files changed, 59 insertions(+), 32 deletions(-)
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 9784f8a..71a7edb 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -97,7 +97,7 @@ def run_system_tasks(no_input_store=False):
if no_input_store: log.debug("Not updating the inputs") - task_classes.pop(UpdateInputsAndResources) + task_classes.remove(UpdateInputsAndResources)
for task_class in task_classes: task = task_class() diff --git a/ooni/director.py b/ooni/director.py index ad6d1e2..f6311ac 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -141,7 +141,7 @@ class Director(object): self._tor_starting.addCallback(self._tor_startup_success)
def _tor_startup_failure(self, failure): - log.msg("Failed to start tor") + log.err("Failed to start tor") log.exception(failure) self._reset_tor_state() self.notify(DirectorEvent("error", @@ -382,7 +382,7 @@ class Director(object): yield self._tor_starting defer.returnValue(self._tor_state)
- log.msg("Starting Tor...") + log.msg("Starting Tor") self._tor_state = 'starting' if check_incoherences: try: @@ -405,7 +405,7 @@ class Director(object): data_dir = os.path.expanduser(config.tor.data_dir)
if not os.path.exists(data_dir): - log.msg("%s does not exist. Creating it." % data_dir) + log.debug("%s does not exist. Creating it." % data_dir) os.makedirs(data_dir) tor_config.DataDirectory = data_dir
diff --git a/ooni/reporter.py b/ooni/reporter.py index e2f155f..20a13f5 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -342,9 +342,12 @@ class OONIBReporter(OReporter): log.msg("Try running a different test or try reporting to a " "different collector.") raise errors.OONIBReportCreationError - except Exception, e: + except errors.OONIBError: log.err("Failed to connect to reporter backend") - log.exception(e) + raise errors.OONIBReportCreationError + except Exception as exc: + log.err("Failed to connect to reporter backend") + log.exception(exc) raise errors.OONIBReportCreationError
self.reportId = response['report_id'].encode('ascii') diff --git a/ooni/resources.py b/ooni/resources.py index 47ebf86..aef0f13 100644 --- a/ooni/resources.py +++ b/ooni/resources.py @@ -1,24 +1,25 @@ -import gzip import json -import shutil
from twisted.python.filepath import FilePath from twisted.internet import defer from twisted.web.client import downloadPage, getPage, HTTPClientFactory
+from ooni.utils import log, gunzip, rename +from ooni.settings import config + # Disable logs of HTTPClientFactory HTTPClientFactory.noisy = False
-from ooni.utils import log, gunzip, rename -from ooni.settings import config
class UpdateFailure(Exception): pass
+ def get_download_url(tag_name, filename): return ("https://github.com/OpenObservatory/ooni-resources/releases" "/download/{0}/{1}".format(tag_name, filename))
+ def get_current_version(): manifest = FilePath(config.resources_directory).child("manifest.json") if not manifest.exists(): @@ -27,6 +28,7 @@ def get_current_version(): manifest = json.load(f) return int(manifest["version"])
+ @defer.inlineCallbacks def get_latest_version(): """ @@ -41,7 +43,8 @@ def get_latest_version():
def get_out_of_date_resources(current_manifest, new_manifest, - country_code=None): + country_code=None, + resources_directory=config.resources_directory): current_res = {} new_res = {} for r in current_manifest["resources"]: @@ -51,11 +54,13 @@ def get_out_of_date_resources(current_manifest, new_manifest, new_res[r["path"]] = r
paths_to_delete = [ - current_res[path] for path in list(set(current_res.keys()) - - set(new_res.keys())) - ] + current_res[path] for path in list( + set(current_res.keys()) - + set(new_res.keys()) + ) + ] paths_to_update = [] - _resources = FilePath(config.resources_directory) + _resources = FilePath(resources_directory) for path, info in new_res.items(): if (country_code is not None and info["country_code"] != "ALL" and @@ -78,13 +83,6 @@ def get_out_of_date_resources(current_manifest, new_manifest,
return paths_to_update, paths_to_delete
-def gunzip(file_path): - tmp_location = FilePath(file_path).temporarySibling() - in_file = gzip.open(file_path) - with tmp_location.open('w') as out_file: - shutil.copyfileobj(in_file, out_file) - in_file.close() - rename(tmp_location.path, file_path)
@defer.inlineCallbacks def check_for_update(country_code=None): diff --git a/ooni/tests/test_geoip.py b/ooni/tests/test_geoip.py index 8eb964d..b5e76f3 100644 --- a/ooni/tests/test_geoip.py +++ b/ooni/tests/test_geoip.py @@ -1,4 +1,5 @@ - +import os +import shutil from twisted.internet import defer
from ooni.tests import is_internet_connected, bases @@ -23,6 +24,17 @@ class TestGeoIP(bases.ConfigTestCase): assert len(res.split('.')) == 4
def test_geoip_database_version(self): + maxmind_dir = os.path.join(self.config.resources_directory, + 'maxmind-geoip') + try: + os.mkdir(maxmind_dir) + except OSError: + pass + with open(os.path.join(maxmind_dir, 'GeoIP.dat'), 'w+') as f: + f.write("XXX") + with open(os.path.join(maxmind_dir, 'GeoIPASNum.dat'), 'w+') as f: + f.write("XXX") + version = geoip.database_version() assert 'GeoIP' in version.keys() assert 'GeoIPASNum' in version.keys() @@ -31,3 +43,5 @@ class TestGeoIP(bases.ConfigTestCase): assert isinstance(version['GeoIP']['timestamp'], float) assert len(version['GeoIPASNum']['sha256']) == 64 assert isinstance(version['GeoIPASNum']['timestamp'], float) + + shutil.rmtree(maxmind_dir) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index c3facee..06a21ff 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -55,7 +55,6 @@ advanced: oonid_api_port: 8042 tor: socks_port: 9050 - """ % config.data_directory
@@ -149,8 +148,8 @@ class TestRunDirector(ConfigTestCase):
yield self.run_helper('blocking/dns_consistency', ['-b', '8.8.8.8:53', - '-t', '8.8.8.8', - '-f', 'example-input.txt'], + '-t', '8.8.8.8', + '-f', 'example-input.txt'], verify_function)
@defer.inlineCallbacks diff --git a/ooni/tests/test_resources.py b/ooni/tests/test_resources.py index 45473e9..6d1bb3c 100644 --- a/ooni/tests/test_resources.py +++ b/ooni/tests/test_resources.py @@ -1,3 +1,7 @@ +import os +import shutil +import tempfile + from ooni.resources import get_out_of_date_resources, check_for_update from ooni.tests.bases import ConfigTestCase
@@ -36,7 +40,17 @@ class TestResourceUpdate(ConfigTestCase): return check_for_update()
def test_resources_out_of_date(self): + tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(tmp_dir, 'some')) + original_paths = map(lambda r: r['path'], + SAMPLE_CURRENT_MANIFEST['resources']) + for path in original_paths: + with open(os.path.join(tmp_dir, path), 'w+'): + pass paths_to_update, paths_to_delete = get_out_of_date_resources( - SAMPLE_CURRENT_MANIFEST, SAMPLE_NEW_MANIFEST) + SAMPLE_CURRENT_MANIFEST, SAMPLE_NEW_MANIFEST, + resources_directory=tmp_dir) self.assertEqual(paths_to_update[0]["path"], "some/file-to-update.txt") self.assertEqual(paths_to_delete[0]["path"], "some/file-to-delete.txt") + + shutil.rmtree(tmp_dir) diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index bbaa26b..ea1d858 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -26,19 +26,19 @@ class TestUtils(unittest.TestCase):
def test_generate_filename(self): filename = generate_filename(self.test_details) - self.assertEqual(filename, 'foo-2016-01-01T012222Z') + self.assertEqual(filename, '20160101T012222Z-ZZ-AS0-foo')
def test_generate_filename_with_extension(self): filename = generate_filename(self.test_details, extension=self.extension) - self.assertEqual(filename, 'foo-2016-01-01T012222Z.ext') + self.assertEqual(filename, '20160101T012222Z-ZZ-AS0-foo.ext')
def test_generate_filename_with_prefix(self): filename = generate_filename(self.test_details, prefix=self.prefix) - self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z') + self.assertEqual(filename, 'prefix-20160101T012222Z-ZZ-AS0-foo')
def test_generate_filename_with_extension_and_prefix(self): filename = generate_filename(self.test_details, prefix=self.prefix, extension=self.extension) - self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z.ext') + self.assertEqual(filename, 'prefix-20160101T012222Z-ZZ-AS0-foo.ext')
def test_get_addresses(self): addresses = net.getAddresses() diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index e18a6ee..df9dfec 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -244,7 +244,6 @@ class TorLauncherWithRetries(object): @defer.inlineCallbacks def _state_complete(self, state): config.tor_state = state - log.msg("Successfully bootstrapped Tor") log.debug("We now have the following circuits: ") for circuit in state.circuits.values(): log.debug(" * %s" % circuit)