[tor-commits] [ooni-probe/master] Fix unittests
art at torproject.org
art at torproject.org
Mon Sep 19 12:14:24 UTC 2016
commit 2f582d81059a42af499264a7f05b163437386f16
Author: Arturo Filastò <arturo at filasto.net>
Date: Tue Jul 26 15:15:04 2016 +0200
Fix unittests
* Improvements to the web UI
---
ooni/deck.py | 24 ++++++++++++------------
ooni/results.py | 1 +
ooni/settings.py | 7 ++++---
ooni/tests/test_deck.py | 22 +++++++++++++---------
ooni/tests/test_nettest.py | 16 +++++++---------
ooni/ui/web/client/index.html | 2 +-
ooni/ui/web/server.py | 10 ++++++++++
ooni/utils/__init__.py | 4 ++--
8 files changed, 50 insertions(+), 36 deletions(-)
diff --git a/ooni/deck.py b/ooni/deck.py
index eacd256..7b85b5c 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -594,6 +594,13 @@ class DeckTask(object):
self._load(data)
+ def _get_option(self, name, task_data, default=None):
+ try:
+ return self.global_options[name]
+ except KeyError:
+ return task_data.pop(name,
+ self.parent_metadata.get(name, default))
+
def _load_ooni(self, task_data):
required_keys = ["test_name"]
for required_key in required_keys:
@@ -604,20 +611,13 @@ class DeckTask(object):
nettest_path = nettest_to_path(task_data.pop("test_name"),
self._arbitrary_paths)
- try:
- annotations = task_data.pop('annotations')
- except KeyError:
- annotations = self.parent_metadata.get('annotations', {})
-
- try:
- collector_address = task_data.pop('collector')
- except KeyError:
- collector_address = self.parent_metadata.get('collector', None)
+ annotations = self._get_option('annotations', task_data, {})
+ collector_address = self._get_option('collector', task_data, {})
try:
- self.output_path = task_data.pop('reportfile')
+ self.output_path = self.global_options['reportfile']
except KeyError:
- self.output_path = self.global_options.get('reportfile', None)
+ self.output_path = task_data.pop('reportfile', None)
if task_data.get('no-collector', False):
collector_address = None
@@ -777,7 +777,7 @@ class NGDeck(object):
def load(self, deck_data, global_options=None):
if global_options is not None:
- self.global_options = global_options
+ self.global_options = normalize_options(global_options)
if isinstance(deck_data, list):
deck_data = convert_legacy_deck(deck_data)
diff --git a/ooni/results.py b/ooni/results.py
index 21fe997..39477a1 100644
--- a/ooni/results.py
+++ b/ooni/results.py
@@ -23,6 +23,7 @@ def generate_summary(input_file, output_file):
result = getattr(Process, entry['test_name'])(entry)
result['idx'] = idx
results['test_name'] = entry['test_name']
+ results['test_start_time'] = entry['test_start_time']
results['country_code'] = entry['probe_cc']
results['asn'] = entry['probe_asn']
results['results'] = results.get('results', [])
diff --git a/ooni/settings.py b/ooni/settings.py
index 0491f6e..b73e2f2 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -131,10 +131,11 @@ class OConfig(object):
self._custom_home = custom_home
self.set_paths()
- if not os.path.isdir(self.ooni_home):
+ ooni_home = self.ooni_home
+ if not os.path.isdir(ooni_home):
print "Ooni home directory does not exist."
- print "Creating it in '%s'." % self.ooni_home
- os.mkdir(self.ooni_home)
+ print "Creating it in '%s'." % ooni_home
+ os.mkdir(ooni_home)
os.mkdir(self.inputs_directory)
os.mkdir(self.decks_directory)
diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py
index 3d07959..7c18647 100644
--- a/ooni/tests/test_deck.py
+++ b/ooni/tests/test_deck.py
@@ -105,7 +105,7 @@ class TestDeck(BaseTestCase, ConfigTestCase):
def test_open_deck(self):
deck = NGDeck()
deck.open(self.deck_file)
- assert len(deck.tasks.ooni['net_test_loaders']) == 1
+ assert len(deck.tasks) == 1
def test_load_deck_with_global_options(self):
global_options = {
@@ -115,11 +115,11 @@ class TestDeck(BaseTestCase, ConfigTestCase):
deck = NGDeck(global_options=global_options)
deck.open(self.deck_file)
self.assertEqual(
- deck.tasks.ooni['net_test_loaders'][0].annotations,
+ deck.tasks[0].ooni['net_test_loader'].annotations,
global_options['annotations']
)
self.assertEqual(
- deck.tasks.ooni['net_test_loaders'][0].collector.base_address,
+ deck.tasks[0].ooni['net_test_loader'].collector.base_address,
global_options['collector'].replace("httpo://", "http://")
)
@@ -132,21 +132,25 @@ class TestDeck(BaseTestCase, ConfigTestCase):
deck.open(self.deck_file)
self.assertEqual(
- len(deck.tasks.ooni['net_test_loaders'][0].missingTestHelpers), 1)
+ len(deck.tasks[0].ooni['net_test_loader'].missingTestHelpers),
+ 1
+ )
yield lookup_collector_and_test_helpers(
- net_test_loaders=deck.netTestLoaders,
- preferred_backend=deck.preferred_backend,
+ net_test_loaders=[deck.tasks[0].ooni['net_test_loader']],
+ preferred_backend='onion',
bouncer=deck.bouncer
)
self.assertEqual(
- deck.tasks.ooni['net_test_loaders'][0].collector.settings['address'],
+ deck.tasks[0].ooni['net_test_loader'].collector.settings['address'],
'httpo://thirteenchars123.onion'
)
- self.assertEqual(deck.netTestLoaders[0].localOptions['backend'],
- '127.0.0.1')
+ self.assertEqual(
+ deck.tasks[0].ooni['net_test_loader'].localOptions['backend'],
+ '127.0.0.1'
+ )
def test_deck_with_many_tests(self):
diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py
index 1592149..572378a 100644
--- a/ooni/tests/test_nettest.py
+++ b/ooni/tests/test_nettest.py
@@ -1,5 +1,6 @@
import os
import yaml
+import json
from tempfile import mkstemp
from twisted.trial import unittest
@@ -231,7 +232,6 @@ class TestNetTest(ConfigTestCase):
super(TestNetTest, self).setUp()
def tearDown(self):
- super(TestNetTest, self).tearDown()
os.remove(dummyInputFile)
if self.filename != "":
os.remove(self.filename)
@@ -381,17 +381,16 @@ class TestNetTest(ConfigTestCase):
ntl.checkOptions()
director = Director()
- self.filename = 'dummy_report.yamloo'
+ self.filename = 'dummy_report.njson'
d = director.start_net_test_loader(ntl, self.filename)
@d.addCallback
def complete(result):
with open(self.filename) as report_file:
- all_report_entries = yaml.safe_load_all(report_file)
- header = all_report_entries.next()
- results_case_a = all_report_entries.next()
+ all_report_entries = map(json.loads, report_file)
+ results_case_a = all_report_entries[0]['test_keys']
aa_test, ab_test, ac_test = results_case_a.get('results', [])
- results_case_b = all_report_entries.next()
+ results_case_b = all_report_entries[1]['test_keys']
ba_test = results_case_b.get('results', [])[0]
# Within a NetTestCase an inputs object will be consistent
self.assertEqual(aa_test, ab_test, ac_test)
@@ -409,14 +408,13 @@ class TestNetTest(ConfigTestCase):
ntl.checkOptions()
director = Director()
- self.filename = 'dummy_report.yamloo'
+ self.filename = 'dummy_report.njson'
d = director.start_net_test_loader(ntl, self.filename)
@d.addCallback
def complete(result):
with open(self.filename) as report_file:
- all_report_entries = yaml.safe_load_all(report_file)
- header = all_report_entries.next()
+ all_report_entries = map(json.loads, report_file)
results = [x['input'] for x in all_report_entries]
self.assertEqual(results, [9, 8, 7, 6, 5, 5, 3, 2, 1, 0])
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index e90ef83..7859216 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
<app>
Loading...
</app>
- <script type="text/javascript" src="app.bundle.js?6b15c1dd202a0f5a80e7"></script></body>
+ <script type="text/javascript" src="app.bundle.js?de2d27ce59f4cee8dd96"></script></body>
</html>
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index a862fe7..f9886ac 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -17,6 +17,7 @@ from ooni.deck import NGDeck
from ooni.settings import config
from ooni.utils import log
from ooni.director import DirectorEvent
+from ooni.results import generate_summary
config.advanced.debug = True
@@ -193,6 +194,7 @@ class WebUIAPI(object):
'test_start_time': 'some start time'
}
self.status_poller.notify()
+ deck.setup()
d = deck.run(self.director)
d.addCallback(lambda _:
self.completed_measurement(task_id))
@@ -290,6 +292,14 @@ class WebUIAPI(object):
if measurement_dir.child("measurements.njson.progress").exists():
raise WebUIError(400, "measurement in progress")
+ if not measurement_dir.child("summary.json").exists():
+ # XXX we can perhaps remove this.
+ generate_summary(
+ measurement_dir.child("measurements.njson").path,
+ measurement_dir.child("summary.json").path
+ )
+ raise WebUIError(400, "measurement in progress")
+
summary = measurement_dir.child("summary.json")
with summary.open("r") as f:
r = json.load(f)
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 1aaac7b..d202d97 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -109,8 +109,8 @@ def generate_filename(test_details, prefix=None, extension=None):
kwargs["extension"] = extension
filename_format += ".{extension}"
kwargs['test_name'] = test_details['test_name']
- kwargs['probe_cc'] = test_details['probe_cc']
- kwargs['probe_asn'] = test_details['probe_asn']
+ kwargs['probe_cc'] = test_details.get('probe_cc', 'ZZ')
+ kwargs['probe_asn'] = test_details.get('probe_asn', 'AS0')
kwargs['timestamp'] = datetime.strptime(test_details['test_start_time'],
LONG_DATE).strftime(SHORT_DATE)
return filename_format.format(**kwargs)
More information about the tor-commits
mailing list