September 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Cleanup some of the log outputs.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit cf333afdf5dc08e17d02516a0af79552aa6f0275 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jul 28 01:38:41 2016 +0200 Cleanup some of the log outputs. * Fix more unittests. Will you be green green green? --- ooni/agent/scheduler.py | 2 +- ooni/director.py | 6 +++--- ooni/reporter.py | 7 +++++-- ooni/resources.py | 30 ++++++++++++++---------------- ooni/tests/test_geoip.py | 16 +++++++++++++++- ooni/tests/test_oonicli.py | 5 ++--- ooni/tests/test_resources.py | 16 +++++++++++++++- ooni/tests/test_utils.py | 8 ++++---- ooni/utils/onion.py | 1 - 9 files changed, 59 insertions(+), 32 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 9784f8a..71a7edb 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -97,7 +97,7 @@ def run_system_tasks(no_input_store=False): if no_input_store: log.debug("Not updating the inputs") - task_classes.pop(UpdateInputsAndResources) + task_classes.remove(UpdateInputsAndResources) for task_class in task_classes: task = task_class() diff --git a/ooni/director.py b/ooni/director.py index ad6d1e2..f6311ac 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -141,7 +141,7 @@ class Director(object): self._tor_starting.addCallback(self._tor_startup_success) def _tor_startup_failure(self, failure): - log.msg("Failed to start tor") + log.err("Failed to start tor") log.exception(failure) self._reset_tor_state() self.notify(DirectorEvent("error", @@ -382,7 +382,7 @@ class Director(object): yield self._tor_starting defer.returnValue(self._tor_state) - log.msg("Starting Tor...") + log.msg("Starting Tor") self._tor_state = 'starting' if check_incoherences: try: @@ -405,7 +405,7 @@ class Director(object): data_dir = os.path.expanduser(config.tor.data_dir) if not os.path.exists(data_dir): - log.msg("%s does not exist. Creating it." % data_dir) + log.debug("%s does not exist. Creating it." % data_dir) os.makedirs(data_dir) tor_config.DataDirectory = data_dir diff --git a/ooni/reporter.py b/ooni/reporter.py index e2f155f..20a13f5 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -342,9 +342,12 @@ class OONIBReporter(OReporter): log.msg("Try running a different test or try reporting to a " "different collector.") raise errors.OONIBReportCreationError - except Exception, e: + except errors.OONIBError: log.err("Failed to connect to reporter backend") - log.exception(e) + raise errors.OONIBReportCreationError + except Exception as exc: + log.err("Failed to connect to reporter backend") + log.exception(exc) raise errors.OONIBReportCreationError self.reportId = response['report_id'].encode('ascii') diff --git a/ooni/resources.py b/ooni/resources.py index 47ebf86..aef0f13 100644 --- a/ooni/resources.py +++ b/ooni/resources.py @@ -1,24 +1,25 @@ -import gzip import json -import shutil from twisted.python.filepath import FilePath from twisted.internet import defer from twisted.web.client import downloadPage, getPage, HTTPClientFactory +from ooni.utils import log, gunzip, rename +from ooni.settings import config + # Disable logs of HTTPClientFactory HTTPClientFactory.noisy = False -from ooni.utils import log, gunzip, rename -from ooni.settings import config class UpdateFailure(Exception): pass + def get_download_url(tag_name, filename): return ("https://github.com/OpenObservatory/ooni-resources/releases" "/download/{0}/{1}".format(tag_name, filename)) + def get_current_version(): manifest = FilePath(config.resources_directory).child("manifest.json") if not manifest.exists(): @@ -27,6 +28,7 @@ def get_current_version(): manifest = json.load(f) return int(manifest["version"]) + @defer.inlineCallbacks def get_latest_version(): """ @@ -41,7 +43,8 @@ def get_latest_version(): def get_out_of_date_resources(current_manifest, new_manifest, - country_code=None): + country_code=None, + resources_directory=config.resources_directory): current_res = {} new_res = {} for r in current_manifest["resources"]: @@ -51,11 +54,13 @@ def get_out_of_date_resources(current_manifest, new_manifest, new_res[r["path"]] = r paths_to_delete = [ - current_res[path] for path in list(set(current_res.keys()) - - set(new_res.keys())) - ] + current_res[path] for path in list( + set(current_res.keys()) - + set(new_res.keys()) + ) + ] paths_to_update = [] - _resources = FilePath(config.resources_directory) + _resources = FilePath(resources_directory) for path, info in new_res.items(): if (country_code is not None and info["country_code"] != "ALL" and @@ -78,13 +83,6 @@ def get_out_of_date_resources(current_manifest, new_manifest, return paths_to_update, paths_to_delete -def gunzip(file_path): - tmp_location = FilePath(file_path).temporarySibling() - in_file = gzip.open(file_path) - with tmp_location.open('w') as out_file: - shutil.copyfileobj(in_file, out_file) - in_file.close() - rename(tmp_location.path, file_path) @defer.inlineCallbacks def check_for_update(country_code=None): diff --git a/ooni/tests/test_geoip.py b/ooni/tests/test_geoip.py index 8eb964d..b5e76f3 100644 --- a/ooni/tests/test_geoip.py +++ b/ooni/tests/test_geoip.py @@ -1,4 +1,5 @@ - +import os +import shutil from twisted.internet import defer from ooni.tests import is_internet_connected, bases @@ -23,6 +24,17 @@ class TestGeoIP(bases.ConfigTestCase): assert len(res.split('.')) == 4 def test_geoip_database_version(self): + maxmind_dir = os.path.join(self.config.resources_directory, + 'maxmind-geoip') + try: + os.mkdir(maxmind_dir) + except OSError: + pass + with open(os.path.join(maxmind_dir, 'GeoIP.dat'), 'w+') as f: + f.write("XXX") + with open(os.path.join(maxmind_dir, 'GeoIPASNum.dat'), 'w+') as f: + f.write("XXX") + version = geoip.database_version() assert 'GeoIP' in version.keys() assert 'GeoIPASNum' in version.keys() @@ -31,3 +43,5 @@ class TestGeoIP(bases.ConfigTestCase): assert isinstance(version['GeoIP']['timestamp'], float) assert len(version['GeoIPASNum']['sha256']) == 64 assert isinstance(version['GeoIPASNum']['timestamp'], float) + + shutil.rmtree(maxmind_dir) diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py index c3facee..06a21ff 100644 --- a/ooni/tests/test_oonicli.py +++ b/ooni/tests/test_oonicli.py @@ -55,7 +55,6 @@ advanced: oonid_api_port: 8042 tor: socks_port: 9050 - """ % config.data_directory @@ -149,8 +148,8 @@ class TestRunDirector(ConfigTestCase): yield self.run_helper('blocking/dns_consistency', ['-b', '8.8.8.8:53', - '-t', '8.8.8.8', - '-f', 'example-input.txt'], + '-t', '8.8.8.8', + '-f', 'example-input.txt'], verify_function) @defer.inlineCallbacks diff --git a/ooni/tests/test_resources.py b/ooni/tests/test_resources.py index 45473e9..6d1bb3c 100644 --- a/ooni/tests/test_resources.py +++ b/ooni/tests/test_resources.py @@ -1,3 +1,7 @@ +import os +import shutil +import tempfile + from ooni.resources import get_out_of_date_resources, check_for_update from ooni.tests.bases import ConfigTestCase @@ -36,7 +40,17 @@ class TestResourceUpdate(ConfigTestCase): return check_for_update() def test_resources_out_of_date(self): + tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(tmp_dir, 'some')) + original_paths = map(lambda r: r['path'], + SAMPLE_CURRENT_MANIFEST['resources']) + for path in original_paths: + with open(os.path.join(tmp_dir, path), 'w+'): + pass paths_to_update, paths_to_delete = get_out_of_date_resources( - SAMPLE_CURRENT_MANIFEST, SAMPLE_NEW_MANIFEST) + SAMPLE_CURRENT_MANIFEST, SAMPLE_NEW_MANIFEST, + resources_directory=tmp_dir) self.assertEqual(paths_to_update[0]["path"], "some/file-to-update.txt") self.assertEqual(paths_to_delete[0]["path"], "some/file-to-delete.txt") + + shutil.rmtree(tmp_dir) diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py index bbaa26b..ea1d858 100644 --- a/ooni/tests/test_utils.py +++ b/ooni/tests/test_utils.py @@ -26,19 +26,19 @@ class TestUtils(unittest.TestCase): def test_generate_filename(self): filename = generate_filename(self.test_details) - self.assertEqual(filename, 'foo-2016-01-01T012222Z') + self.assertEqual(filename, '20160101T012222Z-ZZ-AS0-foo') def test_generate_filename_with_extension(self): filename = generate_filename(self.test_details, extension=self.extension) - self.assertEqual(filename, 'foo-2016-01-01T012222Z.ext') + self.assertEqual(filename, '20160101T012222Z-ZZ-AS0-foo.ext') def test_generate_filename_with_prefix(self): filename = generate_filename(self.test_details, prefix=self.prefix) - self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z') + self.assertEqual(filename, 'prefix-20160101T012222Z-ZZ-AS0-foo') def test_generate_filename_with_extension_and_prefix(self): filename = generate_filename(self.test_details, prefix=self.prefix, extension=self.extension) - self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z.ext') + self.assertEqual(filename, 'prefix-20160101T012222Z-ZZ-AS0-foo.ext') def test_get_addresses(self): addresses = net.getAddresses() diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index e18a6ee..df9dfec 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -244,7 +244,6 @@ class TorLauncherWithRetries(object): @defer.inlineCallbacks def _state_complete(self, state): config.tor_state = state - log.msg("Successfully bootstrapped Tor") log.debug("We now have the following circuits: ") for circuit in state.circuits.values(): log.debug(" * %s" % circuit)

1 0

[ooni-probe/master] Implement tasks and add new APIs
by art＠torproject.org 19 Sep '16

19 Sep '16

commit c71375eed5db6dd1e11881c7045c2392c68aa42c Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 29 00:20:45 2016 +0200 Implement tasks and add new APIs * Add new API endpoints for deleting and keeping reports * Implement tasks for uploading not submitted reports * Implement tasks for cleaning up old reports --- ooni/agent/agent.py | 11 +++++++ ooni/agent/scheduler.py | 69 ++++++++++++++++++++++++++++++++++++------- ooni/deck/deck.py | 2 +- ooni/measurements.py | 67 +++++++++++++++++++++++++++++++++++++++++ ooni/results.py | 40 ------------------------- ooni/scripts/oonireport.py | 8 +++++ ooni/ui/web/client/index.html | 2 +- ooni/ui/web/server.py | 40 +++++++++++++++++++++++-- ooni/utils/__init__.py | 6 ++-- 9 files changed, 188 insertions(+), 57 deletions(-) diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py index a1394f0..f5322ed 100644 --- a/ooni/agent/agent.py +++ b/ooni/agent/agent.py @@ -1,6 +1,7 @@ from twisted.application import service from ooni.director import Director from ooni.settings import config +from ooni.utils import log from ooni.ui.web.web import WebUIService from ooni.agent.scheduler import SchedulerService @@ -19,3 +20,13 @@ class AgentService(service.MultiService): self.scheduler_service = SchedulerService(director) self.scheduler_service.setServiceParent(self) + + def startService(self): + service.MultiService.startService(self) + + log.start() + + def stopService(self): + service.MultiService.stopService(self) + + log.stop() diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 1288d6c..167a14a 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -4,12 +4,17 @@ from twisted.application import service from twisted.internet import task, defer from twisted.python.filepath import FilePath +from ooni.scripts import oonireport from ooni import resources -from ooni.utils import log +from ooni.utils import log, SHORT_DATE from ooni.deck.store import input_store from ooni.settings import config from ooni.contrib import croniter from ooni.geoip import probe_ip +from ooni.measurements import list_measurements + +class DidNotRun(Exception): + pass class ScheduledTask(object): _time_format = "%Y-%m-%dT%H:%M:%SZ" @@ -58,7 +63,7 @@ class ScheduledTask(object): yield self._last_run_lock.deferUntilLocked() if not self.should_run: self._last_run_lock.unlock() - defer.returnValue(None) + raise DidNotRun try: yield self.task() self._update_last_run() @@ -68,7 +73,7 @@ class ScheduledTask(object): self._last_run_lock.unlock() class UpdateInputsAndResources(ScheduledTask): - identifier = "ooni-update-inputs" + identifier = "update-inputs" schedule = "@daily" @defer.inlineCallbacks @@ -78,13 +83,50 @@ class UpdateInputsAndResources(ScheduledTask): yield resources.check_for_update(probe_ip.geodata['countrycode']) yield input_store.update(probe_ip.geodata['countrycode']) -class CleanupInProgressReports(ScheduledTask): - identifier = 'ooni-cleanup-reports' + +class UploadReports(ScheduledTask): + """ + This task is used to submit to the collector reports that have not been + submitted and those that have been partially uploaded. + """ + identifier = 'upload-reports' + schedule = '@hourly' + + @defer.inlineCallbacks + def task(self): + yield oonireport.upload_all(upload_incomplete=True) + + +class DeleteOldReports(ScheduledTask): + """ + This task is used to delete reports that are older than a week. + """ + identifier = 'delete-old-reports' schedule = '@daily' -class UploadMissingReports(ScheduledTask): - identifier = 'ooni-cleanup-reports' - schedule = '@weekly' + def task(self): + measurement_path = FilePath(config.measurements_directory) + for measurement in list_measurements(): + if measurement['keep'] is True: + continue + delta = datetime.utcnow() - \ + datetime.strptime(measurement['test_start_time'], + SHORT_DATE) + if delta.days >= 7: + log.debug("Deleting old report {0}".format(measurement["id"])) + measurement_path.child(measurement['id']).remove() + +class SendHeartBeat(ScheduledTask): + """ + This task is used to send a heartbeat that the probe is still alive and + well. + """ + identifier = 'send-heartbeat' + schedule = '@hourly' + + def task(self): + # XXX implement this + pass # Order mattters SYSTEM_TASKS = [ @@ -122,8 +164,12 @@ class SchedulerService(service.MultiService): def schedule(self, task): self._scheduled_tasks.append(task) + def _task_did_not_run(self, failure, task): + failure.trap(DidNotRun) + log.debug("Did not run {0}".format(task.identifier)) + def _task_failed(self, failure, task): - log.debug("Failed to run {0}".format(task.identifier)) + log.err("Failed to run {0}".format(task.identifier)) log.exception(failure) def _task_success(self, result, task): @@ -137,13 +183,16 @@ class SchedulerService(service.MultiService): for task in self._scheduled_tasks: log.debug("Running task {0}".format(task.identifier)) d = task.run() - d.addErrback(self._task_failed, task) + d.addErrback(self._task_did_not_run, task) d.addCallback(self._task_success, task) + d.addErrback(self._task_failed, task) def startService(self): service.MultiService.startService(self) self.schedule(UpdateInputsAndResources()) + self.schedule(UploadReports()) + self.schedule(DeleteOldReports()) self._looping_call.start(self.interval) diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py index c9b3fc5..868bfb8 100644 --- a/ooni/deck/deck.py +++ b/ooni/deck/deck.py @@ -13,7 +13,7 @@ from ooni.deck.legacy import convert_legacy_deck from ooni.deck.store import input_store from ooni.geoip import probe_ip from ooni.nettest import NetTestLoader, nettest_to_path -from ooni.results import generate_summary +from ooni.measurements import generate_summary from ooni.settings import config from ooni.utils import log, generate_filename diff --git a/ooni/measurements.py b/ooni/measurements.py new file mode 100644 index 0000000..f830a99 --- /dev/null +++ b/ooni/measurements.py @@ -0,0 +1,67 @@ +import json +from twisted.python.filepath import FilePath +from ooni.settings import config + +class Process(): + supported_tests = [ + "web_connectivity" + ] + @staticmethod + def web_connectivity(entry): + result = {} + result['anomaly'] = False + if entry['test_keys']['blocking'] is not False: + result['anomaly'] = True + result['url'] = entry['input'] + return result + +def generate_summary(input_file, output_file): + results = {} + with open(input_file) as in_file: + for idx, line in enumerate(in_file): + entry = json.loads(line.strip()) + result = {} + if entry['test_name'] in Process.supported_tests: + result = getattr(Process, entry['test_name'])(entry) + result['idx'] = idx + results['test_name'] = entry['test_name'] + results['test_start_time'] = entry['test_start_time'] + results['country_code'] = entry['probe_cc'] + results['asn'] = entry['probe_asn'] + results['results'] = results.get('results', []) + results['results'].append(result) + + with open(output_file, "w") as fw: + json.dump(results, fw) + + +def list_measurements(): + measurements = [] + measurement_path = FilePath(config.measurements_directory) + for measurement_id in measurement_path.listdir(): + measurement = measurement_path.child(measurement_id) + completed = True + keep = False + if measurement.child("measurement.njson.progress").exists(): + completed = False + if measurement.child("keep").exists(): + keep = True + test_start_time, country_code, asn, test_name = \ + measurement_id.split("-")[:4] + measurements.append({ + "test_name": test_name, + "country_code": country_code, + "asn": asn, + "test_start_time": test_start_time, + "id": measurement_id, + "completed": completed, + "keep": keep + }) + return measurements + +if __name__ == "__main__": + import sys + if len(sys.argv) != 3: + print("Usage: {0} [input_file] [output_file]".format(sys.argv[0])) + sys.exit(1) + generate_summary(sys.argv[1], sys.argv[2]) diff --git a/ooni/results.py b/ooni/results.py deleted file mode 100644 index 39477a1..0000000 --- a/ooni/results.py +++ /dev/null @@ -1,40 +0,0 @@ -import json - -class Process(): - supported_tests = [ - "web_connectivity" - ] - @staticmethod - def web_connectivity(entry): - result = {} - result['anomaly'] = False - if entry['test_keys']['blocking'] is not False: - result['anomaly'] = True - result['url'] = entry['input'] - return result - -def generate_summary(input_file, output_file): - results = {} - with open(input_file) as in_file: - for idx, line in enumerate(in_file): - entry = json.loads(line.strip()) - result = {} - if entry['test_name'] in Process.supported_tests: - result = getattr(Process, entry['test_name'])(entry) - result['idx'] = idx - results['test_name'] = entry['test_name'] - results['test_start_time'] = entry['test_start_time'] - results['country_code'] = entry['probe_cc'] - results['asn'] = entry['probe_asn'] - results['results'] = results.get('results', []) - results['results'].append(result) - - with open(output_file, "w") as fw: - json.dump(results, fw) - -if __name__ == "__main__": - import sys - if len(sys.argv) != 3: - print("Usage: {0} [input_file] [output_file]".format(sys.argv[0])) - sys.exit(1) - generate_summary(sys.argv[1], sys.argv[2]) diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py index f59a843..13d8473 100644 --- a/ooni/scripts/oonireport.py +++ b/ooni/scripts/oonireport.py @@ -122,6 +122,14 @@ def upload_all(collector=None, bouncer=None, upload_incomplete=False): except Exception as exc: log.exception(exc) + if upload_incomplete: + reports_to_upload = yield oonib_report_log.get_incomplete() + for report_file, value in reports_to_upload: + try: + yield upload(report_file, collector, bouncer, + value['measurement_id']) + except Exception as exc: + log.exception(exc) def print_report(report_file, value): print("* %s" % report_file) diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index c6d83b7..7ba33fb 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?16bac0b4c21c5b120b04"></script></body> + <script type="text/javascript" src="app.bundle.js?afba2f26969b4c8f00ec"></script></body> </html> diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index a6fd315..9d98e62 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -20,7 +20,7 @@ from ooni.deck import NGDeck from ooni.settings import config from ooni.utils import log from ooni.director import DirectorEvent -from ooni.results import generate_summary +from ooni.measurements import generate_summary from ooni.geoip import probe_ip config.advanced.debug = True @@ -64,7 +64,8 @@ def xsrf_protect(check=True): if (token_cookie != instance._xsrf_token and instance._enable_xsrf_protection): request.addCookie(u'XSRF-TOKEN', - instance._xsrf_token) + instance._xsrf_token, + path=u'/') if should_check and token_cookie != token_header: raise WebUIError(404, "Invalid XSRF token") return f(instance, request, *a, **kw) @@ -353,6 +354,41 @@ class WebUIAPI(object): return self.render_json(r, request) + @app.route('/api/measurement/<string:measurement_id>', methods=["DELETE"]) + @xsrf_protect(check=True) + def api_measurement_delete(self, request, measurement_id): + try: + measurement_dir = self.measurement_path.child(measurement_id) + except InsecurePath: + raise WebUIError(500, "invalid measurement id") + + if measurement_dir.child("measurements.njson.progress").exists(): + raise WebUIError(400, "measurement in progress") + + try: + measurement_dir.remove() + except: + raise WebUIError(400, "Failed to delete report") + + return self.render_json({"result": "ok"}, request) + + @app.route('/api/measurement/<string:measurement_id>/keep', methods=["POST"]) + @xsrf_protect(check=True) + def api_measurement_keep(self, request, measurement_id): + try: + measurement_dir = self.measurement_path.child(measurement_id) + except InsecurePath: + raise WebUIError(500, "invalid measurement id") + + if measurement_dir.child("measurements.njson.progress").exists(): + raise WebUIError(400, "measurement in progress") + + summary = measurement_dir.child("keep") + with summary.open("w+") as f: + pass + + return self.render_json({"result": "ok"}, request) + @app.route('/api/measurement/<string:measurement_id>/<int:idx>', methods=["GET"]) @xsrf_protect(check=False) diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index f8ee953..35d419d 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -92,6 +92,9 @@ def randomStr(length, num=True): chars += string.digits return ''.join(random.choice(chars) for x in range(length)) +LONG_DATE = "%Y-%m-%d %H:%M:%S" +SHORT_DATE = "%Y%m%dT%H%M%SZ" + def generate_filename(test_details, prefix=None, extension=None): """ Returns a filename for every test execution. @@ -99,9 +102,6 @@ def generate_filename(test_details, prefix=None, extension=None): It's used to assure that all files of a certain test have a common basename but different extension. """ - LONG_DATE = "%Y-%m-%d %H:%M:%S" - SHORT_DATE = "%Y%m%dT%H%M%SZ" - kwargs = {} filename_format = "" if prefix is not None:

1 0

[ooni-probe/master] Add minimal outline of the ooniprobe-agent and new deck format
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 7829363f1066a469995c0410025d7e895522363f Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Jul 25 16:08:05 2016 +0200 Add minimal outline of the ooniprobe-agent and new deck format * Use DuckDuckGo to perform geoip lookups instead of torproject.org * Big refactoring of the Director --- ooni/agent/__init__.py | 0 ooni/agent/agent.py | 21 ++ ooni/agent/scheduler.py | 27 ++ ooni/deck.py | 715 +++++++++++++++++++++++++++++------------ ooni/director.py | 231 ++++++++----- ooni/geoip.py | 8 +- ooni/measurements.py | 43 --- ooni/nettest.py | 3 +- ooni/results.py | 39 +++ ooni/tests/bases.py | 2 +- ooni/tests/test_deck.py | 57 +++- ooni/tests/test_director.py | 12 +- ooni/tests/test_nettest.py | 8 +- ooni/tests/test_oonideckgen.py | 7 +- ooni/tests/test_oonireport.py | 18 +- ooni/ui/cli.py | 8 +- ooni/ui/web/client/index.html | 2 +- ooni/ui/web/server.py | 203 ++++++------ ooni/ui/web/web.py | 56 +--- ooni/utils/onion.py | 14 - 20 files changed, 974 insertions(+), 500 deletions(-) diff --git a/ooni/agent/__init__.py b/ooni/agent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py new file mode 100644 index 0000000..a1394f0 --- /dev/null +++ b/ooni/agent/agent.py @@ -0,0 +1,21 @@ +from twisted.application import service +from ooni.director import Director +from ooni.settings import config + +from ooni.ui.web.web import WebUIService +from ooni.agent.scheduler import SchedulerService + +class AgentService(service.MultiService): + def __init__(self): + service.MultiService.__init__(self) + + director = Director() + config.set_paths() + config.initialize_ooni_home() + config.read_config_file() + + self.web_ui_service = WebUIService(director) + self.web_ui_service.setServiceParent(self) + + self.scheduler_service = SchedulerService(director) + self.scheduler_service.setServiceParent(self) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py new file mode 100644 index 0000000..1004597 --- /dev/null +++ b/ooni/agent/scheduler.py @@ -0,0 +1,27 @@ +from twisted.application import service +from twisted.internet import task + +class SchedulerService(service.MultiService): + """ + This service is responsible for running the periodic tasks. + """ + def __init__(self, director, interval=30): + service.MultiService.__init__(self) + self.director = director + self.interval = interval + self._looping_call = task.LoopingCall(self._should_run) + + def _should_run(self): + """ + This function is called every self.interval seconds to check + which periodic tasks should be run. + """ + pass + + def startService(self): + service.MultiService.startService(self) + self._looping_call.start(self.interval) + + def stopService(self): + service.MultiService.stopService(self) + self._looping_call.stop() diff --git a/ooni/deck.py b/ooni/deck.py index 9f17530..1746d26 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -1,27 +1,28 @@ # -*- coding: utf-8 -*- -import csv import os -import yaml +import csv import json +from copy import deepcopy from hashlib import sha256 -from datetime import datetime -from ooni.backend_client import CollectorClient, BouncerClient -from ooni.backend_client import WebConnectivityClient, guess_backend_type -from ooni.nettest import NetTestLoader -from ooni.settings import config -from ooni.otime import timestampNowISO8601UTC +import yaml -from ooni.resources.update import check_for_update +from twisted.internet import defer +from twisted.python.filepath import FilePath -from ooni.utils import log from ooni import constants from ooni import errors as e +from ooni.backend_client import CollectorClient, BouncerClient +from ooni.backend_client import WebConnectivityClient, guess_backend_type +from ooni.nettest import NetTestLoader +from ooni.otime import timestampNowISO8601UTC +from ooni.resources import check_for_update +from ooni.settings import config +from ooni.utils import generate_filename +from ooni.utils import log -from twisted.python.filepath import FilePath -from twisted.internet import defer - +from ooni.results import generate_summary class InputFile(object): def __init__(self, input_hash, base_path=config.inputs_directory): @@ -116,6 +117,25 @@ def nettest_to_path(path, allow_arbitrary_paths=False): return found_path +def get_preferred_bouncer(): + preferred_backend = config.advanced.get( + "preferred_backend", "onion" + ) + bouncer_address = getattr( + constants, "CANONICAL_BOUNCER_{0}".format( + preferred_backend.upper() + ) + ) + if preferred_backend == "cloudfront": + return BouncerClient( + settings={ + 'address': bouncer_address[0], + 'front': bouncer_address[1], + 'type': 'cloudfront' + }) + else: + return BouncerClient(bouncer_address) + class Deck(InputFile): # this exists so we can mock it out in unittests _BouncerClient = BouncerClient @@ -227,175 +247,10 @@ class Deck(InputFile): if self.bouncer: log.msg("Looking up collector and test helpers with {0}".format( self.bouncer.base_address)) - yield self.lookupCollectorAndTestHelpers() - - - def sortAddressesByPriority(self, priority_address, alternate_addresses): - prioritised_addresses = [] - - backend_type = guess_backend_type(priority_address) - priority_address = { - 'address': priority_address, - 'type': backend_type - } - address_priority = ['onion', 'https', 'cloudfront', 'http'] - address_priority.remove(self.preferred_backend) - address_priority.insert(0, self.preferred_backend) - - def filter_by_type(collectors, collector_type): - return filter(lambda x: x['type'] == collector_type, collectors) - - if (priority_address['type'] != self.preferred_backend): - valid_alternatives = filter_by_type(alternate_addresses, - self.preferred_backend) - if len(valid_alternatives) > 0: - alternate_addresses += [priority_address] - priority_address = valid_alternatives[0] - alternate_addresses.remove(priority_address) - - prioritised_addresses += [priority_address] - for address_type in address_priority: - prioritised_addresses += filter_by_type(alternate_addresses, - address_type) - - return prioritised_addresses - - @defer.inlineCallbacks - def getReachableCollector(self, collector_address, collector_alternate): - # We prefer onion collector to https collector to cloudfront - # collectors to plaintext collectors - for collector_settings in self.sortAddressesByPriority(collector_address, - collector_alternate): - collector = self._CollectorClient(settings=collector_settings) - if not collector.isSupported(): - log.err("Unsupported %s collector %s" % ( - collector_settings['type'], - collector_settings['address'])) - continue - reachable = yield collector.isReachable() - if not reachable: - log.err("Unreachable %s collector %s" % ( - collector_settings['type'], - collector_settings['address'])) - continue - defer.returnValue(collector) - - raise e.NoReachableCollectors - - @defer.inlineCallbacks - def getReachableTestHelper(self, test_helper_name, test_helper_address, - test_helper_alternate): - # For the moment we look for alternate addresses only of - # web_connectivity test helpers. - if test_helper_name == 'web-connectivity': - for web_connectivity_settings in self.sortAddressesByPriority( - test_helper_address, test_helper_alternate): - web_connectivity_test_helper = WebConnectivityClient( - settings=web_connectivity_settings) - if not web_connectivity_test_helper.isSupported(): - log.err("Unsupported %s web_connectivity test_helper " - "%s" % ( - web_connectivity_settings['type'], - web_connectivity_settings['address'] - )) - continue - reachable = yield web_connectivity_test_helper.isReachable() - if not reachable: - log.err("Unreachable %s web_connectivity test helper %s" % ( - web_connectivity_settings['type'], - web_connectivity_settings['address'] - )) - continue - defer.returnValue(web_connectivity_settings) - raise e.NoReachableTestHelpers - else: - defer.returnValue(test_helper_address.encode('ascii')) - - @defer.inlineCallbacks - def getReachableTestHelpersAndCollectors(self, net_tests): - for net_test in net_tests: - - primary_address = net_test['collector'] - alternate_addresses = net_test.get('collector-alternate', []) - net_test['collector'] = yield self.getReachableCollector( - primary_address, - alternate_addresses - ) - - for test_helper_name, test_helper_address in net_test['test-helpers'].items(): - test_helper_alternate = \ - net_test.get('test-helpers-alternate', {}).get(test_helper_name, []) - net_test['test-helpers'][test_helper_name] = \ - yield self.getReachableTestHelper( - test_helper_name, - test_helper_address, - test_helper_alternate) - - defer.returnValue(net_tests) - - @defer.inlineCallbacks - def lookupCollectorAndTestHelpers(self): - required_nettests = [] - - requires_test_helpers = False - requires_collector = False - for net_test_loader in self.netTestLoaders: - nettest = { - 'name': net_test_loader.testName, - 'version': net_test_loader.testVersion, - 'test-helpers': [], - 'input-hashes': [x['hash'] for x in net_test_loader.inputFiles] - } - if not net_test_loader.collector and not self.no_collector: - requires_collector = True - - if len(net_test_loader.missingTestHelpers) > 0: - requires_test_helpers = True - nettest['test-helpers'] += map(lambda x: x[1], - net_test_loader.missingTestHelpers) - - required_nettests.append(nettest) - - if not requires_test_helpers and not requires_collector: - defer.returnValue(None) - - response = yield self.bouncer.lookupTestCollector(required_nettests) - try: - provided_net_tests = yield self.getReachableTestHelpersAndCollectors(response['net-tests']) - except e.NoReachableCollectors: - log.err("Could not find any reachable collector") - raise - except e.NoReachableTestHelpers: - log.err("Could not find any reachable test helpers") - raise - - def find_collector_and_test_helpers(test_name, test_version, input_files): - input_files = [u""+x['hash'] for x in input_files] - for net_test in provided_net_tests: - if net_test['name'] != test_name: - continue - if net_test['version'] != test_version: - continue - if set(net_test['input-hashes']) != set(input_files): - continue - return net_test['collector'], net_test['test-helpers'] - - for net_test_loader in self.netTestLoaders: - log.msg("Setting collector and test helpers for %s" % - net_test_loader.testName) - - collector, test_helpers = \ - find_collector_and_test_helpers(test_name=net_test_loader.testName, - test_version=net_test_loader.testVersion, - input_files=net_test_loader.inputFiles) - - for option, name in net_test_loader.missingTestHelpers: - test_helper_address_or_settings = test_helpers[name] - net_test_loader.localOptions[option] = test_helper_address_or_settings - net_test_loader.testHelpers[option] = test_helper_address_or_settings - - if not net_test_loader.collector: - net_test_loader.collector = collector + yield lookup_collector_and_test_helpers(self.netTestLoaders, + self.bouncer, + self.preferred_backend, + self.no_collector) @defer.inlineCallbacks def fetchAndVerifyNetTestInput(self, net_test_loader): @@ -419,17 +274,197 @@ class Deck(InputFile): i['test_options'][i['key']] = input_file.cached_file +def lookup_collector_and_test_helpers(net_test_loaders, + bouncer, + preferred_backend, + no_collector=False): + required_nettests = [] + + requires_test_helpers = False + requires_collector = False + for net_test_loader in net_test_loaders: + nettest = { + 'name': net_test_loader.testName, + 'version': net_test_loader.testVersion, + 'test-helpers': [], + 'input-hashes': [x['hash'] for x in net_test_loader.inputFiles] + } + if not net_test_loader.collector and not no_collector: + requires_collector = True + + if len(net_test_loader.missingTestHelpers) > 0: + requires_test_helpers = True + nettest['test-helpers'] += map(lambda x: x[1], + net_test_loader.missingTestHelpers) + + required_nettests.append(nettest) + + if not requires_test_helpers and not requires_collector: + defer.returnValue(None) + + response = yield bouncer.lookupTestCollector(required_nettests) + try: + provided_net_tests = yield get_reachable_test_helpers_and_collectors( + response['net-tests'], preferred_backend) + except e.NoReachableCollectors: + log.err("Could not find any reachable collector") + raise + except e.NoReachableTestHelpers: + log.err("Could not find any reachable test helpers") + raise + + def find_collector_and_test_helpers(test_name, test_version, input_files): + input_files = [u""+x['hash'] for x in input_files] + for net_test in provided_net_tests: + if net_test['name'] != test_name: + continue + if net_test['version'] != test_version: + continue + if set(net_test['input-hashes']) != set(input_files): + continue + return net_test['collector'], net_test['test-helpers'] + + for net_test_loader in net_test_loaders: + log.msg("Setting collector and test helpers for %s" % + net_test_loader.testName) + + collector, test_helpers = \ + find_collector_and_test_helpers(test_name=net_test_loader.testName, + test_version=net_test_loader.testVersion, + input_files=net_test_loader.inputFiles) + + for option, name in net_test_loader.missingTestHelpers: + test_helper_address_or_settings = test_helpers[name] + net_test_loader.localOptions[option] = test_helper_address_or_settings + net_test_loader.testHelpers[option] = test_helper_address_or_settings + + if not net_test_loader.collector: + net_test_loader.collector = collector + + +(a)defer.inlineCallbacks +def get_reachable_test_helpers_and_collectors(net_tests, preferred_backend): + for net_test in net_tests: + primary_address = net_test['collector'] + alternate_addresses = net_test.get('collector-alternate', []) + net_test['collector'] = yield get_reachable_collector( + primary_address, alternate_addresses, preferred_backend) + + for test_helper_name, test_helper_address in net_test['test-helpers'].items(): + test_helper_alternate = \ + net_test.get('test-helpers-alternate', {}).get(test_helper_name, []) + net_test['test-helpers'][test_helper_name] = \ + yield get_reachable_test_helper(test_helper_name, + test_helper_address, + test_helper_alternate, + preferred_backend) + + defer.returnValue(net_tests) + +(a)defer.inlineCallbacks +def get_reachable_collector(collector_address, collector_alternate, + preferred_backend): + # We prefer onion collector to https collector to cloudfront + # collectors to plaintext collectors + for collector_settings in sort_addresses_by_priority( + collector_address, + collector_alternate, + preferred_backend): + collector = CollectorClient(settings=collector_settings) + if not collector.isSupported(): + log.err("Unsupported %s collector %s" % ( + collector_settings['type'], + collector_settings['address'])) + continue + reachable = yield collector.isReachable() + if not reachable: + log.err("Unreachable %s collector %s" % ( + collector_settings['type'], + collector_settings['address'])) + continue + defer.returnValue(collector) + + raise e.NoReachableCollectors + + +(a)defer.inlineCallbacks +def get_reachable_test_helper(test_helper_name, test_helper_address, + test_helper_alternate, preferred_backend): + # For the moment we look for alternate addresses only of + # web_connectivity test helpers. + if test_helper_name == 'web-connectivity': + for web_connectivity_settings in sort_addresses_by_priority( + test_helper_address, test_helper_alternate, + preferred_backend): + web_connectivity_test_helper = WebConnectivityClient( + settings=web_connectivity_settings) + if not web_connectivity_test_helper.isSupported(): + log.err("Unsupported %s web_connectivity test_helper " + "%s" % ( + web_connectivity_settings['type'], + web_connectivity_settings['address'] + )) + continue + reachable = yield web_connectivity_test_helper.isReachable() + if not reachable: + log.err("Unreachable %s web_connectivity test helper %s" % ( + web_connectivity_settings['type'], + web_connectivity_settings['address'] + )) + continue + defer.returnValue(web_connectivity_settings) + raise e.NoReachableTestHelpers + else: + defer.returnValue(test_helper_address.encode('ascii')) + +def sort_addresses_by_priority(priority_address, + alternate_addresses, + preferred_backend): + prioritised_addresses = [] + + backend_type = guess_backend_type(priority_address) + priority_address = { + 'address': priority_address, + 'type': backend_type + } + address_priority = ['onion', 'https', 'cloudfront', 'http'] + address_priority.remove(preferred_backend) + address_priority.insert(0, preferred_backend) + + def filter_by_type(collectors, collector_type): + return filter(lambda x: x['type'] == collector_type, collectors) + + if (priority_address['type'] != preferred_backend): + valid_alternatives = filter_by_type(alternate_addresses, + preferred_backend) + if len(valid_alternatives) > 0: + alternate_addresses += [priority_address] + priority_address = valid_alternatives[0] + alternate_addresses.remove(priority_address) + + prioritised_addresses += [priority_address] + for address_type in address_priority: + prioritised_addresses += filter_by_type(alternate_addresses, + address_type) + + return prioritised_addresses + + +class InputNotFound(Exception): + pass + + class InputStore(object): def __init__(self): self.path = FilePath(config.inputs_directory) self.resources = FilePath(config.resources_directory) + self._cache_stale = True + self._cache = {} @defer.inlineCallbacks def update_url_lists(self, country_code): countries = ["global"] - if country_code == "ZZ": - country_code = None - else: + if country_code != "ZZ": countries.append(country_code) for cc in countries: @@ -466,30 +501,58 @@ class InputStore(object): "name": name, "filepath": out_file.path, "last_updated": timestampNowISO8601UTC(), - "id": "citizenlab_test_lists_{0}_txt".format(cc), + "id": "citizenlab_{0}_urls".format(cc), "type": "file/url" }, out_fh) @defer.inlineCallbacks def create(self, country_code=None): + # XXX This is a hax to avoid race conditions in testing because this + # object is a singleton and config can have a custom home directory + # passed at runtime. + self.path = FilePath(config.inputs_directory) + self.resources = FilePath(config.resources_directory) + self.path.child("descriptors").makedirs(ignoreExistingDirectory=True) self.path.child("data").makedirs(ignoreExistingDirectory=True) yield self.update_url_lists(country_code) @defer.inlineCallbacks def update(self, country_code=None): - yield self.update_url_lists(country_code) + # XXX why do we make a difference between create and update? + yield self.create(country_code) - def list(self): - inputs = [] + def _update_cache(self): descs = self.path.child("descriptors") if not descs.exists(): - return inputs + self._cache = {} + return for fn in descs.listdir(): with descs.child(fn).open("r") as in_fh: - inputs.append(json.load(in_fh)) - return inputs + input_desc = json.load(in_fh) + self._cache[input_desc.pop("id")] = input_desc + self._cache_stale = False + return + + def list(self): + if self._cache_stale: + self._update_cache() + return self._cache + + def get(self, input_id): + if self._cache_stale: + self._update_cache() + try: + input_desc = self._cache[input_id] + except KeyError: + raise InputNotFound(input_id) + return input_desc + + def getContent(self, input_id): + input_desc = self.get(input_id) + with open(input_desc["filepath"]) as fh: + return fh.read() class DeckStore(object): def __init__(self): @@ -501,10 +564,268 @@ class DeckStore(object): def get(self): pass -class NGInput(object): - def __init__(self, input_name): - pass +def resolve_file_path(v, prepath=None): + if v.startswith("$"): + # This raises InputNotFound and we let it carry onto the caller + return input_store.get(v[1:])["filepath"] + elif prepath is not None and (not os.path.isabs(v)): + return FilePath(prepath).preauthChild(v).path + return v + +def options_to_args(options, prepath=None): + args = [] + for k, v in options.items(): + if v is None: + continue + if k == "file": + v = resolve_file_path(v, prepath) + args.append('--'+k) + args.append(v) + return args + +class UnknownTaskKey(Exception): + pass + +class MissingTaskDataKey(Exception): + pass + +class DeckTask(object): + _metadata_keys = ["name"] + _supported_tasks = ["ooni"] + + def __init__(self, data, parent_metadata={}, cwd=None): + self.parent_metadata = parent_metadata + self.cwd = cwd + self.data = deepcopy(data) + + self.id = "" + + self.type = None + self.metadata = {} + self.requires_tor = False + self.requires_bouncer = False + + self.ooni = { + 'bouncer_client': None, + 'test_details': {} + } + + self._load(data) + + def _load_ooni(self, task_data): + required_keys = ["test_name"] + for required_key in required_keys: + if required_key not in task_data: + raise MissingTaskDataKey(required_key) + + # This raises e.NetTestNotFound, we let it go onto the caller + nettest_path = nettest_to_path(task_data.pop("test_name")) + + try: + annotations = task_data.pop('annotations') + except KeyError: + annotations = self.parent_metadata.get('annotations', {}) + + try: + collector_address = task_data.pop('collector') + except KeyError: + collector_address = self.parent_metadata.get('collector', None) + + net_test_loader = NetTestLoader( + options_to_args(task_data), + annotations=annotations, + test_file=nettest_path + ) + + if isinstance(collector_address, dict): + net_test_loader.collector = CollectorClient( + settings=collector_address + ) + elif collector_address is not None: + net_test_loader.collector = CollectorClient( + collector_address + ) + + if (net_test_loader.collector is not None and + net_test_loader.collector.backend_type == "onion"): + self.requires_tor = True + + try: + net_test_loader.checkOptions() + if net_test_loader.requiresTor: + self.requires_tor = True + except e.MissingTestHelper: + self.requires_bouncer = True + + self.ooni['net_test_loader'] = net_test_loader + # Need to ensure that this is called only once we have looked up the + # probe IP address and have geoip data. + self.ooni['test_details'] = net_test_loader.getTestDetails() + self.id = generate_filename(self.ooni['test_details']) + + def _load(self, data): + for key in self._metadata_keys: + try: + self.metadata[key] = data.pop(key) + except KeyError: + continue + + task_type, task_data = data.popitem() + if task_type not in self._supported_tasks: + raise UnknownTaskKey(task_type) + self.type = task_type + getattr(self, "_load_"+task_type)(task_data) + + assert len(data) == 0 class NGDeck(object): - def __init__(self, deck_path): - pass + def __init__(self, deck_data=None, + deck_path=None, no_collector=False): + # Used to resolve relative paths inside of decks. + self.deck_directory = None + self.requires_tor = False + self.no_collector = no_collector + self.name = "" + self.description = "" + self.schedule = None + + self.metadata = {} + self.bouncer = None + + self._measurement_path = FilePath(config.measurements_directory) + self._tasks = [] + self.task_ids = [] + + if deck_path is not None: + self.open(deck_path) + elif deck_data is not None: + self.load(deck_data) + + def open(self, deck_path): + with open(deck_path) as fh: + deck_data = yaml.safe_load(fh) + self.load(deck_data) + + def write(self, fh): + """ + Writes a properly formatted deck to the supplied file handle. + :param fh: an open file handle + :return: + """ + deck_data = { + "name": self.name, + "description": self.description, + "tasks": [task.data for task in self._tasks] + } + if self.schedule is not None: + deck_data["schedule"] = self.schedule + for key, value in self.metadata.items(): + deck_data[key] = value + + fh.write("---\n") + yaml.safe_dump(deck_data, fh, default_flow_style=False) + + def load(self, deck_data): + self.name = deck_data.pop("name", "Un-named Deck") + self.description = deck_data.pop("description", "No description") + + bouncer_address = deck_data.pop("bouncer", None) + if bouncer_address is None: + self.bouncer = get_preferred_bouncer() + elif isinstance(bouncer_address, dict): + self.bouncer = BouncerClient(settings=bouncer_address) + else: + self.bouncer = BouncerClient(bouncer_address) + + self.schedule = deck_data.pop("schedule", None) + + tasks_data = deck_data.pop("tasks", []) + for key, metadata in deck_data.items(): + self.metadata[key] = metadata + + for task_data in tasks_data: + deck_task = DeckTask(task_data, self.metadata, self.deck_directory) + if deck_task.requires_tor: + self.requires_tor = True + if (deck_task.requires_bouncer and + self.bouncer.backend_type == "onion"): + self.requires_tor = True + self._tasks.append(deck_task) + self.task_ids.append(deck_task.id) + + @defer.inlineCallbacks + def query_bouncer(self): + preferred_backend = config.advanced.get( + "preferred_backend", "onion" + ) + log.msg("Looking up collector and test helpers with {0}".format( + self.bouncer.base_address) + ) + net_test_loaders = [] + for task in self._tasks: + if task.type == "ooni": + net_test_loaders.append(task.ooni["net_test_loader"]) + + yield lookup_collector_and_test_helpers( + net_test_loaders, + self.bouncer, + preferred_backend, + self.no_collector + ) + + def _measurement_completed(self, result, measurement_id): + log.msg("{0}".format(result)) + measurement_dir = self._measurement_path.child(measurement_id) + measurement_dir.child("measurements.njson.progress").moveTo( + measurement_dir.child("measurements.njson") + ) + generate_summary( + measurement_dir.child("measurements.njson").path, + measurement_dir.child("summary.json").path + ) + measurement_dir.child("running.pid").remove() + + def _measurement_failed(self, failure, measurement_id): + measurement_dir = self._measurement_path.child(measurement_id) + measurement_dir.child("running.pid").remove() + # XXX do we also want to delete measurements.njson.progress? + return failure + + def _run_ooni_task(self, task, director): + net_test_loader = task.ooni["net_test_loader"] + test_details = task.ooni["test_details"] + measurement_id = task.id + + measurement_dir = self._measurement_path.child(measurement_id) + measurement_dir.createDirectory() + + report_filename = measurement_dir.child("measurements.njson.progress").path + pid_file = measurement_dir.child("running.pid") + + with pid_file.open('w') as out_file: + out_file.write("{0}".format(os.getpid())) + + d = director.start_net_test_loader( + net_test_loader, + report_filename, + test_details=test_details + ) + d.addCallback(self._measurement_completed, measurement_id) + d.addErrback(self._measurement_failed, measurement_id) + return d + + @defer.inlineCallbacks + def run(self, director): + tasks = [] + preferred_backend = config.advanced.get("preferred_backend", "onion") + yield self.query_bouncer() + for task in self._tasks: + if task.requires_tor: + yield director.start_tor() + elif task.requires_bouncer and preferred_backend == "onion": + yield director.start_tor() + if task.type == "ooni": + tasks.append(self._run_ooni_task(task, director)) + defer.returnValue(tasks) + +input_store = InputStore() diff --git a/ooni/director.py b/ooni/director.py index d39a11b..793975e 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -1,19 +1,24 @@ import pwd import os +from twisted.internet import defer +from twisted.python.failure import Failure + from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report from ooni.utils import log, generate_filename from ooni.utils.net import randomFreePort from ooni.nettest import NetTest, getNetTestInformation from ooni.settings import config -from ooni import errors from ooni.nettest import normalizeTestName from ooni.deck import InputStore from ooni.utils.onion import start_tor, connect_to_control_port -from twisted.internet import defer +class DirectorEvent(object): + def __init__(self, type="update", message=""): + self.type = type + self.message = message class Director(object): @@ -86,8 +91,6 @@ class Director(object): self.failures = [] - self.torControlProtocol = None - # This deferred is fired once all the measurements and their reporting # tasks are completed. self.allTestsDone = defer.Deferred() @@ -95,6 +98,58 @@ class Director(object): self.input_store = InputStore() + self._reset_director_state() + self._reset_tor_state() + + self._subscribers = [] + + def subscribe(self, handler): + self._subscribers.append(handler) + + def unsubscribe(self, handler): + self._subscribers.remove(handler) + + def notify(self, event): + for handler in self._subscribers: + handler(event) + + def _reset_director_state(self): + self._director_state = 'not-running' + self._director_starting = defer.Deferred() + self._director_starting.addErrback(self._director_startup_failure) + self._director_starting.addCallback(self._director_startup_success) + + def _director_startup_failure(self, failure): + self._reset_director_state() + self.notify(DirectorEvent("error", + "Failed to start the director")) + return failure + + def _director_startup_success(self, result): + self._director_state = 'running' + self.notify(DirectorEvent("success", + "Successfully started the director")) + return result + + def _reset_tor_state(self): + # This can be either 'not-running', 'starting' or 'running' + self._tor_state = 'not-running' + self._tor_starting = defer.Deferred() + self._tor_starting.addErrback(self._tor_startup_failure) + self._tor_starting.addCallback(self._tor_startup_success) + + def _tor_startup_failure(self, failure): + self._reset_tor_state() + self.notify(DirectorEvent("error", + "Failed to start Tor")) + return failure + + def _tor_startup_success(self, result): + self._tor_state = 'running' + self.notify(DirectorEvent("success", + "Successfully started Tor")) + return result + def getNetTests(self): nettests = {} @@ -126,16 +181,11 @@ class Director(object): return nettests @defer.inlineCallbacks - def start(self, start_tor=False, check_incoherences=True): + def _start(self, start_tor, check_incoherences): self.netTests = self.getNetTests() if start_tor: - if check_incoherences: - yield config.check_tor() - if config.advanced.start_tor and config.tor_state is None: - yield self.startTor() - elif config.tor.control_port and config.tor_state is None: - yield connect_to_control_port() + yield self.start_tor(check_incoherences) if config.global_options.get('no-geoip'): aux = [False] @@ -146,8 +196,21 @@ class Director(object): log.msg("You should add annotations for the country, city and ASN") else: yield config.probe_ip.lookup() + self.notify(DirectorEvent("success", + "Looked up Probe IP")) yield self.input_store.create(config.probe_ip.geodata["countrycode"]) + self.notify(DirectorEvent("success", + "Created input store")) + + @defer.inlineCallbacks + def start(self, start_tor=False, check_incoherences=True): + self._director_state = 'starting' + try: + yield self._start(start_tor, check_incoherences) + self._director_starting.callback(self._director_state) + except Exception as exc: + self._director_starting.errback(Failure(exc)) @property def measurementSuccessRatio(self): @@ -217,26 +280,17 @@ class Director(object): measurement.result = failure return measurement - def reporterFailed(self, failure, net_test): - """ - This gets called every time a reporter is failing and has been removed - from the reporters of a NetTest. - Once a report has failed to be created that net_test will never use the - reporter again. - - XXX hook some logic here. - note: failure contains an extra attribute called failure.reporter - """ - pass - def netTestDone(self, net_test): + self.notify(DirectorEvent("success", + "Successfully ran net_test")) self.activeNetTests.remove(net_test) if len(self.activeNetTests) == 0: self.allTestsDone.callback(None) @defer.inlineCallbacks - def startNetTest(self, net_test_loader, report_filename, - collector_client=None, no_yamloo=False): + def start_net_test_loader(self, net_test_loader, report_filename, + collector_client=None, no_yamloo=False, + test_details=None): """ Create the Report for the NetTest and start the report NetTest. @@ -244,14 +298,15 @@ class Director(object): net_test_loader: an instance of :class:ooni.nettest.NetTestLoader """ - test_details = net_test_loader.getTestDetails() + if test_details is None: + test_details = net_test_loader.getTestDetails() test_cases = net_test_loader.getTestCases() if self.allTestsDone.called: self.allTestsDone = defer.Deferred() if config.privacy.includepcap or config.global_options.get('pcapfile', None): - self.startSniffing(test_details) + self.start_sniffing(test_details) report = Report(test_details, report_filename, self.reportEntryManager, collector_client, @@ -271,7 +326,7 @@ class Director(object): finally: self.netTestDone(net_test) - def startSniffing(self, test_details): + def start_sniffing(self, test_details): """ Start sniffing with Scapy. Exits if required privileges (root) are not available. """ @@ -303,57 +358,83 @@ class Director(object): log.msg("Starting packet capture to: %s" % filename_pcap) - def startTor(self): + @defer.inlineCallbacks + def start_tor(self, check_incoherences=False): """ Starts Tor Launches a Tor with :param: socks_port :param: control_port :param: tor_binary set in ooniprobe.conf """ - log.msg("Starting Tor...") - from txtorcon import TorConfig + if self._tor_state == 'running': + log.debug("Tor is already running") + defer.returnValue(self._tor_state) + elif self._tor_state == 'starting': + yield self._tor_starting + defer.returnValue(self._tor_state) - tor_config = TorConfig() - if config.tor.control_port is None: - config.tor.control_port = int(randomFreePort()) - if config.tor.socks_port is None: - config.tor.socks_port = int(randomFreePort()) - - tor_config.ControlPort = config.tor.control_port - tor_config.SocksPort = config.tor.socks_port - - if config.tor.data_dir: - data_dir = os.path.expanduser(config.tor.data_dir) - - if not os.path.exists(data_dir): - log.msg("%s does not exist. Creating it." % data_dir) - os.makedirs(data_dir) - tor_config.DataDirectory = data_dir - - if config.tor.bridges: - tor_config.UseBridges = 1 - if config.advanced.obfsproxy_binary: - tor_config.ClientTransportPlugin = ( - 'obfs2,obfs3 exec %s managed' % - config.advanced.obfsproxy_binary - ) - bridges = [] - with open(config.tor.bridges) as f: - for bridge in f: - if 'obfs' in bridge: - if config.advanced.obfsproxy_binary: + log.msg("Starting Tor...") + self._tor_state = 'starting' + if check_incoherences: + yield config.check_tor() + + if config.advanced.start_tor and config.tor_state is None: + tor_config = TorConfig() + if config.tor.control_port is None: + config.tor.control_port = int(randomFreePort()) + if config.tor.socks_port is None: + config.tor.socks_port = int(randomFreePort()) + + tor_config.ControlPort = config.tor.control_port + tor_config.SocksPort = config.tor.socks_port + + if config.tor.data_dir: + data_dir = os.path.expanduser(config.tor.data_dir) + + if not os.path.exists(data_dir): + log.msg("%s does not exist. Creating it." % data_dir) + os.makedirs(data_dir) + tor_config.DataDirectory = data_dir + + if config.tor.bridges: + tor_config.UseBridges = 1 + if config.advanced.obfsproxy_binary: + tor_config.ClientTransportPlugin = ( + 'obfs2,obfs3 exec %s managed' % + config.advanced.obfsproxy_binary + ) + bridges = [] + with open(config.tor.bridges) as f: + for bridge in f: + if 'obfs' in bridge: + if config.advanced.obfsproxy_binary: + bridges.append(bridge.strip()) + else: bridges.append(bridge.strip()) - else: - bridges.append(bridge.strip()) - tor_config.Bridge = bridges - - if config.tor.torrc: - for i in config.tor.torrc.keys(): - setattr(tor_config, i, config.tor.torrc[i]) - - if os.geteuid() == 0: - tor_config.User = pwd.getpwuid(os.geteuid()).pw_name - - tor_config.save() - log.debug("Setting control port as %s" % tor_config.ControlPort) - log.debug("Setting SOCKS port as %s" % tor_config.SocksPort) - return start_tor(tor_config) + tor_config.Bridge = bridges + + if config.tor.torrc: + for i in config.tor.torrc.keys(): + setattr(tor_config, i, config.tor.torrc[i]) + + if os.geteuid() == 0: + tor_config.User = pwd.getpwuid(os.geteuid()).pw_name + + tor_config.save() + log.debug("Setting control port as %s" % tor_config.ControlPort) + log.debug("Setting SOCKS port as %s" % tor_config.SocksPort) + try: + yield start_tor(tor_config) + log.err("Calling tor callback") + self._tor_starting.callback(self._tor_state) + log.err("called") + except Exception as exc: + log.err("Failed to start tor") + log.exc(exc) + self._tor_starting.errback(Failure(exc)) + + elif config.tor.control_port and config.tor_state is None: + try: + yield connect_to_control_port() + self._tor_starting.callback(self._tor_state) + except Exception as exc: + self._tor_starting.errback(Failure(exc)) diff --git a/ooni/geoip.py b/ooni/geoip.py index fa6d1ae..28e0e1e 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -136,11 +136,11 @@ class UbuntuGeoIP(HTTPGeoIPLookupper): probe_ip = m.group(1) return probe_ip -class TorProjectGeoIP(HTTPGeoIPLookupper): - url = "https://check.torproject.org/" +class DuckDuckGoGeoIP(HTTPGeoIPLookupper): + url = "https://duckduckgo.com/?q=ip&ia=answer" def parseResponse(self, response_body): - regexp = "Your IP address appears to be: <strong>((\d+\.)+(\d+))" + regexp = "Your IP address is (.*) in " probe_ip = re.search(regexp, response_body).group(1) return probe_ip @@ -151,7 +151,7 @@ class ProbeIP(object): def __init__(self): self.geoIPServices = { 'ubuntu': UbuntuGeoIP, - 'torproject': TorProjectGeoIP + 'duckduckgo': DuckDuckGoGeoIP } self.geodata = { 'asn': 'AS0', diff --git a/ooni/measurements.py b/ooni/measurements.py deleted file mode 100644 index 976b125..0000000 --- a/ooni/measurements.py +++ /dev/null @@ -1,43 +0,0 @@ -import json - -class GenerateResults(object): - supported_tests = [ - "web_connectivity" - ] - - def __init__(self, input_file): - self.input_file = input_file - - def process_web_connectivity(self, entry): - result = {} - result['anomaly'] = False - if entry['test_keys']['blocking'] is not False: - result['anomaly'] = True - result['url'] = entry['input'] - return result - - def output(self, output_file): - results = {} - with open(self.input_file) as in_file: - for idx, line in enumerate(in_file): - entry = json.loads(line.strip()) - if entry['test_name'] not in self.supported_tests: - raise Exception("Unsupported test") - result = getattr(self, 'process_'+entry['test_name'])(entry) - result['idx'] = idx - results['test_name'] = entry['test_name'] - results['country_code'] = entry['probe_cc'] - results['asn'] = entry['probe_asn'] - results['results'] = results.get('results', []) - results['results'].append(result) - - with open(output_file, "w") as fw: - json.dump(results, fw) - -if __name__ == "__main__": - import sys - if len(sys.argv) != 3: - print("Usage: {0} [input_file] [output_file]".format(sys.argv[0])) - sys.exit(1) - gr = GenerateResults(sys.argv[1]) - gr.output(sys.argv[2]) diff --git a/ooni/nettest.py b/ooni/nettest.py index d01cf7b..2a33a2f 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -254,7 +254,8 @@ class NetTestLoader(object): h = sha256() for l in f: h.update(l) - except: + except Exception as exc: + log.exception(exc) raise e.InvalidInputFile(filename) input_file['hash'] = h.hexdigest() self.inputFiles.append(input_file) diff --git a/ooni/results.py b/ooni/results.py new file mode 100644 index 0000000..21fe997 --- /dev/null +++ b/ooni/results.py @@ -0,0 +1,39 @@ +import json + +class Process(): + supported_tests = [ + "web_connectivity" + ] + @staticmethod + def web_connectivity(entry): + result = {} + result['anomaly'] = False + if entry['test_keys']['blocking'] is not False: + result['anomaly'] = True + result['url'] = entry['input'] + return result + +def generate_summary(input_file, output_file): + results = {} + with open(input_file) as in_file: + for idx, line in enumerate(in_file): + entry = json.loads(line.strip()) + result = {} + if entry['test_name'] in Process.supported_tests: + result = getattr(Process, entry['test_name'])(entry) + result['idx'] = idx + results['test_name'] = entry['test_name'] + results['country_code'] = entry['probe_cc'] + results['asn'] = entry['probe_asn'] + results['results'] = results.get('results', []) + results['results'].append(result) + + with open(output_file, "w") as fw: + json.dump(results, fw) + +if __name__ == "__main__": + import sys + if len(sys.argv) != 3: + print("Usage: {0} [input_file] [output_file]".format(sys.argv[0])) + sys.exit(1) + generate_summary(sys.argv[1], sys.argv[2]) diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py index 904aaba..40e3b5e 100644 --- a/ooni/tests/bases.py +++ b/ooni/tests/bases.py @@ -10,7 +10,7 @@ class ConfigTestCase(unittest.TestCase): def setUp(self): self.ooni_home_dir = os.path.abspath("ooni_home") self.config = config - self.config.initialize_ooni_home("ooni_home") + self.config.initialize_ooni_home(self.ooni_home_dir) super(ConfigTestCase, self).setUp() def skipTest(self, reason): diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py index ba87ec8..455f4e1 100644 --- a/ooni/tests/test_deck.py +++ b/ooni/tests/test_deck.py @@ -1,11 +1,15 @@ import os +from copy import deepcopy + from twisted.internet import defer from twisted.trial import unittest from hashlib import sha256 from ooni import errors -from ooni.deck import InputFile, Deck, nettest_to_path +from ooni.deck import input_store, lookup_collector_and_test_helpers +from ooni.nettest import NetTestLoader +from ooni.deck import InputFile, Deck, nettest_to_path, DeckTask, NGDeck from ooni.tests.bases import ConfigTestCase from ooni.tests.mocks import MockBouncerClient, MockCollectorClient @@ -182,7 +186,8 @@ class TestDeck(BaseTestCase, ConfigTestCase): self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1) - yield deck.lookupCollectorAndTestHelpers() + yield lookup_collector_and_test_helpers(deck.preferred_backend, + deck.netTestLoaders) self.assertEqual(deck.netTestLoaders[0].collector.settings['address'], 'httpo://thirteenchars123.onion') @@ -229,7 +234,8 @@ class TestDeck(BaseTestCase, ConfigTestCase): self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1) - yield deck.lookupCollectorAndTestHelpers() + yield lookup_collector_and_test_helpers(deck.preferred_backend, + deck.netTestLoaders) self.assertEqual( deck.netTestLoaders[0].collector.settings['address'], @@ -258,7 +264,8 @@ class TestDeck(BaseTestCase, ConfigTestCase): self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1) - yield deck.lookupCollectorAndTestHelpers() + yield lookup_collector_and_test_helpers(deck.preferred_backend, + deck.netTestLoaders) self.assertEqual( deck.netTestLoaders[0].collector.settings['address'], @@ -269,3 +276,45 @@ class TestDeck(BaseTestCase, ConfigTestCase): deck.netTestLoaders[0].localOptions['backend'], '127.0.0.1' ) + +class TestInputStore(ConfigTestCase): + @defer.inlineCallbacks + def test_update_input_store(self): + self.skipTest("antani") + yield input_store.update("ZZ") + print os.listdir(os.path.join( + self.config.resources_directory, "citizenlab-test-lists")) + print os.listdir(os.path.join(self.config.inputs_directory)) + +TASK_DATA = { + "name": "Some Task", + "ooni": { + "test_name": "web_connectivity", + "file": "$citizen_lab_global_urls" + } +} + +DECK_DATA = { + "name": "My deck", + "description": "Something", + "tasks": [ + deepcopy(TASK_DATA) + ] +} +class TestNGDeck(ConfigTestCase): + skip = True + def test_deck_task(self): + if self.skip: + self.skipTest("Skip is set to true") + yield input_store.update("ZZ") + deck_task = DeckTask(TASK_DATA) + self.assertIsInstance(deck_task.ooni["net_test_loader"], + NetTestLoader) + + @defer.inlineCallbacks + def test_deck_load(self): + if self.skip: + self.skipTest("Skip is set to true") + yield input_store.update("ZZ") + deck = NGDeck(deck_data=DECK_DATA) + self.assertEqual(len(deck.tasks), 1) diff --git a/ooni/tests/test_director.py b/ooni/tests/test_director.py index 18377f5..6638adb 100644 --- a/ooni/tests/test_director.py +++ b/ooni/tests/test_director.py @@ -73,7 +73,7 @@ class TestDirector(ConfigTestCase): @defer.inlineCallbacks def director_start_tor(): director = Director() - yield director.startTor() + yield director.start_tor() assert config.tor.socks_port == 4242 assert config.tor.control_port == 4242 @@ -93,7 +93,7 @@ class TestDirector(ConfigTestCase): net_test_loader.loadNetTestString(test_failing_twice) director = Director() director.netTestDone = net_test_done - director.startNetTest(net_test_loader, None, no_yamloo=True) + director.start_net_test_loader(net_test_loader, None, no_yamloo=True) return finished @@ -113,7 +113,7 @@ class TestStartSniffing(unittest.TestCase): def test_start_sniffing_once(self): with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory: with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: - self.director.startSniffing(self.testDetails) + self.director.start_sniffing(self.testDetails) sniffer = mock_scapy_sniffer.return_value mock_scapy_factory.registerProtocol.assert_called_once_with(sniffer) @@ -122,7 +122,7 @@ class TestStartSniffing(unittest.TestCase): with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: sniffer = mock_scapy_sniffer.return_value sniffer.pcapwriter.filename = 'foo1_filename' - self.director.startSniffing(self.testDetails) + self.director.start_sniffing(self.testDetails) self.assertEqual(len(self.director.sniffers), 1) self.testDetails = { @@ -132,13 +132,13 @@ class TestStartSniffing(unittest.TestCase): with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: sniffer = mock_scapy_sniffer.return_value sniffer.pcapwriter.filename = 'foo2_filename' - self.director.startSniffing(self.testDetails) + self.director.start_sniffing(self.testDetails) self.assertEqual(len(self.director.sniffers), 2) def test_measurement_succeeded(self): with patch('ooni.settings.config.scapyFactory') as mock_scapy_factory: with patch('ooni.utils.txscapy.ScapySniffer') as mock_scapy_sniffer: - self.director.startSniffing(self.testDetails) + self.director.start_sniffing(self.testDetails) self.assertEqual(len(self.director.sniffers), 1) measurement = MagicMock() measurement.testInstance = self.FooTestCase() diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py index 239080a..1592149 100644 --- a/ooni/tests/test_nettest.py +++ b/ooni/tests/test_nettest.py @@ -355,7 +355,7 @@ class TestNetTest(ConfigTestCase): director = Director() self.filename = 'dummy_report.yamloo' - d = director.startNetTest(ntl, self.filename) + d = director.start_net_test_loader(ntl, self.filename) @d.addCallback def complete(result): @@ -382,7 +382,7 @@ class TestNetTest(ConfigTestCase): director = Director() self.filename = 'dummy_report.yamloo' - d = director.startNetTest(ntl, self.filename) + d = director.start_net_test_loader(ntl, self.filename) @d.addCallback def complete(result): @@ -410,7 +410,7 @@ class TestNetTest(ConfigTestCase): director = Director() self.filename = 'dummy_report.yamloo' - d = director.startNetTest(ntl, self.filename) + d = director.start_net_test_loader(ntl, self.filename) @d.addCallback def complete(result): @@ -469,7 +469,7 @@ class TestNettestTimeout(ConfigTestCase): director = Director() self.filename = 'dummy_report.yamloo' - d = director.startNetTest(ntl, self.filename) + d = director.start_net_test_loader(ntl, self.filename) @d.addCallback def complete(result): diff --git a/ooni/tests/test_oonideckgen.py b/ooni/tests/test_oonideckgen.py index 4a52377..11a852f 100644 --- a/ooni/tests/test_oonideckgen.py +++ b/ooni/tests/test_oonideckgen.py @@ -1,10 +1,11 @@ import os -import yaml import tempfile +import yaml + +from ooni.scripts import oonideckgen from .bases import ConfigTestCase -from ooni.deckgen import cli class TestOONIDeckgen(ConfigTestCase): def setUp(self): @@ -25,7 +26,7 @@ class TestOONIDeckgen(ConfigTestCase): def test_generate_deck(self): temp_dir = tempfile.mkdtemp() - cli.generate_deck({ + oonideckgen.generate_deck({ "country-code": "it", "output": temp_dir, "collector": None, diff --git a/ooni/tests/test_oonireport.py b/ooni/tests/test_oonireport.py index 2275672..d71a403 100644 --- a/ooni/tests/test_oonireport.py +++ b/ooni/tests/test_oonireport.py @@ -5,8 +5,6 @@ from mock import patch, MagicMock from twisted.internet import defer from ooni.tests.bases import ConfigTestCase -from ooni.report import tool - mock_tor_check = MagicMock(return_value=True) class TestOONIReport(ConfigTestCase): @@ -51,9 +49,9 @@ class TestOONIReport(ConfigTestCase): cli.run(["upload"]) self.assertTrue(mock_tool.upload_all.called) - @patch('ooni.report.tool.CollectorClient') - @patch('ooni.report.tool.OONIBReportLog') - @patch('ooni.report.tool.OONIBReporter') + @patch('ooni.report.cli.CollectorClient') + @patch('ooni.report.cli.OONIBReportLog') + @patch('ooni.report.cli.OONIBReporter') def test_tool_upload(self, mock_oonib_reporter, mock_oonib_report_log, mock_collector_client): @@ -70,7 +68,7 @@ class TestOONIReport(ConfigTestCase): self._create_reporting_yaml(report_name) self._write_dummy_report(report_name) - d = tool.upload(report_name) + d = cli.upload(report_name) @d.addCallback def cb(result): mock_oonib_reporter_i.writeReportEntry.assert_called_with( @@ -78,9 +76,9 @@ class TestOONIReport(ConfigTestCase): ) return d - @patch('ooni.report.tool.CollectorClient') - @patch('ooni.report.tool.OONIBReportLog') - @patch('ooni.report.tool.OONIBReporter') + @patch('ooni.report.cli.CollectorClient') + @patch('ooni.report.cli.OONIBReportLog') + @patch('ooni.report.cli.OONIBReporter') def test_tool_upload_all(self, mock_oonib_reporter, mock_oonib_report_log, mock_collector_client): @@ -98,7 +96,7 @@ class TestOONIReport(ConfigTestCase): self._create_reporting_yaml(report_name) self._write_dummy_report(report_name) - d = tool.upload_all() + d = cli.upload_all() @d.addCallback def cb(result): mock_oonib_reporter_i.writeReportEntry.assert_called_with( diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py index 7a0036e..fe24bf6 100644 --- a/ooni/ui/cli.py +++ b/ooni/ui/cli.py @@ -334,10 +334,10 @@ def runTestWithDirector(director, global_options, url=None, start_tor=True): collector_client = setupCollector(global_options, net_test_loader.collector) - yield director.startNetTest(net_test_loader, - global_options['reportfile'], - collector_client, - global_options['no-yamloo']) + yield director.start_net_test_loader(net_test_loader, + global_options['reportfile'], + collector_client, + global_options['no-yamloo']) d.addCallback(setup_nettest) d.addCallback(post_director_start) diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index cc45067..e306ef2 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?27ae67e2c74ae4ae9a82"></script></body> + <script type="text/javascript" src="app.bundle.js?7ed7d7510803fa1a4ad8"></script></body> </html> diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index e63c08f..a862fe7 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -13,44 +13,58 @@ from werkzeug.exceptions import NotFound from ooni import __version__ as ooniprobe_version from ooni import errors -from ooni.deck import Deck +from ooni.deck import NGDeck from ooni.settings import config -from ooni.nettest import NetTestLoader -from ooni.measurements import GenerateResults -from ooni.utils import generate_filename +from ooni.utils import log +from ooni.director import DirectorEvent + +config.advanced.debug = True def rpath(*path): context = os.path.abspath(os.path.dirname(__file__)) return os.path.join(context, *path) -def getNetTestLoader(test_options, test_file): - """ - Args: - test_options: (dict) containing as keys the option names. - - test_file: (string) the path to the test_file to be run. - Returns: - an instance of :class:`ooni.nettest.NetTestLoader` with the specified - test_file and the specified options. - """ - options = [] - for k, v in test_options.items(): - if v is None: - print("Skipping %s because none" % k) - continue - options.append('--'+k) - options.append(v) - - net_test_loader = NetTestLoader(options, - test_file=test_file) - return net_test_loader - - class WebUIError(Exception): def __init__(self, code, message): self.code = code self.message = message +class LongPoller(object): + def __init__(self, timeout, _reactor=reactor): + self.lock = defer.DeferredLock() + + self.deferred_subscribers = [] + self._reactor = _reactor + self._timeout = timeout + + self.timer = task.LoopingCall( + self.notify, + DirectorEvent("null", "No updates"), + ) + self.timer.clock = self._reactor + + def start(self): + self.timer.start(self._timeout) + + def stop(self): + self.timer.stop() + + def _notify(self, lock, event): + for d in self.deferred_subscribers[:]: + assert not d.called, "Deferred is already called" + d.callback(event) + self.deferred_subscribers.remove(d) + self.timer.reset() + lock.release() + + def notify(self, event=None): + self.lock.acquire().addCallback(self._notify, event) + + def get(self): + d = defer.Deferred() + self.deferred_subscribers.append(d) + return d + class WebUIAPI(object): app = Klein() # Maximum number in seconds after which to return a result even if not @@ -58,7 +72,7 @@ class WebUIAPI(object): _long_polling_timeout = 5 _reactor = reactor - def __init__(self, config, director): + def __init__(self, config, director, _reactor=reactor): self.director = director self.config = config self.measurement_path = FilePath(config.measurements_directory) @@ -74,12 +88,26 @@ class WebUIAPI(object): "director_started": False, "failures": [] } - self.status_updates = [] - d = self.director.start(start_tor=True) + + self.status_poller = LongPoller( + self._long_polling_timeout, _reactor) + self.director_event_poller = LongPoller( + self._long_polling_timeout, _reactor) + + # XXX move this elsewhere + self.director_event_poller.start() + self.status_poller.start() + + self.director.subscribe(self.handle_director_event) + d = self.director.start() d.addCallback(self.director_started) d.addErrback(self.director_startup_failed) - d.addBoth(lambda _: self.broadcast_status_update()) + d.addBoth(lambda _: self.status_poller.notify()) + + def handle_director_event(self, event): + log.msg("Handling event {0}".format(event.type)) + self.director_event_poller.notify(event) def add_failure(self, failure): self.status['failures'].append(str(failure)) @@ -92,26 +120,12 @@ class WebUIAPI(object): def director_startup_failed(self, failure): self.add_failure(failure) - def broadcast_status_update(self): - for su in self.status_updates: - if not su.called: - su.callback(None) - def completed_measurement(self, measurement_id): del self.status['active_measurements'][measurement_id] self.status['completed_measurements'].append(measurement_id) - measurement_dir = self.measurement_path.child(measurement_id) - - measurement = measurement_dir.child('measurements.njson.progress') - - # Generate the summary.json file - summary = measurement_dir.child('summary.json') - gr = GenerateResults(measurement.path) - gr.output(summary.path) - - measurement.moveTo(measurement_dir.child('measurements.njson')) def failed_measurement(self, measurement_id, failure): + log.exception(failure) del self.status['active_measurements'][measurement_id] self.add_failure(str(failure)) @@ -119,8 +133,9 @@ class WebUIAPI(object): def not_found(self, request, _): request.redirect('/client/') - @app.handle_error(WebUIError) - def web_ui_error(self, request, error): + @app.handle_errors(WebUIError) + def web_ui_error(self, request, failure): + error = failure.value request.setResponseCode(error.code) return self.render_json({ "error_code": error.code, @@ -133,24 +148,28 @@ class WebUIAPI(object): request.setHeader('Content-Length', len(json_string)) return json_string + @app.route('/api/notify', methods=["GET"]) + def api_notify(self, request): + def got_director_event(event): + return self.render_json({ + "type": event.type, + "message": event.message + }, request) + d = self.director_event_poller.get() + d.addCallback(got_director_event) + return d + @app.route('/api/status', methods=["GET"]) def api_status(self, request): return self.render_json(self.status, request) @app.route('/api/status/update', methods=["GET"]) def api_status_update(self, request): - status_update = defer.Deferred() - status_update.addCallback(lambda _: - self.status_updates.remove(status_update)) - status_update.addCallback(lambda _: self.api_status(request)) - - self.status_updates.append(status_update) - - # After long_polling_timeout we fire the callback - task.deferLater(self._reactor, self._long_polling_timeout, - status_update.callback, None) - - return status_update + def got_status_update(event): + return self.api_status(request) + d = self.status_poller.get() + d.addCallback(got_status_update) + return d @app.route('/api/deck/generate', methods=["GET"]) def api_deck_generate(self, request): @@ -167,37 +186,23 @@ class WebUIAPI(object): return self.render_json({"command": "deck-list"}, request) - @defer.inlineCallbacks def run_deck(self, deck): - yield deck.setup() - measurement_ids = [] - for net_test_loader in deck.netTestLoaders: - # XXX synchronize this with startNetTest - test_details = net_test_loader.getTestDetails() - measurement_id = generate_filename(test_details) - - measurement_dir = self.measurement_path.child(measurement_id) - measurement_dir.createDirectory() - - report_filename = measurement_dir.child( - "measurements.njson.progress").path - - measurement_ids.append(measurement_id) - self.status['active_measurements'][measurement_id] = { - 'test_name': test_details['test_name'], - 'test_start_time': test_details['test_start_time'] + for task_id in deck.task_ids: + self.status['active_measurements'][task_id] = { + 'test_name': 'foobar', + 'test_start_time': 'some start time' } - self.broadcast_status_update() - d = self.director.startNetTest(net_test_loader, report_filename) - d.addCallback(lambda _: - self.completed_measurement(measurement_id)) - d.addErrback(lambda failure: - self.failed_measurement(measurement_id, failure)) + self.status_poller.notify() + d = deck.run(self.director) + d.addCallback(lambda _: + self.completed_measurement(task_id)) + d.addErrback(lambda failure: + self.failed_measurement(task_id, failure)) @app.route('/api/nettest/<string:test_name>/start', methods=["POST"]) def api_nettest_start(self, request, test_name): try: - net_test = self.director.netTests[test_name] + _ = self.director.netTests[test_name] except KeyError: raise WebUIError(500, 'Could not find the specified test') @@ -206,10 +211,15 @@ class WebUIAPI(object): except ValueError: raise WebUIError(500, 'Invalid JSON message recevied') - deck = Deck(no_collector=True) # XXX remove no_collector - net_test_loader = getNetTestLoader(test_options, net_test['path']) + test_options["test_name"] = test_name + deck_data = { + "tasks": [ + {"ooni": test_options} + ] + } try: - deck.insert(net_test_loader) + deck = NGDeck(no_collector=True) + deck.load(deck_data) self.run_deck(deck) except errors.MissingRequiredOption, option_name: @@ -237,6 +247,19 @@ class WebUIAPI(object): def api_input_list(self, request): return self.render_json(self.director.input_store.list(), request) + @app.route('/api/input/<string:input_id>/content', methods=["GET"]) + def api_input_content(self, request, input_id): + content = self.director.input_store.getContent(input_id) + request.setHeader('Content-Type', 'text/plain') + request.setHeader('Content-Length', len(content)) + return content + + @app.route('/api/input/<string:input_id>', methods=["GET"]) + def api_input_details(self, request, input_id): + return self.render_json( + self.director.input_store.get(input_id), request + ) + @app.route('/api/measurement', methods=["GET"]) def api_measurement_list(self, request): measurements = [] @@ -299,7 +322,3 @@ class WebUIAPI(object): def static(self, request): path = rpath("client") return static.File(path) -<<<<<<< acda284b56fa3a75acbe7d000fbdefb643839948 - -======= ->>>>>>> [Web UI] Refactoring of web UI diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py index 40ee3b4..eca75cb 100644 --- a/ooni/ui/web/web.py +++ b/ooni/ui/web/web.py @@ -1,53 +1,27 @@ -import os - -from twisted.scripts import twistd -from twisted.python import usage -from twisted.internet import reactor from twisted.web import server +from twisted.internet import reactor from twisted.application import service +from ooni.ui.web.server import WebUIAPI from ooni.settings import config -from ooni.director import Director -from ooni.utils import log - -from .server import WebUIAPI class WebUIService(service.MultiService): - portNum = 8822 + def __init__(self, director, port_number=8842): + service.MultiService.__init__(self) + + self.director = director + self.port_number = port_number + def startService(self): service.MultiService.startService(self) - config.set_paths() - config.initialize_ooni_home() - config.read_config_file() - director = Director() - web_ui_api = WebUIAPI(config, director) - root = server.Site(web_ui_api.app.resource()) - self._port = reactor.listenTCP(self.portNum, root) - d = director.start() + + web_ui_api = WebUIAPI(config, self.director) + self._port = reactor.listenTCP( + self.port_number, + server.Site(web_ui_api.app.resource()) + ) def stopService(self): + service.MultiService.stopService(self) if self._port: self._port.stopListening() - -class StartOoniprobeWebUIPlugin: - tapname = "ooniprobe" - def makeService(self, so): - return WebUIService() - -class OoniprobeTwistdConfig(twistd.ServerOptions): - subCommands = [("StartOoniprobeWebUI", None, usage.Options, "ooniprobe web ui")] - -def start(): - twistd_args = ["--nodaemon"] - twistd_config = OoniprobeTwistdConfig() - twistd_args.append("StartOoniprobeWebUI") - try: - twistd_config.parseOptions(twistd_args) - except usage.error, ue: - print("ooniprobe: usage error from twistd: {}\n".format(ue)) - twistd_config.loadedPlugins = {"StartOoniprobeWebUI": StartOoniprobeWebUIPlugin()} - twistd.runApp(twistd_config) - return 0 - -if __name__ == "__main__": - start() diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py index cc2f2ff..e18a6ee 100644 --- a/ooni/utils/onion.py +++ b/ooni/utils/onion.py @@ -237,20 +237,6 @@ class TorLauncherWithRetries(object): continue setattr(new_tor_config, key, getattr(self.tor_config, key)) self.tor_config = new_tor_config - self.timeout = timeout - - def _reset_tor_config(self): - """ - This is used to reset the Tor configuration to before launch_tor - modified it. This is in particular used to force the regeneration of the - DataDirectory. - """ - new_tor_config = TorConfig() - for key in self.tor_config: - if config.tor.data_dir is None and key == "DataDirectory": - continue - setattr(new_tor_config, key, getattr(self.tor_config, key)) - self.tor_config = new_tor_config def _progress_updates(self, prog, tag, summary): log.msg("%d%%: %s" % (prog, summary))

1 0

[ooni-probe/master] Write ooniprobe reports in JSON format on disk
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 1ec88b611d77048b8281d3358b20883388bd8283 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jul 14 21:30:01 2016 +0200 Write ooniprobe reports in JSON format on disk * Implement various API endpoints --- ooni/measurements.py | 19 +++---- ooni/reporter.py | 116 ++++++++++++++++++++++++++++------------ ooni/settings.py | 3 +- ooni/ui/web/server.py | 142 +++++++++++++++++++++++++++---------------------- ooni/ui/web/web.py | 7 ++- ooni/utils/__init__.py | 6 ++- 6 files changed, 179 insertions(+), 114 deletions(-) diff --git a/ooni/measurements.py b/ooni/measurements.py index 5244ea4..976b125 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -9,26 +9,27 @@ class GenerateResults(object): self.input_file = input_file def process_web_connectivity(self, entry): - anomaly = {} - anomaly['result'] = False + result = {} + result['anomaly'] = False if entry['test_keys']['blocking'] is not False: - anomaly['result'] = True - anomaly['url'] = entry['input'] - return anomaly + result['anomaly'] = True + result['url'] = entry['input'] + return result def output(self, output_file): results = {} with open(self.input_file) as in_file: - for line in in_file: + for idx, line in enumerate(in_file): entry = json.loads(line.strip()) if entry['test_name'] not in self.supported_tests: raise Exception("Unsupported test") - anomaly = getattr(self, 'process_'+entry['test_name'])(entry) + result = getattr(self, 'process_'+entry['test_name'])(entry) + result['idx'] = idx results['test_name'] = entry['test_name'] results['country_code'] = entry['probe_cc'] results['asn'] = entry['probe_asn'] - results['anomalies'] = results.get('anomalies', []) - results['anomalies'].append(anomaly) + results['results'] = results.get('results', []) + results['results'].append(result) with open(output_file, "w") as fw: json.dump(results, fw) diff --git a/ooni/reporter.py b/ooni/reporter.py index f76fada..f07b3cf 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -1,5 +1,6 @@ import uuid import yaml +import json import os from copy import deepcopy @@ -206,6 +207,56 @@ class YAMLReporter(OReporter): def finish(self): self._stream.close() +class NJSONReporter(OReporter): + + """ + report_destination: + the destination directory of the report + + """ + + def __init__(self, test_details, report_filename): + self.report_path = report_filename + OReporter.__init__(self, test_details) + + def _writeln(self, line): + self._write("%s\n" % line) + + def _write(self, data): + if not self._stream: + raise errors.ReportNotCreated + if self._stream.closed: + raise errors.ReportAlreadyClosed + s = str(data) + assert isinstance(s, type('')) + self._stream.write(s) + untilConcludes(self._stream.flush) + + def writeReportEntry(self, entry): + if isinstance(entry, Measurement): + e = deepcopy(entry.testInstance.report) + elif isinstance(entry, dict): + e = deepcopy(entry) + else: + raise Exception("Failed to serialise entry") + report_entry = { + 'input': e.pop('input', None), + 'id': str(uuid.uuid4()), + 'test_start_time': e.pop('test_start_time', None), + 'measurement_start_time': e.pop('measurement_start_time', None), + 'test_runtime': e.pop('test_runtime', None), + 'test_keys': e + } + report_entry.update(self.testDetails) + self._write(json.dumps(report_entry)) + self._write("\n") + + def createReport(self): + self._stream = open(self.report_path, 'w+') + + def finish(self): + self._stream.close() + class OONIBReporter(OReporter): @@ -219,25 +270,20 @@ class OONIBReporter(OReporter): def serializeEntry(self, entry, serialisation_format="yaml"): if serialisation_format == "json": if isinstance(entry, Measurement): - report_entry = { - 'input': entry.testInstance.report.pop('input', None), - 'id': str(uuid.uuid4()), - 'test_start_time': entry.testInstance.report.pop('test_start_time', None), - 'measurement_start_time': entry.testInstance.report.pop('measurement_start_time', None), - 'test_runtime': entry.testInstance.report.pop('test_runtime', None), - 'test_keys': entry.testInstance.report - } + e = deepcopy(entry.testInstance.report) + elif isinstance(entry, dict): - report_entry = { - 'input': entry.pop('input', None), - 'id': str(uuid.uuid4()), - 'test_start_time': entry.pop('test_start_time', None), - 'measurement_start_time': entry.pop('measurement_start_time', None), - 'test_runtime': entry.pop('test_runtime', None), - 'test_keys': entry - } + e = deepcopy(entry) else: raise Exception("Failed to serialise entry") + report_entry = { + 'input': e.pop('input', None), + 'id': str(uuid.uuid4()), + 'test_start_time': e.pop('test_start_time', None), + 'measurement_start_time': e.pop('measurement_start_time', None), + 'test_runtime': e.pop('test_runtime', None), + 'test_keys': e + } report_entry.update(self.testDetails) return report_entry else: @@ -468,7 +514,7 @@ class Report(object): def __init__(self, test_details, report_filename, reportEntryManager, collector_client=None, - no_yamloo=False): + no_njson=False): """ This is an abstraction layer on top of all the configured reporters. @@ -499,9 +545,9 @@ class Report(object): self.report_log = OONIBReportLog() - self.yaml_reporter = None + self.njson_reporter = None self.oonib_reporter = None - self.no_yamloo = no_yamloo + self.no_njson = no_njson self.done = defer.Deferred() self.reportEntryManager = reportEntryManager @@ -509,7 +555,7 @@ class Report(object): def generateReportFilename(self): report_filename = generate_filename(self.test_details, prefix='report', - extension='yamloo') + extension='njson') report_path = os.path.join('.', report_filename) return os.path.abspath(report_path) @@ -543,12 +589,12 @@ class Report(object): self.collector_client) self.test_details['report_id'] = yield self.open_oonib_reporter() - if not self.no_yamloo: - self.yaml_reporter = YAMLReporter(self.test_details, - self.report_filename) + if not self.no_njson: + self.njson_reporter = NJSONReporter(self.test_details, + self.report_filename) if not self.oonib_reporter: yield self.report_log.not_created(self.report_filename) - yield defer.maybeDeferred(self.yaml_reporter.createReport) + yield defer.maybeDeferred(self.njson_reporter.createReport) defer.returnValue(self.reportId) @@ -570,7 +616,7 @@ class Report(object): d = defer.Deferred() deferreds = [] - def yaml_report_failed(failure): + def njson_report_failed(failure): d.errback(failure) def oonib_report_failed(failure): @@ -580,11 +626,11 @@ class Report(object): if not d.called: d.callback(None) - if self.yaml_reporter: - write_yaml_report = ReportEntry(self.yaml_reporter, measurement) - self.reportEntryManager.schedule(write_yaml_report) - write_yaml_report.done.addErrback(yaml_report_failed) - deferreds.append(write_yaml_report.done) + if self.njson_reporter: + write_njson_report = ReportEntry(self.njson_reporter, measurement) + self.reportEntryManager.schedule(write_njson_report) + write_njson_report.done.addErrback(njson_report_failed) + deferreds.append(write_njson_report.done) if self.oonib_reporter: write_oonib_report = ReportEntry(self.oonib_reporter, measurement) @@ -609,7 +655,7 @@ class Report(object): d = defer.Deferred() deferreds = [] - def yaml_report_failed(failure): + def njson_report_failed(failure): d.errback(failure) def oonib_report_closed(result): @@ -623,10 +669,10 @@ class Report(object): if not d.called: d.callback(None) - if self.yaml_reporter: - close_yaml = defer.maybeDeferred(self.yaml_reporter.finish) - close_yaml.addErrback(yaml_report_failed) - deferreds.append(close_yaml) + if self.njson_reporter: + close_njson = defer.maybeDeferred(self.njson_reporter.finish) + close_njson.addErrback(njson_report_failed) + deferreds.append(close_njson) if self.oonib_reporter: close_oonib = self.oonib_reporter.finish() diff --git a/ooni/settings.py b/ooni/settings.py index ffbf68e..5245451 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -106,7 +106,8 @@ class OConfig(object): else: self.decks_directory = os.path.join(self.ooni_home, 'decks') - self.reports_directory = os.path.join(self.ooni_home, 'reports') + self.measurements_directory = os.path.join(self.ooni_home, + 'measurements') self.resources_directory = os.path.join(self.data_directory, "resources") if self.advanced.report_log_file: diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 5be581c..ccc5d87 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -3,34 +3,20 @@ from __future__ import print_function import os import json +from twisted.internet import defer from twisted.python import usage from twisted.python.filepath import FilePath, InsecurePath from twisted.web import static from klein import Klein +from werkzeug.exceptions import NotFound -from ooni.settings import config from ooni import errors +from ooni.deck import Deck +from ooni.settings import config from ooni.nettest import NetTestLoader from ooni.measurements import GenerateResults - -class RouteNotFound(Exception): - def __init__(self, path, method): - self._path = path - self._method = method - - def __repr__(self): - return "<RouteNotFound {0} {1}>".format(self._path, - self._method) - -def _resolvePath(request): - path = b'' - if request.postpath: - path = b'/'.join(request.postpath) - - if not path.startswith(b'/'): - path = b'/' + path - return path +from ooni.utils import generate_filename def rpath(*path): context = os.path.abspath(os.path.dirname(__file__)) @@ -48,6 +34,9 @@ def getNetTestLoader(test_options, test_file): """ options = [] for k, v in test_options.items(): + if v is None: + print("Skipping %s because none" % k) + continue options.append('--'+k) options.append(v) @@ -61,6 +50,11 @@ class WebUIAPI(object): def __init__(self, config, director): self.director = director self.config = config + self.active_measurements = {} + + @app.handle_errors(NotFound) + def not_found(self, request, _): + request.redirect('/client/') def render_json(self, obj, request): json_string = json.dumps(obj) + "\n" @@ -68,28 +62,43 @@ class WebUIAPI(object): request.setHeader('Content-Length', len(json_string)) return json_string - @app.route('/api/decks/generate', methods=["GET"]) - def generate_decks(self, request): + @app.route('/api/deck/generate', methods=["GET"]) + def api_deck_generate(self, request): return self.render_json({"generate": "deck"}, request) - @app.route('/api/decks/<string:deck_name>/start', methods=["POST"]) - def start_deck(self, request, deck_name): + @app.route('/api/deck/<string:deck_name>/start', methods=["POST"]) + def api_deck_start(self, request, deck_name): return self.render_json({"start": deck_name}, request) - @app.route('/api/decks/<string:deck_name>/stop', methods=["POST"]) - def stop_deck(self, request, deck_name): - return self.render_json({"stop": deck_name}, request) - - @app.route('/api/decks/<string:deck_name>', methods=["GET"]) - def deck_status(self, request, deck_name): - return self.render_json({"status": deck_name}, request) - - @app.route('/api/decks', methods=["GET"]) - def deck_list(self, request): + @app.route('/api/deck', methods=["GET"]) + def api_deck_list(self, request): return self.render_json({"command": "deck-list"}, request) - @app.route('/api/net-tests/<string:test_name>/start', methods=["POST"]) - def test_start(self, request, test_name): + @defer.inlineCallbacks + def run_deck(self, deck): + yield deck.setup() + measurement_ids = [] + for net_test_loader in deck.netTestLoaders: + # XXX synchronize this with startNetTest + test_details = net_test_loader.getTestDetails() + measurement_id = generate_filename(test_details) + + measurement_dir = os.path.join( + config.measurements_directory, + measurement_id + ) + os.mkdir(measurement_dir) + report_filename = os.path.join(measurement_dir, + "measurements.njson") + measurement_ids.append(measurement_id) + self.active_measurements[measurement_id] = { + 'test_name': test_details['test_name'], + 'test_start_time': test_details['test_start_time'] + } + self.director.startNetTest(net_test_loader, report_filename) + + @app.route('/api/nettest/<string:test_name>/start', methods=["POST"]) + def api_nettest_start(self, request, test_name): try: net_test = self.director.netTests[test_name] except KeyError: @@ -99,21 +108,19 @@ class WebUIAPI(object): 'error_message': 'Could not find the specified test' }, request) try: - test_options = json.load(request.content.read()) + test_options = json.load(request.content) except ValueError: return self.render_json({ 'error_code': 500, 'error_message': 'Invalid JSON message recevied' }, request) + deck = Deck(no_collector=True) # XXX remove no_collector net_test_loader = getNetTestLoader(test_options, net_test['path']) try: - net_test_loader.checkOptions() - # XXX we actually want to generate the report_filename in a smart - # way so that we can know where it is located and learn the results - # of the measurement. - report_filename = None - self.director.startNetTest(net_test_loader, report_filename) + deck.insert(net_test_loader) + self.run_deck(deck) + except errors.MissingRequiredOption, option_name: request.setResponseCode(500) return self.render_json({ @@ -134,25 +141,18 @@ class WebUIAPI(object): 'error_message': 'Insufficient priviledges' }, request) - return self.render_json({"deck": "list"}, request) - - @app.route('/api/net-tests/<string:test_name>/start', methods=["POST"]) - def test_stop(self, request, test_name): - return self.render_json({ - "command": "test-stop", - "test-name": test_name - }, request) - - @app.route('/api/net-tests/<string:test_name>', methods=["GET"]) - def test_status(self, request, test_name): - return self.render_json({"command": "test-stop"}, request) + return self.render_json({"status": "started"}, request) - @app.route('/api/net-tests', methods=["GET"]) - def test_list(self, request): + @app.route('/api/nettest', methods=["GET"]) + def api_nettest_list(self, request): return self.render_json(self.director.netTests, request) + @app.route('/api/status', methods=["GET"]) + def api_status(self): + return self.render_json() + @app.route('/api/measurement', methods=["GET"]) - def measurement_list(self, request): + def api_measurement_list(self, request): measurement_ids = os.listdir(os.path.join(config.ooni_home, "measurements")) measurements = [] @@ -169,8 +169,8 @@ class WebUIAPI(object): return self.render_json({"measurements": measurements}, request) @app.route('/api/measurement/<string:measurement_id>', methods=["GET"]) - def measurement_summary(self, request, measurement_id): - measurement_path = FilePath(config.ooni_home).child("measurements") + def api_measurement_summary(self, request, measurement_id): + measurement_path = FilePath(config.measurements_directory) try: measurement_dir = measurement_path.child(measurement_id) except InsecurePath: @@ -189,13 +189,29 @@ class WebUIAPI(object): @app.route('/api/measurement/<string:measurement_id>/<int:idx>', methods=["GET"]) - def measurement_open(self, request, measurement_id, idx): - return self.render_json({"command": "results"}, request) + def api_measurement_view(self, request, measurement_id, idx): + measurement_path = FilePath(config.measurements_directory) + try: + measurement_dir = measurement_path.child(measurement_id) + except InsecurePath: + return self.render_json({"error": "invalid measurement id"}) + measurements = measurement_dir.child("measurements.njson") + + # XXX maybe implement some caching here + with measurements.open("r") as f: + r = None + for f_idx, line in enumerate(f): + if f_idx == idx: + r = json.loads(line) + break + if r is None: + return self.render_json({"error": "Could not find measurement " + "with this idx"}, request) + return self.render_json(r, request) @app.route('/client/', branch=True) def static(self, request): - path = rpath("build") - print(path) + path = rpath("client") return static.File(path) <<<<<<< acda284b56fa3a75acbe7d000fbdefb643839948 diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py index f709c18..6c6971c 100644 --- a/ooni/ui/web/web.py +++ b/ooni/ui/web/web.py @@ -24,10 +24,9 @@ class WebUIService(service.MultiService): root = server.Site(WebUIAPI(config, director).app.resource()) self._port = reactor.listenTCP(self.portNum, root) director = Director() - #d = director.start() - #d.addCallback(_started) - #d.addErrback(self._startupFailed) - _started(None) + d = director.start() + d.addCallback(_started) + d.addErrback(self._startupFailed) def _startupFailed(self, err): log.err("Failed to start the director") diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index b28aadb..1aaac7b 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -97,18 +97,20 @@ def generate_filename(test_details, prefix=None, extension=None): extension. """ LONG_DATE = "%Y-%m-%d %H:%M:%S" - SHORT_DATE = "%Y-%m-%dT%H%M%SZ" + SHORT_DATE = "%Y%m%dT%H%M%SZ" kwargs = {} filename_format = "" if prefix is not None: kwargs["prefix"] = prefix filename_format += "{prefix}-" - filename_format += "{test_name}-{timestamp}" + filename_format += "{timestamp}-{probe_cc}-{probe_asn}-{test_name}" if extension is not None: kwargs["extension"] = extension filename_format += ".{extension}" kwargs['test_name'] = test_details['test_name'] + kwargs['probe_cc'] = test_details['probe_cc'] + kwargs['probe_asn'] = test_details['probe_asn'] kwargs['timestamp'] = datetime.strptime(test_details['test_start_time'], LONG_DATE).strftime(SHORT_DATE) return filename_format.format(**kwargs)

1 0

[ooni-probe/master] Fix setup.py bump version number
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 71c8f5bce08f5eaa4c5d17822a6b4345e5fd30bf Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jul 26 12:25:28 2016 +0200 Fix setup.py bump version number --- ooni/__init__.py | 13 +++++-------- setup.py | 58 +++++--------------------------------------------------- 2 files changed, 10 insertions(+), 61 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 9d99323..c5c3c0a 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,17 +1,14 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.6.2.dev0" -# This is the version number of resources to be downloaded -# when a release is made it should be aligned to __version__ -__resources_version__ = "1.6.0" +__version__ = "2.0.0.dev1" __all__ = [ + 'agent', 'common', - 'deckgen', - 'nettest', - 'report', - 'resources', + 'nettests', + 'scripts', 'templates', + 'ui', 'utils' ] diff --git a/setup.py b/setup.py index d401a65..c1ae189 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,6 @@ from __future__ import print_function import os import sys -import glob import shutil import tempfile import subprocess @@ -102,7 +101,6 @@ from setuptools.command.install import install from distutils.spawn import find_executable from ooni import __version__, __author__ -from ooni.scripts import ooniresources GEOIP_ASN_URL = "https://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz" GEOIP_URL = "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.da…" @@ -302,57 +300,13 @@ class CreateOoniResources(Command): shutil.rmtree(tmp_dir, ignore_errors=True) print("Written ooniresources to {0}".format(dst_path)) - - -class GenerateComponent(Command): - description = ("Generate a new component for the web ui") - user_options = [] - - def initialize_options(self): - pass - def finalize_options(self): - pass - def run(self): - from jinja2 import Template - context = os.path.abspath(os.path.dirname(__file__)) - component_dir = os.path.join(context, "ooni", "ui", "web", "client", - "app", "components") - component_name = raw_input("Enter component name: ") - lower_name = component_name.lower().replace(" ", "-") - upper_name = lower_name[0].upper() + lower_name[1:] - dst_dir = os.path.join(component_dir, lower_name) - os.mkdir(dst_dir) - for template_file in glob.glob("data/component-template/*"): - target_filename = os.path.basename(template_file).replace("templ", - lower_name) - target_path = os.path.join(dst_dir, target_filename) - template = Template(open(template_file).read()) - with open(target_path, "w+") as fw: - fw.write(template.render(name=lower_name, - nameUpper=upper_name)) - fw.write("\n") - print("Written %s" % target_path) - print("Component \"%s\" created!" % component_name) - print("You should now edit " - "\"ooni/ui/web/client/app/components/components.js\" " - "to require the newly create component like so:") - print(""" -var {upper_name} = require("{lower_name}/{lower_name}"); -var componentsModule = angular.module("app.components", [ - // In here are the other components - {upper_name} -]).name; -""".format(lower_name=lower_name, upper_name=upper_name)) - install_requires = [] dependency_links = [] data_files = [] packages = [ 'ooni', - 'ooni.api', + 'ooni.agent', 'ooni.common', - 'ooni.deckgen', - 'ooni.deckgen.processors', 'ooni.kit', 'ooni.nettests', 'ooni.nettests.manipulation', @@ -360,10 +314,11 @@ packages = [ 'ooni.nettests.scanning', 'ooni.nettests.blocking', 'ooni.nettests.third_party', - 'ooni.report', - 'ooni.resources', + 'ooni.scripts', 'ooni.templates', 'ooni.tests', + 'ooni.ui', + 'ooni.ui.web', 'ooni.utils' ] @@ -390,8 +345,6 @@ setup( data_files=data_files, packages=packages, include_package_data=True, - scripts=["bin/oonideckgen", "bin/ooniprobe", - "bin/oonireport", "bin/ooniresources"], dependency_links=dependency_links, install_requires=install_requires, zip_safe=False, @@ -407,8 +360,7 @@ setup( }, cmdclass={ "install": OoniInstall, - "create_ooniresources": CreateOoniResources, - "generate_component": GenerateComponent + "create_ooniresources": CreateOoniResources }, classifiers=( "Development Status :: 5 - Production/Stable",

1 0

[ooni-probe/master] Delete the app component template
by art＠torproject.org 19 Sep '16

19 Sep '16

commit d8e04435e4f154466fa9354ce215fff2c7af77d9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Jul 18 19:19:24 2016 +0200 Delete the app component template --- data/component-template/templ.component.js | 10 ---------- data/component-template/templ.controller.js | 5 ----- data/component-template/templ.css | 0 data/component-template/templ.html | 8 -------- data/component-template/templ.js | 19 ------------------- 5 files changed, 42 deletions(-) diff --git a/data/component-template/templ.component.js b/data/component-template/templ.component.js deleted file mode 100644 index 10c3c88..0000000 --- a/data/component-template/templ.component.js +++ /dev/null @@ -1,10 +0,0 @@ -var template = require("./{{name}}.html"); -var controller = require("./{{name}}.controller"); -require("./{{name}}.css"); - -module.exports = { - restrict: 'E', - bindings: {}, - template: template, - controller: controller -}; diff --git a/data/component-template/templ.controller.js b/data/component-template/templ.controller.js deleted file mode 100644 index aaf9110..0000000 --- a/data/component-template/templ.controller.js +++ /dev/null @@ -1,5 +0,0 @@ -function {{nameUpper}}Controller() { - this.example = "bar"; -} - -module.exports = {{nameUpper}}Controller; diff --git a/data/component-template/templ.css b/data/component-template/templ.css deleted file mode 100644 index e69de29..0000000 diff --git a/data/component-template/templ.html b/data/component-template/templ.html deleted file mode 100644 index 6b047b9..0000000 --- a/data/component-template/templ.html +++ /dev/null @@ -1,8 +0,0 @@ -<navbar></navbar> -<header> -</header> -<main> - <div> - <h1>{{nameUpper}}</h1> - </div> -</main> diff --git a/data/component-template/templ.js b/data/component-template/templ.js deleted file mode 100644 index 75ee1cf..0000000 --- a/data/component-template/templ.js +++ /dev/null @@ -1,19 +0,0 @@ -var angular = require("angular"); -var uiRouter = require("angular-ui-router"); -var {{name}}Component = require("./{{name}}.component"); - -var {{name}}Module = angular.module("{{name}}", [ - uiRouter -]) -.config(function($stateProvider, $urlRouterProvider){ - - $stateProvider.state('{{name}}', { - url: '/{{name}}', - template: '<{{name}}></{{name}}>' - }); - -}) -.component("{{name}}", {{name}}Component) -.name; - -module.exports = {{name}}Module;

1 0

[ooni-probe/master] Add two basic unittests
by art＠torproject.org 19 Sep '16

19 Sep '16

commit ae5d78b838fd910d042bdddd65ff96fbc8a1103f Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Jul 26 12:39:40 2016 +0200 Add two basic unittests --- ooni/tests/test_resources.py | 41 +++++++++++++++++++++++++++++++++++++++++ ooni/tests/test_socks.py | 10 ++++++++++ 2 files changed, 51 insertions(+) diff --git a/ooni/tests/test_resources.py b/ooni/tests/test_resources.py new file mode 100644 index 0000000..92961ea --- /dev/null +++ b/ooni/tests/test_resources.py @@ -0,0 +1,41 @@ +from ooni.resources import get_out_of_date_resources, check_for_update +from ooni.tests.bases import ConfigTestCase + +SAMPLE_CURRENT_MANIFEST = { + "resources": [ + { + "version": 0, + "path": "some/file-to-update.txt" + }, + { + "version": 0, + "path": "some/file-stays-stame.txt" + }, + { + "version": 0, + "path": "some/file-to-delete.txt" + } + ] +} + +SAMPLE_NEW_MANIFEST = { + "resources": [ + { + "version": 1, + "path": "some/file-to-update.txt" + }, + { + "version": 0, + "path": "some/file-stays-stame.txt" + } + ] +} +class TestResourceUpdate(ConfigTestCase): + def test_check_for_updates(self): + return check_for_update() + + def test_resources_out_of_date(self): + paths_to_update, paths_to_delete = get_out_of_date_resources( + SAMPLE_CURRENT_MANIFEST, SAMPLE_NEW_MANIFEST) + self.assertEqual(paths_to_update[0]["path"], "some/file-to-update.txt") + self.assertEqual(paths_to_delete[0]["path"], "some/file-to-delete.txt") diff --git a/ooni/tests/test_socks.py b/ooni/tests/test_socks.py new file mode 100644 index 0000000..2d2cdd8 --- /dev/null +++ b/ooni/tests/test_socks.py @@ -0,0 +1,10 @@ +from twisted.trial import unittest + +from twisted.internet.endpoints import TCP4ClientEndpoint +from twisted.internet import reactor +from ooni.utils.socks import TrueHeadersSOCKS5Agent + +class TestSocks(unittest.TestCase): + def test_create_agent(self): + proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', 9050) + agent = TrueHeadersSOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)

1 0

[ooni-probe/master] Implement CSRF protection based on double-submit token.
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 84e7083fe0c0ad53286756395f07b7a9cbb10c18 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Jul 27 18:00:34 2016 +0200 Implement CSRF protection based on double-submit token. --- ooni/deck.py | 2 +- ooni/ui/web/server.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/ooni/deck.py b/ooni/deck.py index 7976e5e..6844fda 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -492,7 +492,7 @@ class InputStore(object): def list(self): if self._cache_stale: self._update_cache() - return self._cache + return deepcopy(self._cache) def get(self, input_id): if self._cache_stale: diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index f14f6b8..a6fd315 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -2,6 +2,9 @@ from __future__ import print_function import os import json +import string +from functools import wraps +from random import SystemRandom from twisted.internet import defer, task, reactor from twisted.python import usage @@ -26,11 +29,52 @@ def rpath(*path): context = os.path.abspath(os.path.dirname(__file__)) return os.path.join(context, *path) + class WebUIError(Exception): def __init__(self, code, message): self.code = code self.message = message + +def xsrf_protect(check=True): + """ + This is a decorator that implements double submit token CSRF protection. + + Basically we set a cookie and ensure that every request contains the + same value inside of the cookie and the request header. + + It's based on the assumption that an attacker cannot read the cookie + that is set by the server (since it would be violating the SOP) and hence + is not possible to make a browser trigger requests that contain the + cookie value inside of the requests it sends. + + If you wish to disable checking of the token set the value check to False. + This will still lead to the cookie being set. + + This decorator needs to be applied after the decorator that registers + the routes. + """ + def deco(f): + + @wraps(f) + def wrapper(instance, request, *a, **kw): + should_check = check and instance._enable_xsrf_protection + token_cookie = request.getCookie(u'XSRF-TOKEN') + token_header = request.getHeader(b"X-XSRF-TOKEN") + if (token_cookie != instance._xsrf_token and + instance._enable_xsrf_protection): + request.addCookie(u'XSRF-TOKEN', + instance._xsrf_token) + if should_check and token_cookie != token_header: + raise WebUIError(404, "Invalid XSRF token") + return f(instance, request, *a, **kw) + + return wrapper + + return deco + + + class LongPoller(object): def __init__(self, timeout, _reactor=reactor): self.lock = defer.DeferredLock() @@ -73,6 +117,7 @@ class WebUIAPI(object): # change happenned. _long_polling_timeout = 5 _reactor = reactor + _enable_xsrf_protection = True def __init__(self, config, director, _reactor=reactor): self.director = director @@ -80,6 +125,12 @@ class WebUIAPI(object): self.measurement_path = FilePath(config.measurements_directory) self.decks_path = FilePath(config.decks_directory) + # We use a double submit token to protect against XSRF + rng = SystemRandom() + token_space = string.letters+string.digits + self._xsrf_token = ''.join([rng.choice(token_space) + for _ in range(30)]) + self.status = { "software_version": ooniprobe_version, "software_name": "ooniprobe", @@ -113,10 +164,12 @@ class WebUIAPI(object): self.status["country_code"] = probe_ip.geodata['countrycode'] @app.handle_errors(NotFound) + @xsrf_protect(check=False) def not_found(self, request, _): request.redirect('/client/') @app.handle_errors(WebUIError) + @xsrf_protect(check=False) def web_ui_error(self, request, failure): error = failure.value request.setResponseCode(error.code) @@ -132,6 +185,7 @@ class WebUIAPI(object): return json_string @app.route('/api/notify', methods=["GET"]) + @xsrf_protect(check=False) def api_notify(self, request): def got_director_event(event): return self.render_json({ @@ -143,10 +197,12 @@ class WebUIAPI(object): return d @app.route('/api/status', methods=["GET"]) + @xsrf_protect(check=False) def api_status(self, request): return self.render_json(self.status, request) @app.route('/api/status/update', methods=["GET"]) + @xsrf_protect(check=False) def api_status_update(self, request): def got_status_update(event): return self.api_status(request) @@ -155,14 +211,17 @@ class WebUIAPI(object): return d @app.route('/api/deck/generate', methods=["GET"]) + @xsrf_protect(check=False) def api_deck_generate(self, request): return self.render_json({"generate": "deck"}, request) @app.route('/api/deck/<string:deck_name>/start', methods=["POST"]) + @xsrf_protect(check=True) def api_deck_start(self, request, deck_name): return self.render_json({"start": deck_name}, request) @app.route('/api/deck', methods=["GET"]) + @xsrf_protect(check=False) def api_deck_list(self, request): for deck_id in self.decks_path.listdir(): pass @@ -180,6 +239,7 @@ class WebUIAPI(object): "Failed to start deck")) @app.route('/api/nettest/<string:test_name>/start', methods=["POST"]) + @xsrf_protect(check=True) def api_nettest_start(self, request, test_name): try: _ = self.director.netTests[test_name] @@ -220,10 +280,12 @@ class WebUIAPI(object): return self.render_json({"status": "started"}, request) @app.route('/api/nettest', methods=["GET"]) + @xsrf_protect(check=False) def api_nettest_list(self, request): return self.render_json(self.director.netTests, request) @app.route('/api/input', methods=["GET"]) + @xsrf_protect(check=False) def api_input_list(self, request): input_store_list = self.director.input_store.list() for key, value in input_store_list.items(): @@ -231,6 +293,7 @@ class WebUIAPI(object): return self.render_json(input_store_list, request) @app.route('/api/input/<string:input_id>/content', methods=["GET"]) + @xsrf_protect(check=False) def api_input_content(self, request, input_id): content = self.director.input_store.getContent(input_id) request.setHeader('Content-Type', 'text/plain') @@ -238,12 +301,14 @@ class WebUIAPI(object): return content @app.route('/api/input/<string:input_id>', methods=["GET"]) + @xsrf_protect(check=False) def api_input_details(self, request, input_id): return self.render_json( self.director.input_store.get(input_id), request ) @app.route('/api/measurement', methods=["GET"]) + @xsrf_protect(check=False) def api_measurement_list(self, request): measurements = [] for measurement_id in self.measurement_path.listdir(): @@ -264,6 +329,7 @@ class WebUIAPI(object): return self.render_json({"measurements": measurements}, request) @app.route('/api/measurement/<string:measurement_id>', methods=["GET"]) + @xsrf_protect(check=False) def api_measurement_summary(self, request, measurement_id): try: measurement_dir = self.measurement_path.child(measurement_id) @@ -289,6 +355,7 @@ class WebUIAPI(object): @app.route('/api/measurement/<string:measurement_id>/<int:idx>', methods=["GET"]) + @xsrf_protect(check=False) def api_measurement_view(self, request, measurement_id, idx): try: measurement_dir = self.measurement_path.child(measurement_id) @@ -310,6 +377,7 @@ class WebUIAPI(object): return self.render_json(r, request) @app.route('/client/', branch=True) + @xsrf_protect(check=False) def static(self, request): path = rpath("client") return static.File(path)

1 0

[ooni-probe/master] Add support for listing enabled and disabled decks
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 7c35e65485418d1dcb682bad1cf4eb62ef318e81 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Jul 30 16:27:20 2016 +0200 Add support for listing enabled and disabled decks * Fix various bugs --- ooni/agent/scheduler.py | 2 +- ooni/deck/store.py | 40 +++++++++++++++++---- ooni/director.py | 5 +-- ooni/measurements.py | 11 ++++-- ooni/settings.py | 9 +++-- ooni/ui/web/client/index.html | 2 +- ooni/ui/web/server.py | 81 +++++++++++++++++++++++++++++++++---------- 7 files changed, 117 insertions(+), 33 deletions(-) diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py index 43715e8..7a77afb 100644 --- a/ooni/agent/scheduler.py +++ b/ooni/agent/scheduler.py @@ -131,7 +131,7 @@ class RunDecks(ScheduledTask): @defer.inlineCallbacks def task(self): - for deck_id, deck in deck_store.list(): + for deck_id, deck in deck_store.list_enabled(): yield deck.setup() yield deck.run(self.director) diff --git a/ooni/deck/store.py b/ooni/deck/store.py index 7c90204..695f97d 100644 --- a/ooni/deck/store.py +++ b/ooni/deck/store.py @@ -9,7 +9,6 @@ from ooni.deck.deck import NGDeck from ooni.otime import timestampNowISO8601UTC from ooni.resources import check_for_update from ooni.settings import config -from ooni.utils import log class InputNotFound(Exception): pass @@ -121,25 +120,54 @@ class InputStore(object): class DeckStore(object): def __init__(self): - self.path = FilePath(config.decks_directory) + self.enabled_directory = FilePath(config.decks_enabled_directory) + self.available_directory = FilePath(config.decks_available_directory) self._cache = {} self._cache_stale = True - def list(self): - decks = [] + def _list(self): if self._cache_stale: self._update_cache() for deck_id, deck in self._cache.iteritems(): + yield (deck_id, deck) + + def list(self): + decks = [] + for deck_id, deck in self._list(): decks.append((deck_id, deck)) return decks + def list_enabled(self): + decks = [] + for deck_id, deck in self._list(): + if self.is_enabled(deck_id): + continue + decks.append((deck_id, deck)) + return decks + + def is_enabled(self, deck_id): + return self.enabled_directory.child(deck_id + '.yaml').exists() + + def enable(self, deck_id): + deck_path = self.available_directory.child(deck_id + '.yaml') + if not deck_path.exists(): + raise DeckNotFound(deck_id) + deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml') + deck_enabled_path.linkTo(deck_path) + + def disable(self, deck_id): + deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml') + if not deck_enabled_path.exists(): + raise DeckNotFound(deck_id) + deck_enabled_path.remove() + def _update_cache(self): - for deck_path in self.path.listdir(): + for deck_path in self.available_directory.listdir(): if not deck_path.endswith('.yaml'): continue deck_id = deck_path[:-1*len('.yaml')] deck = NGDeck( - deck_path=self.path.child(deck_path).path + deck_path=self.available_directory.child(deck_path).path ) self._cache[deck_id] = deck diff --git a/ooni/director.py b/ooni/director.py index 464a203..304a14a 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -9,7 +9,7 @@ from ooni.utils import log, generate_filename from ooni.nettest import NetTest, getNetTestInformation from ooni.settings import config from ooni.nettest import normalizeTestName -from ooni.deck.store import InputStore +from ooni.deck.store import input_store, deck_store from ooni.geoip import probe_ip from ooni.agent.scheduler import run_system_tasks @@ -96,7 +96,8 @@ class Director(object): self.allTestsDone = defer.Deferred() self.sniffers = {} - self.input_store = InputStore() + self.input_store = input_store + self.deck_store = deck_store self._reset_director_state() self._reset_tor_state() diff --git a/ooni/measurements.py b/ooni/measurements.py index 50efd87..cdf6b14 100644 --- a/ooni/measurements.py +++ b/ooni/measurements.py @@ -5,6 +5,9 @@ import signal from twisted.python.filepath import FilePath from ooni.settings import config +class MeasurementInProgress(Exception): + pass + class Process(): supported_tests = [ "web_connectivity", @@ -66,7 +69,7 @@ def get_measurement(measurement_id): running = False completed = True keep = False - if measurement.child("measurement.njson.progress").exists(): + if measurement.child("measurements.njson.progress").exists(): completed = False # XXX this is done quite often around the code, probably should # be moved into some utility function. @@ -97,10 +100,14 @@ def get_measurement(measurement_id): def get_summary(measurement_id): measurement_path = FilePath(config.measurements_directory) measurement = measurement_path.child(measurement_id) + + if measurement.child("measurements.njson.progress").exists(): + raise MeasurementInProgress + summary = measurement.child("summary.json") if not summary.exists(): generate_summary( - measurement.child("measurement.njson").path, + measurement.child("measurements.njson").path, summary.path ) diff --git a/ooni/settings.py b/ooni/settings.py index 3d33601..2161560 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -135,9 +135,13 @@ class OConfig(object): self.inputs_directory = os.path.join(self.running_path, 'inputs') self.scheduler_directory = os.path.join(self.running_path, 'scheduler') - self.decks_directory = os.path.join(self.running_path, 'decks') self.resources_directory = os.path.join(self.running_path, 'resources') + self.decks_available_directory = os.path.join(self.running_path, + 'decks-available') + self.decks_enabled_directory = os.path.join(self.running_path, + 'decks-enabled') + self.measurements_directory = os.path.join(self.running_path, 'measurements') @@ -166,7 +170,8 @@ class OConfig(object): # also ensure the subdirectories exist sub_directories = [ self.inputs_directory, - self.decks_directory, + self.decks_enabled_directory, + self.decks_available_directory, self.scheduler_directory, self.measurements_directory, self.resources_directory diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html index ebed106..e363ba0 100644 --- a/ooni/ui/web/client/index.html +++ b/ooni/ui/web/client/index.html @@ -13,5 +13,5 @@ <app> Loading... </app> - <script type="text/javascript" src="app.bundle.js?2777836bc218e75c3be5"></script></body> + <script type="text/javascript" src="app.bundle.js?9d3ccb3bc67af5ed4453"></script></body> </html> diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py index 74cb235..50db3ad 100644 --- a/ooni/ui/web/server.py +++ b/ooni/ui/web/server.py @@ -1,6 +1,5 @@ from __future__ import print_function -import os import json import string from functools import wraps @@ -17,11 +16,12 @@ from werkzeug.exceptions import NotFound from ooni import __version__ as ooniprobe_version from ooni import errors from ooni.deck import NGDeck +from ooni.deck.store import DeckNotFound, InputNotFound from ooni.settings import config from ooni.utils import log from ooni.director import DirectorEvent -from ooni.measurements import get_summary, get_measurement -from ooni.measurements import list_measurements, MeasurementNotFound +from ooni.measurements import get_summary, get_measurement, list_measurements +from ooni.measurements import MeasurementNotFound, MeasurementInProgress from ooni.geoip import probe_ip config.advanced.debug = True @@ -120,7 +120,6 @@ class WebUIAPI(object): self.director = director self.config = config self.measurement_path = FilePath(config.measurements_directory) - self.decks_path = FilePath(config.decks_directory) # We use a double submit token to protect against XSRF rng = SystemRandom() @@ -207,23 +206,61 @@ class WebUIAPI(object): d.addCallback(got_status_update) return d - @app.route('/api/deck/generate', methods=["GET"]) - @xsrf_protect(check=False) - def api_deck_generate(self, request): - return self.render_json({"generate": "deck"}, request) - - @app.route('/api/deck/<string:deck_name>/start', methods=["POST"]) + @app.route('/api/deck/<string:deck_id>/start', methods=["POST"]) @xsrf_protect(check=True) - def api_deck_start(self, request, deck_name): - return self.render_json({"start": deck_name}, request) + def api_deck_start(self, request, deck_id): + try: + deck = self.director.deck_store.get(deck_id) + except DeckNotFound: + raise WebUIError(404, "Deck not found") + + try: + self.run_deck(deck) + except: + raise WebUIError(500, "Failed to start deck") + + return self.render_json({"status": "started " + deck.name}, request) @app.route('/api/deck', methods=["GET"]) @xsrf_protect(check=False) def api_deck_list(self, request): - for deck_id in self.decks_path.listdir(): - pass + deck_list = { + 'available': {}, + 'enabled': {} + } + for deck_id, deck in self.director.deck_store.list(): + deck_list['available'][deck_id] = { + 'name': deck.name, + 'description': deck.description + } + + for deck_id, deck in self.director.deck_store.list_enabled(): + deck_list['enabled'][deck_id] = { + 'name': deck.name, + 'description': deck.description + } + + return self.render_json(deck_list, request) - return self.render_json({"command": "deck-list"}, request) + @app.route('/api/deck/<string:deck_id>/enable', methods=["POST"]) + @xsrf_protect(check=True) + def api_deck_enable(self, request, deck_id): + try: + self.director.deck_store.enable(deck_id) + except DeckNotFound: + raise WebUIError(404, "Deck not found") + + return self.render_json({"status": "enabled"}, request) + + @app.route('/api/deck/<string:deck_id>/disable', methods=["POST"]) + @xsrf_protect(check=True) + def api_deck_disable(self, request, deck_id): + try: + self.director.deck_store.disable(deck_id) + except DeckNotFound: + raise WebUIError(404, "Deck not found") + + return self.render_json({"status": "disabled"}, request) @defer.inlineCallbacks def run_deck(self, deck): @@ -261,17 +298,21 @@ class WebUIAPI(object): except errors.MissingRequiredOption, option_name: raise WebUIError( - 501, 'Missing required option: "{}"'.format(option_name) + 400, 'Missing required option: "{}"'.format(option_name) ) except usage.UsageError: raise WebUIError( - 502, 'Error in parsing options' + 400, 'Error in parsing options' ) except errors.InsufficientPrivileges: raise WebUIError( - 502, 'Insufficient priviledges' + 400, 'Insufficient priviledges' + ) + except: + raise WebUIError( + 500, 'Failed to start nettest' ) return self.render_json({"status": "started"}, request) @@ -319,6 +360,8 @@ class WebUIAPI(object): raise WebUIError(500, "invalid measurement id") except MeasurementNotFound: raise WebUIError(404, "measurement not found") + except MeasurementInProgress: + raise WebUIError(400, "measurement in progress") if measurement['completed'] is False: raise WebUIError(400, "measurement in progress") @@ -359,7 +402,7 @@ class WebUIAPI(object): with summary.open("w+") as f: pass - return self.render_json({"result": "ok"}, request) + return self.render_json({"status": "ok"}, request) @app.route('/api/measurement/<string:measurement_id>/<int:idx>', methods=["GET"])

1 0

[ooni-probe/master] Fix path to the web ui root
by art＠torproject.org 19 Sep '16

19 Sep '16

commit 9768924e8cd53c31f751fa0b605a6f6c502b23be Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Jul 29 23:54:19 2016 +0200 Fix path to the web ui root --- ooni/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/settings.py b/ooni/settings.py index 5762919..3d33601 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -131,7 +131,7 @@ class OConfig(object): def set_paths(self): self.nettest_directory = os.path.join(get_ooni_root(), 'nettests') - self.web_ui_directory = os.path.join(get_ooni_root(), 'web', 'client') + self.web_ui_directory = os.path.join(get_ooni_root(), 'ui', 'web', 'client') self.inputs_directory = os.path.join(self.running_path, 'inputs') self.scheduler_directory = os.path.join(self.running_path, 'scheduler')

1 0