commit 24ea012b508526165b1a7d1eef1d76475bd313e6 Author: Arturo Filastò <arturo@filasto.net> Date: Tue May 17 20:02:09 2016 +0200 Various improvement related to backend_client * Include web_connectivity in default generated deck * Fix serialisation of HTTP Headers * Support setting collector and bouncer in oonireport upload --- ooni/deck.py | 4 +++- ooni/deckgen/cli.py | 22 +++++++--------------- ooni/report/cli.py | 2 +- ooni/report/tool.py | 30 +++++++++++++++++++++--------- ooni/reporter.py | 3 ++- ooni/templates/httpt.py | 2 +- 6 files changed, 35 insertions(+), 28 deletions(-) diff --git a/ooni/deck.py b/ooni/deck.py index 007097e..404a4d0 100644 --- a/ooni/deck.py +++ b/ooni/deck.py @@ -136,10 +136,12 @@ class Deck(InputFile): net_test_loader = NetTestLoader(test['options']['subargs'], annotations=test['options'].get('annotations', {}), test_file=nettest_path) - if test['options']['collector']: + if test['options']['collector'] is not None: net_test_loader.collector = CollectorClient( test['options']['collector'] ) + if test['options']['bouncer'] is not None: + self.bouncer = test['options']['bouncer'] self.insert(net_test_loader) def insert(self, net_test_loader): diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py index 2e0e976..a6046f3 100644 --- a/ooni/deckgen/cli.py +++ b/ooni/deckgen/cli.py @@ -27,6 +27,7 @@ class Options(usage.Options): "generate the deck."], ["collector", None, None, "Specify a custom collector to use when " "submitting reports"], + ["bouncer", None, None, "Specify a custom bouncer to use"], ["output", "o", None, "Specify the directory where to write output."] ] @@ -44,7 +45,6 @@ class Deck(object): "annotations": None, "collector": None, - # XXX setting this is currently not supported "bouncer": None, "reportfile": None, @@ -56,13 +56,15 @@ class Deck(object): } } - def __init__(self, collector=None): + def __init__(self, collector=None, bouncer=None): self.deck_entries = [] self.collector = collector + self.bouncer = bouncer def add_test(self, test_file, subargs=[]): deck_entry = copy.deepcopy(self._base_entry) deck_entry['options']['collector'] = self.collector + deck_entry['options']['bouncer'] = self.bouncer deck_entry['options']['test_file'] = test_file deck_entry['options']['subargs'] = subargs self.deck_entries.append(deck_entry) @@ -89,24 +91,14 @@ def generate_deck(options): url_list_global = citizenlab_test_lists.generate_global_input( options['output'] ) - dns_servers = namebench_dns_servers.generate_country_input( - options['country-code'], - options['output'] - ) - deck = Deck(collector=options['collector']) + deck = Deck(collector=options['collector'], bouncer=options['bouncer']) deck.add_test('manipulation/http_invalid_request_line') deck.add_test('manipulation/http_header_field_manipulation') if url_list_country is not None: - deck.add_test('blocking/http_requests', ['-f', url_list_country]) - deck.add_test('blocking/http_requests', ['-f', url_list_global]) - - if url_list_country is not None: - deck.add_test('blocking/dns_consistency', - ['-f', url_list_country, '-T', dns_servers]) - deck.add_test('blocking/dns_consistency', - ['-f', url_list_global, '-T', dns_servers]) + deck.add_test('blocking/web_connectivity', ['-f', url_list_country]) + deck.add_test('blocking/web_connectivity', ['-f', url_list_global]) if config.advanced.debug: deck.pprint() diff --git a/ooni/report/cli.py b/ooni/report/cli.py index c1b4a22..e52b022 100644 --- a/ooni/report/cli.py +++ b/ooni/report/cli.py @@ -20,7 +20,7 @@ class Options(usage.Options): "Specify the configuration file to use."], ["collector", "c", None, "Specify the collector to upload the result to."], - ["bouncer", "b", "httpo://nkvphnp3p6agi5qq.onion", + ["bouncer", "b", None, "Specify the bouncer to query for a collector."] ] diff --git a/ooni/report/tool.py b/ooni/report/tool.py index fd504a6..d80a8e0 100644 --- a/ooni/report/tool.py +++ b/ooni/report/tool.py @@ -9,12 +9,15 @@ from ooni.reporter import OONIBReporter, OONIBReportLog from ooni.utils import log from ooni.report import parser from ooni.settings import config -from ooni.backend_client import BouncerClient +from ooni.backend_client import BouncerClient, CollectorClient @defer.inlineCallbacks def upload(report_file, collector=None, bouncer=None): oonib_report_log = OONIBReportLog() + collector_client = None + if collector: + collector_client = CollectorClient(address=collector) log.msg("Attempting to upload %s" % report_file) @@ -22,7 +25,7 @@ def upload(report_file, collector=None, bouncer=None): report_log = yaml.safe_load(f) report = parser.ReportLoader(report_file) - if bouncer and not collector: + if bouncer and collector_client is None: oonib_client = BouncerClient(bouncer) net_tests = [{ 'test-helpers': [], @@ -33,24 +36,33 @@ def upload(report_file, collector=None, bouncer=None): result = yield oonib_client.lookupTestCollector( net_tests ) - collector = str(result['net-tests'][0]['collector']) + collector_client = CollectorClient( + address=result['net-tests'][0]['collector'] + ) - if collector is None: + if collector_client is None: try: - collector = report_log[report_file]['collector'] - if collector is None: + collector_settings = report_log[report_file]['collector'] + if collector_settings is None: raise KeyError + elif isinstance(collector_settings, dict): + collector_client = CollectorClient(settings=collector_settings) + elif isinstance(collector_settings, str): + collector_client = CollectorClient(address=collector_settings) except KeyError: raise Exception( "No collector or bouncer specified" " and collector not in report log." ) - oonib_reporter = OONIBReporter(report.header, collector) - log.msg("Creating report for %s with %s" % (report_file, collector)) + oonib_reporter = OONIBReporter(report.header, collector_client) + log.msg("Creating report for %s with %s" % (report_file, + collector_client.settings)) report_id = yield oonib_reporter.createReport() report.header['report_id'] = report_id - yield oonib_report_log.created(report_file, collector, report_id) + yield oonib_report_log.created(report_file, + collector_client.settings, + report_id) log.msg("Writing report entries") for entry in report: yield oonib_reporter.writeReportEntry(entry) diff --git a/ooni/reporter.py b/ooni/reporter.py index 70c2f56..f76fada 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -454,7 +454,8 @@ class OONIBReportLog(object): def _closed(self, report_file): with self.edit_log() as report: - if report[report_file]['status'] != "created": + rs = report[report_file]['status'] + if rs != "created" and rs != "incomplete": raise errors.ReportNotCreated() del report[report_file] diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index a4844c8..7cbfd1d 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -42,7 +42,7 @@ class StreamListener(StreamListenerMixin): def _representHeaders(headers): represented_headers = {} for name, value in headers.getAllRawHeaders(): - represented_headers[name] = value[0] + represented_headers[name] = unicode(value[0], errors='ignore') return represented_headers def _representBody(body):