[tor-commits] [ooni-probe/master] Various improvement related to backend_client
art at torproject.org
art at torproject.org
Mon May 30 16:28:33 UTC 2016
commit 24ea012b508526165b1a7d1eef1d76475bd313e6
Author: Arturo Filastò <arturo at filasto.net>
Date: Tue May 17 20:02:09 2016 +0200
Various improvement related to backend_client
* Include web_connectivity in default generated deck
* Fix serialisation of HTTP Headers
* Support setting collector and bouncer in oonireport upload
---
ooni/deck.py | 4 +++-
ooni/deckgen/cli.py | 22 +++++++---------------
ooni/report/cli.py | 2 +-
ooni/report/tool.py | 30 +++++++++++++++++++++---------
ooni/reporter.py | 3 ++-
ooni/templates/httpt.py | 2 +-
6 files changed, 35 insertions(+), 28 deletions(-)
diff --git a/ooni/deck.py b/ooni/deck.py
index 007097e..404a4d0 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -136,10 +136,12 @@ class Deck(InputFile):
net_test_loader = NetTestLoader(test['options']['subargs'],
annotations=test['options'].get('annotations', {}),
test_file=nettest_path)
- if test['options']['collector']:
+ if test['options']['collector'] is not None:
net_test_loader.collector = CollectorClient(
test['options']['collector']
)
+ if test['options']['bouncer'] is not None:
+ self.bouncer = test['options']['bouncer']
self.insert(net_test_loader)
def insert(self, net_test_loader):
diff --git a/ooni/deckgen/cli.py b/ooni/deckgen/cli.py
index 2e0e976..a6046f3 100644
--- a/ooni/deckgen/cli.py
+++ b/ooni/deckgen/cli.py
@@ -27,6 +27,7 @@ class Options(usage.Options):
"generate the deck."],
["collector", None, None, "Specify a custom collector to use when "
"submitting reports"],
+ ["bouncer", None, None, "Specify a custom bouncer to use"],
["output", "o", None,
"Specify the directory where to write output."]
]
@@ -44,7 +45,6 @@ class Deck(object):
"annotations": None,
"collector": None,
- # XXX setting this is currently not supported
"bouncer": None,
"reportfile": None,
@@ -56,13 +56,15 @@ class Deck(object):
}
}
- def __init__(self, collector=None):
+ def __init__(self, collector=None, bouncer=None):
self.deck_entries = []
self.collector = collector
+ self.bouncer = bouncer
def add_test(self, test_file, subargs=[]):
deck_entry = copy.deepcopy(self._base_entry)
deck_entry['options']['collector'] = self.collector
+ deck_entry['options']['bouncer'] = self.bouncer
deck_entry['options']['test_file'] = test_file
deck_entry['options']['subargs'] = subargs
self.deck_entries.append(deck_entry)
@@ -89,24 +91,14 @@ def generate_deck(options):
url_list_global = citizenlab_test_lists.generate_global_input(
options['output']
)
- dns_servers = namebench_dns_servers.generate_country_input(
- options['country-code'],
- options['output']
- )
- deck = Deck(collector=options['collector'])
+ deck = Deck(collector=options['collector'], bouncer=options['bouncer'])
deck.add_test('manipulation/http_invalid_request_line')
deck.add_test('manipulation/http_header_field_manipulation')
if url_list_country is not None:
- deck.add_test('blocking/http_requests', ['-f', url_list_country])
- deck.add_test('blocking/http_requests', ['-f', url_list_global])
-
- if url_list_country is not None:
- deck.add_test('blocking/dns_consistency',
- ['-f', url_list_country, '-T', dns_servers])
- deck.add_test('blocking/dns_consistency',
- ['-f', url_list_global, '-T', dns_servers])
+ deck.add_test('blocking/web_connectivity', ['-f', url_list_country])
+ deck.add_test('blocking/web_connectivity', ['-f', url_list_global])
if config.advanced.debug:
deck.pprint()
diff --git a/ooni/report/cli.py b/ooni/report/cli.py
index c1b4a22..e52b022 100644
--- a/ooni/report/cli.py
+++ b/ooni/report/cli.py
@@ -20,7 +20,7 @@ class Options(usage.Options):
"Specify the configuration file to use."],
["collector", "c", None,
"Specify the collector to upload the result to."],
- ["bouncer", "b", "httpo://nkvphnp3p6agi5qq.onion",
+ ["bouncer", "b", None,
"Specify the bouncer to query for a collector."]
]
diff --git a/ooni/report/tool.py b/ooni/report/tool.py
index fd504a6..d80a8e0 100644
--- a/ooni/report/tool.py
+++ b/ooni/report/tool.py
@@ -9,12 +9,15 @@ from ooni.reporter import OONIBReporter, OONIBReportLog
from ooni.utils import log
from ooni.report import parser
from ooni.settings import config
-from ooni.backend_client import BouncerClient
+from ooni.backend_client import BouncerClient, CollectorClient
@defer.inlineCallbacks
def upload(report_file, collector=None, bouncer=None):
oonib_report_log = OONIBReportLog()
+ collector_client = None
+ if collector:
+ collector_client = CollectorClient(address=collector)
log.msg("Attempting to upload %s" % report_file)
@@ -22,7 +25,7 @@ def upload(report_file, collector=None, bouncer=None):
report_log = yaml.safe_load(f)
report = parser.ReportLoader(report_file)
- if bouncer and not collector:
+ if bouncer and collector_client is None:
oonib_client = BouncerClient(bouncer)
net_tests = [{
'test-helpers': [],
@@ -33,24 +36,33 @@ def upload(report_file, collector=None, bouncer=None):
result = yield oonib_client.lookupTestCollector(
net_tests
)
- collector = str(result['net-tests'][0]['collector'])
+ collector_client = CollectorClient(
+ address=result['net-tests'][0]['collector']
+ )
- if collector is None:
+ if collector_client is None:
try:
- collector = report_log[report_file]['collector']
- if collector is None:
+ collector_settings = report_log[report_file]['collector']
+ if collector_settings is None:
raise KeyError
+ elif isinstance(collector_settings, dict):
+ collector_client = CollectorClient(settings=collector_settings)
+ elif isinstance(collector_settings, str):
+ collector_client = CollectorClient(address=collector_settings)
except KeyError:
raise Exception(
"No collector or bouncer specified"
" and collector not in report log."
)
- oonib_reporter = OONIBReporter(report.header, collector)
- log.msg("Creating report for %s with %s" % (report_file, collector))
+ oonib_reporter = OONIBReporter(report.header, collector_client)
+ log.msg("Creating report for %s with %s" % (report_file,
+ collector_client.settings))
report_id = yield oonib_reporter.createReport()
report.header['report_id'] = report_id
- yield oonib_report_log.created(report_file, collector, report_id)
+ yield oonib_report_log.created(report_file,
+ collector_client.settings,
+ report_id)
log.msg("Writing report entries")
for entry in report:
yield oonib_reporter.writeReportEntry(entry)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 70c2f56..f76fada 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -454,7 +454,8 @@ class OONIBReportLog(object):
def _closed(self, report_file):
with self.edit_log() as report:
- if report[report_file]['status'] != "created":
+ rs = report[report_file]['status']
+ if rs != "created" and rs != "incomplete":
raise errors.ReportNotCreated()
del report[report_file]
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index a4844c8..7cbfd1d 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -42,7 +42,7 @@ class StreamListener(StreamListenerMixin):
def _representHeaders(headers):
represented_headers = {}
for name, value in headers.getAllRawHeaders():
- represented_headers[name] = value[0]
+ represented_headers[name] = unicode(value[0], errors='ignore')
return represented_headers
def _representBody(body):
More information about the tor-commits
mailing list