May 2016 - tor-commits - lists.torproject.org

[oonib/master] Merge pull request #58 from dstufft/patch-1
by art＠torproject.org 09 May '16

09 May '16

commit 479a1bb154037b834292ccc4b3d593d1472b44de Merge: 93cf891 af27fb6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Jul 27 12:45:33 2015 +0200 Merge pull request #58 from dstufft/patch-1 Update the pip recommendations README.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)

1 0

[oonib/master] Update the pip recommendations
by art＠torproject.org 09 May '16

09 May '16

commit af27fb6f11731f4124813a9bce8953a9e886f3f5 Author: Donald Stufft <donald(a)stufft.io> Date: Thu Jul 16 15:03:25 2015 -0400 Update the pip recommendations * Recommend at least pip 7 because versions since 1.3 have changed the defaults to better protect end users during install. * Use the new official location for the bootstrap script. --- README.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 7cf6fa4..5d7fb58 100644 --- a/README.rst +++ b/README.rst @@ -47,22 +47,24 @@ you'll want to make sure to get our keyring package in Debian:: sudo apt-get update sudo apt-get install deb.torproject.org-keyring tor tor-geoipdb -Pip (>=1.3.0) +Pip (>=7.0.0) ............. -We recommend using the Pip>=1.3.0 because it included several important +We recommend using the Pip>=7.0.0 because it included several important security and privacy related patches: * It forces the use of HTTPS for [PyPI](pypi.python.org). * and checks package hash sums before installation, with support for hashes more collision-resistant than MD5. + * It does not fetch insecure metadata from external sourced by default. + * It does not support an insecure index without explicit opt in. The least painful way (that we know of) to install a newer Pip is to use Pip's get-pip script:: # Grab the get-pip installer to make sure we have pip>=1.3.0 - curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py - sudo python ./get-pip.py ## pip (>=1.3.0) is recommended for security reasons + curl -O https://bootstrap.pypa.io/get-pip.py + sudo python ./get-pip.py ## pip (>=7.0.0) is recommended for security reasons # And make sure we're actually using the newer one: sudo update-alternatives --install /usr/bin/pip pip /usr/local/bin/pip 0

1 0

[oonib/master] Start adding support for reporting via JSON
by art＠torproject.org 09 May '16

09 May '16

commit a592986dfa5a79b8d4f0d11c44a1e18cdf46938b Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 4 17:44:37 2015 +0100 Start adding support for reporting via JSON --- oonib/report/handlers.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 7f7da20..50e0cb7 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -18,6 +18,11 @@ from oonib.config import config def report_file_name(archive_dir, report_details): timestamp = datetime.fromtimestamp(report_details['start_time']) + ext = report_details.get("format") + if ext == "json": + ext = "json" + else: + ext = "yamloo" keys = dict( report_details.items(), iso8601_timestamp=otime.timestamp(timestamp), @@ -26,9 +31,10 @@ def report_file_name(archive_dir, report_details): day=timestamp.strftime("%d"), hour=timestamp.strftime("%H"), minute=timestamp.strftime("%M"), - second=timestamp.strftime("%S") + second=timestamp.strftime("%S"), + ext=ext ) - report_file_template = "{probe_cc}/{test_name}-{iso8601_timestamp}-{probe_asn}-probe.yamloo" + report_file_template = "{probe_cc}/{test_name}-{iso8601_timestamp}-{probe_asn}-probe.{ext}" if config.main.report_file_template: report_file_template = config.main.report_file_template dst_filename = os.path.join(archive_dir, report_file_template.format(**keys)) @@ -45,13 +51,14 @@ class Report(object): stale_time, report_dir, archive_dir, - reports): + reports, file_format="yaml"): self.report_id = report_id self.stale_time = stale_time self.report_dir = report_dir self.archive_dir = archive_dir self.reports = reports + self.file_format = file_format self.refresh() @@ -76,14 +83,24 @@ class Report(object): report_filename = get_report_path(self.report_id) try: with open(report_filename) as fd: - g = yaml.safe_load_all(fd) - report_details = g.next() + if self.file_format == "json": + line = fd.readline() + json.loads(line) + else: + g = yaml.safe_load_all(fd) + report_details = g.next() except IOError: raise e.ReportNotFound dst_filename = report_file_name(self.archive_dir, report_details) shutil.move(report_filename, dst_filename) + if self.file_format == "json": + report_details["record_type"] = "footer" + with open(dst_filename, "a+") as fd: + json.dump(report_details, dst_filename) + fd.write("\n") + if not self.delayed_call.called: self.delayed_call.cancel() del self.reports[self.report_id]

1 0

[oonib/master] Don't override errors set via class attributes
by art＠torproject.org 09 May '16

09 May '16

commit 93cf891c7d77795e59c0763b499d234d3d54992f Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jan 24 15:21:03 2015 +0100 Don't override errors set via class attributes --- oonib/errors.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/oonib/errors.py b/oonib/errors.py index 64b179f..6d97190 100644 --- a/oonib/errors.py +++ b/oonib/errors.py @@ -2,9 +2,14 @@ from cyclone.web import HTTPError class OONIBError(HTTPError): - def __init__(self, status_code=500, log_message='oonib-error'): - self.status_code = status_code - self.log_message = log_message + status_code = 500 + log_message = 'oonib-error' + + def __init__(self, status_code=None, log_message=None): + if status_code: + self.status_code = status_code + if log_message: + self.log_message = log_message class InvalidRequest(OONIBError):

1 0

[oonib/master] Do not perform the conversions to JSON in the backend, but only write to JSON files when it's of file type JSON
by art＠torproject.org 09 May '16

09 May '16

commit 1f2ec237404c7bc5fa0f5d6d8375256e07bda9f8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Jan 27 17:48:25 2016 +0100 Do not perform the conversions to JSON in the backend, but only write to JSON files when it's of file type JSON --- oonib/__init__.py | 22 +------- oonib/report/handlers.py | 128 ++++++++++++++++++++++------------------------- 2 files changed, 61 insertions(+), 89 deletions(-) diff --git a/oonib/__init__.py b/oonib/__init__.py index 773f2db..2593f7d 100644 --- a/oonib/__init__.py +++ b/oonib/__init__.py @@ -61,25 +61,5 @@ def randomStr(length, num=True): chars += string.digits return ''.join(random.choice(chars) for x in range(length)) -def binary_to_base64_dict(data): - from base64 import b64encode - return { - "data": b64encode(data), - "format": "base64" - } - def json_dumps(obj): - def _default(o): - if isinstance(o, set): - return list(o) - elif isinstance(o, str): - try: - o = unicode(o, 'ascii') - except UnicodeDecodeError: - try: - o = unicode(o, 'utf-8') - except UnicodeDecodeError: - o = binary_to_base64_dict(o) - return o - - return json.dumps(obj, default=_default) + return json.dumps(obj) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 7671723..c3af0c6 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -18,6 +18,13 @@ from oonib.config import config def report_file_name(archive_dir, report_details): timestamp = datetime.fromtimestamp(report_details['start_time']) + if report_details['format'] == 'json': + ext = 'json' + elif report_details['format'] == 'yaml': + ext = 'yaml' + else: + ext = 'invalid' + keys = dict( report_details.items(), iso8601_timestamp=otime.timestamp(timestamp), @@ -26,9 +33,10 @@ def report_file_name(archive_dir, report_details): day=timestamp.strftime("%d"), hour=timestamp.strftime("%H"), minute=timestamp.strftime("%M"), - second=timestamp.strftime("%S") + second=timestamp.strftime("%S"), + ext=ext ) - report_file_template = "{iso8601_timestamp}-{test_name}-{probe_asn}-{probe_cc}-probe-0.2.0.json" + report_file_template = "{iso8601_timestamp}-{test_name}-{probe_asn}-{probe_cc}-probe-0.2.0.{ext}" if config.main.report_file_template: report_file_template = config.main.report_file_template dst_filename = os.path.join(archive_dir, report_file_template.format(**keys)) @@ -45,9 +53,12 @@ class Report(object): stale_time, report_dir, archive_dir, - reports): + reports, + report_details): self.report_id = report_id + self.report_details = report_details + self.stale_time = stale_time self.report_dir = report_dir self.archive_dir = archive_dir @@ -75,13 +86,12 @@ class Report(object): report_filename = get_report_path(self.report_id) try: - with open(report_filename) as fd: - line = fd.readline() - report_details = json.loads(line.strip()) + open(report_filename).close() except IOError: raise e.ReportNotFound - dst_filename = report_file_name(self.archive_dir, report_details) + dst_filename = report_file_name(self.archive_dir, + self.report_details) shutil.move(report_filename, dst_filename) if not self.delayed_call.called: @@ -119,6 +129,7 @@ def parseNewReportRequest(request): version_string = re.compile("[0-9A-Za-z_\-\.]+$") name = re.compile("[a-zA-Z0-9_\- ]+$") probe_asn = re.compile("AS[0-9]+$") + probe_cc = re.compile("[A-Z]{2}$") test_helper = re.compile("[A-Za-z0-9_\-]+$") expected_request = { @@ -139,7 +150,7 @@ def parseNewReportRequest(request): except KeyError: raise e.MissingField(k) - print "Matching %s with %s | %s" % (regexp, value_to_check, k) + log.debug("Matching %s with %s | %s" % (regexp, value_to_check, k)) if re.match(regexp, str(value_to_check)): continue else: @@ -152,39 +163,34 @@ def parseNewReportRequest(request): except KeyError: pass - parsed_request['format'] = parsed_request.get('format', 'yaml') - - return parsed_request - -def validate_report_header(report_header): - required_keys = ['probe_asn', 'probe_cc', 'probe_ip', 'software_name', - 'software_version', 'test_name', 'test_version'] - for key in required_keys: - if key not in report_header: - raise e.MissingReportHeaderKey(key) - - if report_header['probe_asn'] is None: - report_header['probe_asn'] = 'AS0' - - if not re.match('AS[0-9]+$', report_header['probe_asn']): - raise e.InvalidReportHeader('probe_asn') - - # If no country is known, set it to be ZZ (user assigned value in ISO 3166) - if report_header['probe_cc'] is None: - report_header['probe_cc'] = 'ZZ' - - if not re.match('[a-zA-Z]{2}$', report_header['probe_cc']): - raise e.InvalidReportHeader('probe_cc') + if 'start_time' not in parsed_request: + try: + header = yaml.safe_load(parsed_request['content']) + parsed_request['start_time'] = header['start_time'] + except Exception as exc: + log.exception(exc) + raise e.InvalidRequestField("start_time") - if not re.match('[a-z_\-]+$', report_header['test_name']): - raise e.InvalidReportHeader('test_name') + try: + parsed_request['start_time'] = float(header['start_time']) + except ValueError as exc: + log.exception(exc) + raise e.InvalidRequestField("start_time") - if not re.match('([0-9]+\.)+[0-9]+$', report_header['test_version']): - raise e.InvalidReportHeader('test_version') + if 'probe_cc' not in parsed_request: + try: + header = yaml.safe_load(parsed_request['content']) + parsed_request['probe_cc'] = header['probe_cc'] + if not re.match(probe_cc, parsed_request['probe_cc']): + raise Exception("Does not match the regexp") + except Exception as exc: + log.exception(exc) + raise e.InvalidRequestField("probe_cc") - return report_header + parsed_request['format'] = parsed_request.get('format', 'yaml') + return parsed_request class ReportHandler(OONIBHandler): @@ -211,21 +217,14 @@ class UpdateReportMixin(object): content_format = parsed_request.get('format', 'yaml') if content_format == 'json': data = json_dumps(parsed_request['content']) + data += "\n" elif content_format == 'yaml': - try: - entry = yaml.safe_load_all(parsed_request['content']).next() - data = json_dumps(entry) - except Exception as exc: - log.error("Received an invalid entry") - log.msg(parsed_request['content']) - log.exception(exc) - raise e.OONIBError(400, "Invalid report entry") + data = parsed_request['content'] else: raise e.InvalidFormatField try: with open(report_filename, 'a+') as fd: fd.write(data) - fd.write("\n") except IOError: raise e.OONIBError(404, "Report not found") self.write({'status': 'success'}) @@ -288,12 +287,6 @@ class NewReportHandlerFile(ReportHandler, UpdateReportMixin): log.debug("Parsed this data %s" % report_data) - software_name = str(report_data['software_name']) - software_version = str(report_data['software_version']) - - probe_asn = str(report_data['probe_asn']) - probe_cc = str(report_data.get('probe_cc', 'ZZ')) - self.testName = str(report_data['test_name']) self.testVersion = str(report_data['test_version']) @@ -304,22 +297,21 @@ class NewReportHandlerFile(ReportHandler, UpdateReportMixin): raise e.InputHashNotProvided self.checkPolicy() + data = None if 'content' in report_data: if report_data['format'] == 'json': content = report_data['content'] + content['backend_version'] = config.backend_version + data = json_dumps(content) elif report_data['format'] == 'yaml': - content = None + content = yaml.safe_load(report_data['content']) + content['backend_version'] = config.backend_version + data = "---\n" + yaml.dump(content) + "...\n" else: raise e.InvalidFormatField - content['backend_version'] = config.backend_version - else: - content = None - - if not probe_asn: - probe_asn = "AS0" report_id = otime.timestamp() + '_' \ - + probe_asn + '_' \ + + report_data.get('probe_asn', 'AS0') + '_' \ + randomStr(50) # The report filename contains the timestamp of the report plus a @@ -340,15 +332,15 @@ class NewReportHandlerFile(ReportHandler, UpdateReportMixin): except KeyError: raise e.TestHelperNotFound - self.reports[report_id] = Report(report_id, - self.stale_time, - self.report_dir, - self.archive_dir, - self.reports) - if content: - # XXX make sure to validate the cotent to ensure it's possible to - # encode as JSON - data = json_dumps(content) + "\n" + self.reports[report_id] = Report(report_id=report_id, + stale_time=self.stale_time, + report_dir=self.report_dir, + archive_dir=self.archive_dir, + reports=self.reports, + report_details=report_data + ) + + if data is not None: self.writeToReport(report_filename, data) else: open(report_filename, 'w+').close()

1 0

[oonib/master] Add support for better handling start_time
by art＠torproject.org 09 May '16

09 May '16

commit 1535967e420fd2b1bd61fd9001188d4a71983684 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Jan 28 18:27:48 2016 +0100 Add support for better handling start_time Invert inlineCallbacks Convert PUT to POST (PUT is supposed to be indempotent) * Better exception handling for DNS --- oonib/oonibackend.py | 1 - oonib/report/handlers.py | 32 +++++++++++++++++--------------- oonib/testhelpers/http_helpers.py | 12 ++++++++++++ 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 9c10dc8..a392b82 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -102,7 +102,6 @@ if config.helpers['http-return-json-headers'].port: multiService.addService(http_return_request_helper) http_return_request_helper.startService() - # add the tor collector service here if config.main.tor_hidden_service: torconfig = TorConfig() diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index c3af0c6..6594805 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -99,7 +99,7 @@ class Report(object): del self.reports[self.report_id] -def parseUpdateReportRequest(request): +def parseUpdateReportRequest(request, report_id=None): #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") # this is the regexp for the reports that include the timestamp @@ -111,14 +111,15 @@ def parseUpdateReportRequest(request): # We are also keeping in memory multiple copies of the same object. A lot # of optimization can be done. parsed_request = json.loads(request) - try: - report_id = parsed_request['report_id'] - except KeyError: + + report_id = parsed_request.get('report_id', report_id) + if not report_id: raise e.MissingField('report_id') if not re.match(report_id_regexp, report_id): raise e.InvalidRequestField('report_id') + parsed_request['report_id'] = report_id return parsed_request @@ -173,7 +174,7 @@ def parseNewReportRequest(request): raise e.InvalidRequestField("start_time") try: - parsed_request['start_time'] = float(header['start_time']) + parsed_request['start_time'] = float(parsed_request['start_time']) except ValueError as exc: log.exception(exc) raise e.InvalidRequestField("start_time") @@ -183,7 +184,7 @@ def parseNewReportRequest(request): header = yaml.safe_load(parsed_request['content']) parsed_request['probe_cc'] = header['probe_cc'] if not re.match(probe_cc, parsed_request['probe_cc']): - raise Exception("Does not match the regexp") + raise e.InvalidRequestField("probe_cc") except Exception as exc: log.exception(exc) raise e.InvalidRequestField("probe_cc") @@ -206,7 +207,7 @@ class ReportHandler(OONIBHandler): class UpdateReportMixin(object): def updateReport(self, report_id, parsed_request): - log.debug("Got this request %s" % parsed_request) + log.msg("Got this request %s" % parsed_request) report_filename = os.path.join(self.report_dir, report_id) try: @@ -279,7 +280,11 @@ class NewReportHandlerFile(ReportHandler, UpdateReportMixin): * Response - {'backend_version': 'XXX', 'report_id': 'XXX'} + { + 'backend_version': 'XXX', + 'report_id': 'XXX', + 'supported_formats': ['yaml', 'json'] + } """ # Note: the request is being validated inside of parseNewReportRequest. @@ -320,7 +325,8 @@ class NewReportHandlerFile(ReportHandler, UpdateReportMixin): response = { 'backend_version': config.backend_version, - 'report_id': report_id + 'report_id': report_id, + 'supported_formats': ['yaml', 'json'] } requested_helper = report_data.get('test_helper') @@ -363,17 +369,13 @@ class NewReportHandlerFile(ReportHandler, UpdateReportMixin): """ parsed_request = parseUpdateReportRequest(self.request.body) report_id = parsed_request['report_id'] - self.updateReport(report_id, parsed_request) class UpdateReportHandlerFile(ReportHandler, UpdateReportMixin): - def post(self, report_id): - try: - parsed_request = json.loads(self.request.body) - except ValueError: - raise e.InvalidRequest + parsed_request = parseUpdateReportRequest(self.request.body, report_id) + report_id = parsed_request['report_id'] self.updateReport(report_id, parsed_request) diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py index a28cbad..603bdc5 100644 --- a/oonib/testhelpers/http_helpers.py +++ b/oonib/testhelpers/http_helpers.py @@ -2,6 +2,10 @@ import json import random import string +from twisted.internet.protocol import Factory, Protocol + +from twisted.internet.endpoints import TCP4ClientEndpoint + from twisted.internet import protocol, defer from cyclone.web import RequestHandler, Application @@ -168,6 +172,14 @@ class HTTPRandomPage(HTTPTrapAll): length = 100000 self.write(self.genRandomPage(length, keyword)) +class TCPConnectProtocol(Protocol): + def connectionMade(self): + self.transport.loseConnection() + +class TCPConnectFactory(Factory): + def buildProtocol(self, addr): + return TCPConnectProtocol() + HTTPRandomPageHelper = Application([ # XXX add regexps here (r"/(.*)/(.*)", HTTPRandomPage)

1 0

[oonib/master] Update README.rst
by art＠torproject.org 09 May '16

09 May '16

commit f556a9adb329a0a16858b25bb04b701ed59766e3 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Feb 3 23:32:52 2016 +0100 Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 5d7fb58..63a242c 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ Distro dependencies (Debian) There are a few dependencies which we recommend you get from your distribution's archives:: - sudo apt-get install build-essential python-dev python-setuptools openssl libsqlite3-dev libffi-dev + sudo apt-get install build-essential python-dev python-setuptools openssl libsqlite3-dev libffi-dev git curl Tor ... @@ -83,7 +83,7 @@ setting up this up and installing are:: # Create the virtualenv for ooni-backend... mkvirtualenv -a $PWD --unzip-setuptools --setuptools --no-site-packages oonib # ...and install ooni-backend (sudo is not necessary since we're in a virtualenv): - pip install -r requirements.txt --use-mirrors + pip install -r requirements.txt # Note: it is important that you install the requirements before you run # the setup.py script. If you fail to do so they will be downloaded over # plaintext.

1 0

[oonib/master] Add report_id to filepath
by art＠torproject.org 09 May '16

09 May '16

commit 8de4e05f7c2b64e2944d5a97dcdc98dfe017ca16 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Feb 5 20:49:11 2016 +0100 Add report_id to filepath --- oonib/report/handlers.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 30d66c6..b27a943 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -16,7 +16,8 @@ from oonib import randomStr, otime, log, json_dumps from oonib.config import config -def report_file_name(archive_dir, report_details): +def report_file_name(archive_dir, report_details, + report_id='no_report_id'): timestamp = datetime.fromtimestamp(report_details['start_time']) if report_details['format'] == 'json': ext = 'json' @@ -34,9 +35,10 @@ def report_file_name(archive_dir, report_details): hour=timestamp.strftime("%H"), minute=timestamp.strftime("%M"), second=timestamp.strftime("%S"), - ext=ext + ext=ext, + report_id=report_id ) - report_file_template = "{iso8601_timestamp}-{test_name}-{probe_asn}-{probe_cc}-probe-0.2.0.{ext}" + report_file_template = "{iso8601_timestamp}-{test_name}-{report_id}-{probe_asn}-{probe_cc}-probe-0.2.0.{ext}" if config.main.report_file_template: report_file_template = config.main.report_file_template dst_filename = os.path.join(archive_dir, report_file_template.format(**keys)) @@ -90,8 +92,9 @@ class Report(object): except IOError: raise e.ReportNotFound - dst_filename = report_file_name(self.archive_dir, - self.report_details) + dst_filename = report_file_name(archive_dir=self.archive_dir, + report_details=self.report_details, + report_id=self.report_id) shutil.move(report_filename, dst_filename) if not self.delayed_call.called:

1 0

[oonib/master] Fixes to unittests of report to changes in backend
by art＠torproject.org 09 May '16

09 May '16

commit aa920512752b1b1b844b45af1b58e4f838c6d42a Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Feb 4 17:42:15 2016 +0100 Fixes to unittests of report to changes in backend * Minor refactoring of unittests * Fix a backward compatibility bug spotted in unittesting --- oonib/report/handlers.py | 27 ++++++++++-------- oonib/test/test_report.py | 71 +++++++++++++++++------------------------------ 2 files changed, 41 insertions(+), 57 deletions(-) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 6594805..e35b60a 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -303,17 +303,22 @@ class NewReportHandlerFile(ReportHandler, UpdateReportMixin): self.checkPolicy() data = None - if 'content' in report_data: - if report_data['format'] == 'json': - content = report_data['content'] - content['backend_version'] = config.backend_version - data = json_dumps(content) - elif report_data['format'] == 'yaml': - content = yaml.safe_load(report_data['content']) - content['backend_version'] = config.backend_version - data = "---\n" + yaml.dump(content) + "...\n" - else: - raise e.InvalidFormatField + if 'content' in report_data and report_data['format'] == 'json': + content = report_data['content'] + content['backend_version'] = config.backend_version + data = json_dumps(content) + elif report_data['format'] == 'yaml': + content = { + 'software_name': str(report_data['software_name']), + 'software_version': str(report_data['software_version']), + 'probe_asn': str(report_data['probe_asn']), + 'probe_cc': str(report_data['probe_cc']), + 'test_name': self.testName, + 'test_version': self.testVersion, + 'input_hashes': self.inputHashes, + 'start_time': report_data.get('start_time', time.time()) + } + data = "---\n" + yaml.dump(content) + "...\n" report_id = otime.timestamp() + '_' \ + report_data.get('probe_asn', 'AS0') + '_' \ diff --git a/oonib/test/test_report.py b/oonib/test/test_report.py index 9da8b08..55f78be 100644 --- a/oonib/test/test_report.py +++ b/oonib/test/test_report.py @@ -1,7 +1,9 @@ -import os +import time import json import yaml +from copy import deepcopy + from twisted.internet import defer from cyclone import web @@ -72,6 +74,16 @@ test_version: 0.1.0 ... """ +dummy_data = { + 'software_name': 'ooni-test', + 'software_version': '0.1', + 'test_name': 'some-test', + 'test_version': '0.1', + 'probe_asn': 'AS0', + 'probe_cc': 'ZZ', + 'start_time': time.time() +} + for _, handler in reportAPI: handler.initialize = mock_initialize @@ -92,14 +104,7 @@ class TestReport(HandlerTestCase): @defer.inlineCallbacks def test_create_valid_report(self): - data = { - 'software_name': 'ooni-test', - 'software_version': '0.1', - 'test_name': 'some-test', - 'test_version': '0.1', - 'probe_asn': 'AS0' - } - response = yield self.request('/report', "POST", data) + response = yield self.request('/report', "POST", dummy_data) response_body = json.loads(response.body) self.assertIn('backend_version', response_body) self.assertIn('report_id', response_body) @@ -107,14 +112,8 @@ class TestReport(HandlerTestCase): @defer.inlineCallbacks def test_create_valid_report_with_content(self): - data = { - 'software_name': 'ooni-test', - 'software_version': '0.1', - 'test_name': 'some-test', - 'test_version': '0.1', - 'probe_asn': 'AS0', - 'content': sample_report_header - } + data = deepcopy(dummy_data) + data['content'] = sample_report_header response = yield self.request('/report', "POST", data) response_body = json.loads(response.body) self.assertIn('backend_version', response_body) @@ -123,13 +122,8 @@ class TestReport(HandlerTestCase): @defer.inlineCallbacks def test_create_invalid_report(self): - data = { - 'software_name': 'ooni-test', - 'software_version': '0.1', - 'test_name': 'some-test', - 'test_version': '0.1', - 'probe_asn': 'XXX' - } + data = deepcopy(dummy_data) + data['probe_asn'] = 'XXXINVALID' response = yield self.request('/report', "POST", data) response_body = json.loads(response.body) self.assertIn('error', response_body) @@ -138,14 +132,7 @@ class TestReport(HandlerTestCase): @defer.inlineCallbacks def test_create_and_update_report(self): - report_header = { - 'software_name': 'ooni-test', - 'software_version': '0.1', - 'test_name': 'some-test', - 'test_version': '0.1', - 'probe_asn': 'AS0' - } - response = yield self.request('/report', "POST", report_header) + response = yield self.request('/report', "POST", dummy_data) response_body = json.loads(response.body) self.assertIn('backend_version', response_body) self.assertIn('report_id', response_body) @@ -159,21 +146,14 @@ class TestReport(HandlerTestCase): written_report = yaml.safe_load_all(f) written_report_header = written_report.next() - for key in report_header.keys(): - self.assertEqual(written_report_header[key], report_header[key]) + for key in dummy_data.keys(): + self.assertEqual(written_report_header[key], dummy_data[key]) self.assertEqual(yaml.safe_load(sample_report_entry), written_report.next()) @defer.inlineCallbacks def test_create_update_and_close_report(self): - report_header = { - 'software_name': 'ooni-test', - 'software_version': '0.1', - 'test_name': 'some-test', - 'test_version': '0.1', - 'probe_asn': 'AS0' - } - response = yield self.request('/report', "POST", report_header) + response = yield self.request('/report', "POST", dummy_data) response_body = json.loads(response.body) self.assertIn('backend_version', response_body) self.assertIn('report_id', response_body) @@ -188,15 +168,16 @@ class TestReport(HandlerTestCase): written_report = yaml.safe_load_all(f) written_report_header = written_report.next() - for key in report_header.keys(): + for key in dummy_data.keys(): self.assertEqual(written_report_header[key], - report_header[key]) + dummy_data[key]) self.assertEqual(yaml.safe_load(sample_report_entry), written_report.next()) response = yield self.request('/report/%s/close' % report_id, "POST") + written_report_header['format'] = 'yaml' written_report_path = report_file_name(".", written_report_header) with open(written_report_path) as f: self.filenames.add(written_report_path) @@ -206,5 +187,3 @@ class TestReport(HandlerTestCase): for i in range(report_entry_count): self.assertEqual(yaml.safe_load(sample_report_entry), written_report.next()) - - self.directories.add('ZZ')

1 0

[oonib/master] More consistency changes and debugging
by art＠torproject.org 09 May '16

09 May '16

commit 426a263ff67ce1fe1fcd6872f4c281efe1495d24 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Feb 5 17:19:03 2016 +0100 More consistency changes and debugging --- .travis.yml | 1 - oonib/report/handlers.py | 75 ++++++++++++++++++++++-------------------------- 2 files changed, 35 insertions(+), 41 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0e20f3a..8e19634 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,6 @@ before_install: python: #- "3.3" - "2.7" - - "2.6" install: - pip install -r requirements.txt - pip install coveralls diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index e35b60a..30d66c6 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -123,10 +123,7 @@ def parseUpdateReportRequest(request, report_id=None): return parsed_request -def parseNewReportRequest(request): - """ - Here we parse a new report request. - """ +def validateHeader(header): version_string = re.compile("[0-9A-Za-z_\-\.]+$") name = re.compile("[a-zA-Z0-9_\- ]+$") probe_asn = re.compile("AS[0-9]+$") @@ -138,16 +135,29 @@ def parseNewReportRequest(request): 'software_version': version_string, 'test_name': name, 'test_version': version_string, - 'probe_asn': probe_asn + 'probe_asn': probe_asn, + 'probe_cc': probe_cc, + 'data_format_version': version_string } - parsed_request = json.loads(request) - if 'probe_asn' not in parsed_request or not parsed_request['probe_asn']: - parsed_request['probe_asn'] = 'AS0' + if not header.get('probe_asn'): + header['probe_asn'] = 'AS0' + + if not header.get('probe_cc'): + header['probe_cc'] = 'ZZ' + + if not header.get('start_time'): + header['start_time'] = time.time() + + if not header.get('start_time'): + header['start_time'] = time.time() + + if not header.get('data_format_version'): + header['data_format_version'] = '0.1.0' for k, regexp in expected_request.items(): try: - value_to_check = parsed_request[k] + value_to_check = header[k] except KeyError: raise e.MissingField(k) @@ -158,39 +168,24 @@ def parseNewReportRequest(request): raise e.InvalidRequestField(k) try: - requested_test_helper = parsed_request['test_helper'] + requested_test_helper = header['test_helper'] if not re.match(test_helper, str(requested_test_helper)): raise e.InvalidRequestField('test_helper') except KeyError: pass + return header - if 'start_time' not in parsed_request: - try: - header = yaml.safe_load(parsed_request['content']) - parsed_request['start_time'] = header['start_time'] - except Exception as exc: - log.exception(exc) - raise e.InvalidRequestField("start_time") - - try: - parsed_request['start_time'] = float(parsed_request['start_time']) - except ValueError as exc: - log.exception(exc) - raise e.InvalidRequestField("start_time") - - if 'probe_cc' not in parsed_request: - try: - header = yaml.safe_load(parsed_request['content']) - parsed_request['probe_cc'] = header['probe_cc'] - if not re.match(probe_cc, parsed_request['probe_cc']): - raise e.InvalidRequestField("probe_cc") - except Exception as exc: - log.exception(exc) - raise e.InvalidRequestField("probe_cc") +def parseNewReportRequest(request): + """ + Here we parse a new report request. + """ + parsed_request = json.loads(request) parsed_request['format'] = parsed_request.get('format', 'yaml') + parsed_request = validateHeader(parsed_request) + return parsed_request class ReportHandler(OONIBHandler): @@ -303,11 +298,7 @@ class NewReportHandlerFile(ReportHandler, UpdateReportMixin): self.checkPolicy() data = None - if 'content' in report_data and report_data['format'] == 'json': - content = report_data['content'] - content['backend_version'] = config.backend_version - data = json_dumps(content) - elif report_data['format'] == 'yaml': + if report_data['format'] == 'yaml' and 'content' not in report_data: content = { 'software_name': str(report_data['software_name']), 'software_version': str(report_data['software_version']), @@ -315,10 +306,14 @@ class NewReportHandlerFile(ReportHandler, UpdateReportMixin): 'probe_cc': str(report_data['probe_cc']), 'test_name': self.testName, 'test_version': self.testVersion, - 'input_hashes': self.inputHashes, - 'start_time': report_data.get('start_time', time.time()) + 'input_hashes': report_data.get('input_hashes', []), + 'start_time': report_data['start_time'], + 'data_format_version': str(report_data['data_format_version']) } data = "---\n" + yaml.dump(content) + "...\n" + elif report_data['format'] == 'yaml' and 'content' in report_data: + header = yaml.safe_load(report_data['content']) + data = "---\n" + yaml.dump(validateHeader(header)) + "...\n" report_id = otime.timestamp() + '_' \ + report_data.get('probe_asn', 'AS0') + '_' \

1 0