commit 46eebbc9010481264d4ed02af96967ec7a55f8ad Author: Arturo Filastò art@fuffa.org Date: Wed Jun 18 11:52:05 2014 +0200

Pep8 related fixes. --- ooni/errors.py | 77 +++++++++++++++++++++++++++++++++----- ooni/reporter.py | 108 ++++++++++++++++++++++++++++++++---------------------- 2 files changed, 133 insertions(+), 52 deletions(-)

diff --git a/ooni/errors.py b/ooni/errors.py index 3b62b2a..9f9ae75 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -15,23 +15,42 @@ from txsocksx.errors import HostUnreachable, ConnectionRefused from txsocksx.errors import TTLExpired, CommandNotSupported

from socket import gaierror + + def handleAllFailures(failure): """ Here we make sure to trap all the failures that are supported by the failureToString function and we return the the string that represents the failure. """ - failure.trap(ConnectionRefusedError, gaierror, DNSLookupError, - TCPTimedOutError, ResponseNeverReceived, DeferTimeoutError, - GenericTimeoutError, - SOCKSError, MethodsNotAcceptedError, AddressNotSupported, - ConnectionError, NetworkUnreachable, ConnectionLostEarly, - ConnectionNotAllowed, NoAcceptableMethods, ServerFailure, - HostUnreachable, ConnectionRefused, TTLExpired, CommandNotSupported, - ConnectError, ConnectionLost, CancelledError) + failure.trap( + ConnectionRefusedError, + gaierror, + DNSLookupError, + TCPTimedOutError, + ResponseNeverReceived, + DeferTimeoutError, + GenericTimeoutError, + SOCKSError, + MethodsNotAcceptedError, + AddressNotSupported, + ConnectionError, + NetworkUnreachable, + ConnectionLostEarly, + ConnectionNotAllowed, + NoAcceptableMethods, + ServerFailure, + HostUnreachable, + ConnectionRefused, + TTLExpired, + CommandNotSupported, + ConnectError, + ConnectionLost, + CancelledError)

return failureToString(failure)

+ def failureToString(failure): """ Given a failure instance return a string representing the kind of error @@ -119,7 +138,7 @@ def failureToString(failure): elif isinstance(failure.value, SOCKSError): log.err("Generic SOCKS error") string = 'socks_error' - + elif isinstance(failure.value, CancelledError): log.err("Task timed out") string = 'task_timed_out' @@ -130,99 +149,139 @@ def failureToString(failure):

return string

+ class DirectorException(Exception): pass

+ class UnableToStartTor(DirectorException): pass

+ class InvalidOONIBCollectorAddress(Exception): pass

+ class InvalidOONIBBouncerAddress(Exception): pass

+ class AllReportersFailed(Exception): pass

+ class GeoIPDataFilesNotFound(Exception): pass

+ class ReportNotCreated(Exception): pass

+ class ReportAlreadyClosed(Exception): pass

+ class TorStateNotFound(Exception): pass

+ class TorControlPortNotFound(Exception): pass

+ class ReportNotCreated(Exception): pass

+ class InsufficientPrivileges(Exception): pass

+ class ProbeIPUnknown(Exception): pass

+ class GeoIPDataFilesNotFound(Exception): pass

+ class NoMoreReporters(Exception): pass

+ class TorNotRunning(Exception): pass

+ class OONIBError(Exception): pass

+ class OONIBReportError(OONIBError): pass

+ class OONIBReportUpdateError(OONIBReportError): pass

+ class OONIBReportCreationError(OONIBReportError): pass

+ class OONIBTestDetailsLookupError(OONIBReportError): pass

+ class UnableToLoadDeckInput(Exception): pass

+ class CouldNotFindTestHelper(Exception): pass

+ class CouldNotFindTestCollector(Exception): pass

+ class NetTestNotFound(Exception): pass

+ class MissingRequiredOption(Exception): pass

+ class FailureToLoadNetTest(Exception): pass

+ class NoPostProcessor(Exception): pass

+ class InvalidOption(Exception): pass

+ class TaskTimedOut(Exception): pass

+ class InvalidInputFile(Exception): pass

+ +class ReporterException(Exception): + pass + + +class InvalidDestination(ReporterException): + pass + + def get_error(error_key): if error_key == 'test-helpers-key-missing': return CouldNotFindTestHelper diff --git a/ooni/reporter.py b/ooni/reporter.py index 33b4fdf..c5d07ae 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -4,10 +4,13 @@ import json import os import re

-from yaml.representer import * -from yaml.emitter import * -from yaml.serializer import * -from yaml.resolver import * +from datetime import datetime +from contextlib import contextmanager + +from yaml.representer import SafeRepresenter +from yaml.emitter import Emitter +from yaml.serializer import Serializer +from yaml.resolver import Resolver from twisted.python.util import untilConcludes from twisted.internet import defer from twisted.internet.error import ConnectionRefusedError @@ -20,6 +23,7 @@ try: from scapy.packet import Packet except ImportError: log.err("Scapy is not installed.") + class Packet(object): pass

@@ -32,8 +36,7 @@ from ooni.utils.net import BodyReceiver, StringProducer from ooni.settings import config

from ooni.tasks import ReportEntry, ReportTracker -class ReporterException(Exception): - pass +

def createPacketReport(packet_list): """ @@ -45,16 +48,19 @@ def createPacketReport(packet_list): report = [] for packet in packet_list: report.append({'raw_packet': str(packet), - 'summary': str([packet])}) + 'summary': str([packet])}) return report

+ class OSafeRepresenter(SafeRepresenter): + """ This is a custom YAML representer that allows us to represent reports safely. It extends the SafeRepresenter to be able to also represent complex numbers and scapy packet. """ + def represent_data(self, data): """ This is very hackish. There is for sure a better way either by using @@ -82,29 +88,32 @@ class OSafeRepresenter(SafeRepresenter): OSafeRepresenter.add_representer(complex, OSafeRepresenter.represent_complex)

+ class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): + """ This is a modification of the YAML Safe Dumper to use our own Safe Representer that supports complex numbers. """ + def __init__(self, stream, - default_style=None, default_flow_style=None, - canonical=None, indent=None, width=None, - allow_unicode=None, line_break=None, - encoding=None, explicit_start=None, explicit_end=None, - version=None, tags=None): + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + Emitter.__init__(self, stream, canonical=canonical, - indent=indent, width=width, - allow_unicode=allow_unicode, line_break=line_break) + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) Serializer.__init__(self, encoding=encoding, - explicit_start=explicit_start, explicit_end=explicit_end, - version=version, tags=tags) + explicit_start=explicit_start, + explicit_end=explicit_end, + version=version, tags=tags) OSafeRepresenter.__init__(self, default_style=default_style, - default_flow_style=default_flow_style) + default_flow_style=default_flow_style) Resolver.__init__(self)

-class NoTestIDSpecified(Exception): - pass

def safe_dump(data, stream=None, **kw): """ @@ -112,7 +121,9 @@ def safe_dump(data, stream=None, **kw): """ return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw)

+ class OReporter(object): + def __init__(self, test_details): self.testDetails = test_details

@@ -149,10 +160,9 @@ class OReporter(object):

return defer.maybeDeferred(self.writeReportEntry, test_report)

-class InvalidDestination(ReporterException): - pass

class YAMLReporter(OReporter): + """ These are useful functions for reporting to YAML format.

@@ -160,11 +170,13 @@ class YAMLReporter(OReporter): the destination directory of the report

""" - def __init__(self, test_details, report_destination='.', report_filename=None): + + def __init__(self, test_details, report_destination='.', + report_filename=None): self.reportDestination = report_destination

if not os.path.isdir(report_destination): - raise InvalidDestination + raise errors.InvalidDestination

if not report_filename: report_filename = "report-" + \ @@ -216,8 +228,11 @@ class YAMLReporter(OReporter):

self._writeln("###########################################")

- self._writeln("# OONI Probe Report for %s (%s)" % (self.testDetails['test_name'], - self.testDetails['test_version'])) + self._writeln("# OONI Probe Report for %s (%s)" % ( + self.testDetails['test_name'], + self.testDetails['test_version']) + ) + self._writeln("# %s" % otime.prettyDateNow()) self._writeln("###########################################")

@@ -226,13 +241,16 @@ class YAMLReporter(OReporter): def finish(self): self._stream.close()

+ def collector_supported(collector_address): if collector_address.startswith('httpo') \ and (not (config.tor_state or config.tor.socks_port)): return False return True

+ class OONIBReporter(OReporter): + def __init__(self, test_details, collector_address): self.collectorAddress = collector_address self.validateCollectorAddress() @@ -265,7 +283,7 @@ class OONIBReporter(OReporter): url = self.collectorAddress + '/report'

request = {'report_id': self.reportID, - 'content': content} + 'content': content}

log.debug("Updating report with id %s (%s)" % (self.reportID, url)) request_json = json.dumps(request) @@ -274,8 +292,8 @@ class OONIBReporter(OReporter): bodyProducer = StringProducer(json.dumps(request))

try: - response = yield self.agent.request("PUT", url, - bodyProducer=bodyProducer) + yield self.agent.request("PUT", url, + bodyProducer=bodyProducer) except: # XXX we must trap this in the runner and make sure to report the # data later. @@ -298,10 +316,10 @@ class OONIBReporter(OReporter):

if self.collectorAddress.startswith('httpo://'): self.collectorAddress = \ - self.collectorAddress.replace('httpo://', 'http://') - self.agent = SOCKS5Agent(reactor, - proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1', - config.tor.socks_port)) + self.collectorAddress.replace('httpo://', 'http://') + proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', + config.tor.socks_port) + self.agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)

elif self.collectorAddress.startswith('https://'): # XXX add support for securely reporting to HTTPS collectors. @@ -313,15 +331,16 @@ class OONIBReporter(OReporter): content += safe_dump(self.testDetails) content += '...\n'

- request = {'software_name': self.testDetails['software_name'], + request = { + 'software_name': self.testDetails['software_name'], 'software_version': self.testDetails['software_version'], 'probe_asn': self.testDetails['probe_asn'], 'test_name': self.testDetails['test_name'], 'test_version': self.testDetails['test_version'], 'input_hashes': self.testDetails['input_hashes'], # XXX there is a bunch of redundancy in the arguments getting sent - # to the backend. This may need to get changed in the client and the - # backend. + # to the backend. This may need to get changed in the client and + # the backend. 'content': content }

@@ -336,9 +355,11 @@ class OONIBReporter(OReporter):

try: response = yield self.agent.request("POST", url, - bodyProducer=bodyProducer) + bodyProducer=bodyProducer) + except ConnectionRefusedError: - log.err("Connection to reporting backend failed (ConnectionRefusedError)") + log.err("Connection to reporting backend failed " + "(ConnectionRefusedError)") raise errors.OONIBReportCreationError

except errors.HostUnreachable: @@ -366,8 +387,10 @@ class OONIBReporter(OReporter):

if response.code == 406: # XXX make this more strict - log.err("The specified input or nettests cannot be submitted to this collector.") - log.msg("Try running a different test or try reporting to a different collector.") + log.err("The specified input or nettests cannot be submitted to " + "this collector.") + log.msg("Try running a different test or try reporting to a " + "different collector.") raise errors.OONIBReportCreationError

self.reportID = parsed_response['report_id'] @@ -380,10 +403,9 @@ class OONIBReporter(OReporter): log.debug("Closing the report %s" % url) response = yield self.agent.request("POST", str(url))

-class ReportClosed(Exception): - pass

class Report(object): + def __init__(self, reporters, reportEntryManager): """ This is an abstraction layer on top of all the configured reporters. @@ -499,7 +521,8 @@ class Report(object): log.err("Failed to write to %s reporter, giving up..." % reporter) self.reporters.remove(reporter) else: - log.err("Failed to write to (already) removed reporter %s" % reporter) + log.err("Failed to write to (already) removed reporter %s" % + reporter)

# Don't forward the exception unless there are no more reporters if len(self.reporters) == 0: @@ -516,7 +539,6 @@ class Report(object): """ log.err("Failed to open %s reporter, giving up..." % reporter) log.err("Reporter %s failed, removing from report..." % reporter) - #log.exception(failure) if reporter in self.reporters: self.reporters.remove(reporter) # Don't forward the exception unless there are no more reporters