November 2012 - tor-commits - lists.torproject.org

[flashproxy/master] Show a message if running in managed mode but missing env variables.
by dcf＠torproject.org 11 Nov '12

11 Nov '12

commit 1a02a1a0c7b81809a6d9a6fa72e3a012a662a3b2 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Nov 10 15:40:53 2012 -0800 Show a message if running in managed mode but missing env variables. --- flashproxy-client | 21 +++++++++++++++------ 1 files changed, 15 insertions(+), 6 deletions(-) diff --git a/flashproxy-client b/flashproxy-client index fb4b4f7..fded7a0 100755 --- a/flashproxy-client +++ b/flashproxy-client @@ -968,10 +968,6 @@ def pt_enverror(msg): pt_line("ENV-ERROR", msg) sys.exit(1) -def pt_versionerror(msg): - pt_line("VERSION-ERROR", msg) - sys.exit(1) - def pt_smethoderror(msg): pt_line("SMETHOD-ERROR", msg) sys.exit(1) @@ -986,12 +982,25 @@ def pt_get_client_transports(known): return result def pt_setup_managed(): - for ver in os.environ.get("TOR_PT_MANAGED_TRANSPORT_VER", "").split(","): + TOR_PT_MANAGED_TRANSPORT_VER = os.environ.get("TOR_PT_MANAGED_TRANSPORT_VER") + if TOR_PT_MANAGED_TRANSPORT_VER is None: + pt_line("VERSION-ERROR", "no-version") + print >> sys.stderr, """\ + +No TOR_PT_MANAGED_TRANSPORT_VER found in environment. +If you are running flashproxy-client from the command line and not from +a ClientTransportPlugin configuration line, you must use the --external +option.\ +""" + sys.exit(1) + + for ver in TOR_PT_MANAGED_TRANSPORT_VER.split(","): if ver == "1": pt_line("VERSION", ver) break else: - pt_versionerror("no-version") + pt_line("VERSION-ERROR", "no-version") + sys.exit(1) client_transports = pt_get_client_transports(["websocket"]) if not client_transports:

1 0

[flashproxy/master] Add pt line output functions.
by dcf＠torproject.org 11 Nov '12

11 Nov '12

commit 84afc97339a720677fbd6be5045c0e4116348f60 Author: David Fifield <david(a)bamsoftware.com> Date: Mon Oct 22 23:29:16 2012 -0700 Add pt line output functions. --- flashproxy-client | 30 ++++++++++++++++++++++++++++++ 1 files changed, 30 insertions(+), 0 deletions(-) diff --git a/flashproxy-client b/flashproxy-client index 8bc3c3f..6c92c91 100755 --- a/flashproxy-client +++ b/flashproxy-client @@ -931,6 +931,36 @@ def build_register_command(method): else: raise ValueError("Unknown registration method \"%s\"" % method) +def pt_escape(s): + result = [] + for c in s: + if c == "\n": + result.append("\\n") + elif c == "\\": + result.append("\\\\") + elif 0 < ord(c) < 128: + result.append(c) + else: + result.append("\\x%02x" % ord(c)) + return "".join(result) + +def pt_line(keyword, *args): + log(keyword + " " + " ".join(pt_escape(x) for x in args)) + print keyword, " ".join(pt_escape(x) for x in args) + sys.stdout.flush() + +def pt_enverror(msg): + pt_line("ENV-ERROR", msg) + sys.exit(1) + +def pt_versionerror(msg): + pt_line("VERSION-ERROR", msg) + sys.exit(1) + +def pt_smethoderror(msg): + pt_line("SMETHOD-ERROR", msg) + sys.exit(1) + def main(): global remote_listen, local_listen global locals, remotes

1 0

[ooni-probe/master] Refactor reporter Object.
by art＠torproject.org 10 Nov '12

10 Nov '12

commit 572b301710f395620c557dde8ecdef0771ccbc88 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 10 23:32:48 2012 +0100 Refactor reporter Object. * We now have a parent OReporter object that is subclassed by OONIBReporter for remote reporting and YAMLReporter for reporting to YAML format on file system. * Move secure YAML serialization hacks to the hacks module * Do more progress on the implementation of reporting to remote systems --- ooni/reporter.py | 282 ++++++++++++++++++++++++++------------------------- ooni/runner.py | 8 +- ooni/utils/hacks.py | 47 +++++++++ 3 files changed, 197 insertions(+), 140 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 5d29d54..b625603 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -26,6 +26,8 @@ from twisted.internet import defer, reactor from ooni.templates.httpt import BodyReceiver, StringProducer from ooni.utils import otime, log, geodata + +from ooni.utils.hacks import OSafeRepresenter, OSafeDumper from ooni import config try: @@ -38,47 +40,8 @@ except: Packet = object packet = FooClass -class OSafeRepresenter(SafeRepresenter): - """ - This is a custom YAML representer that allows us to represent reports - safely. - It extends the SafeRepresenter to be able to also represent complex numbers - """ - def represent_complex(self, data): - if data.imag == 0.0: - data = u'%r' % data.real - elif data.real == 0.0: - data = u'%rj' % data.imag - elif data.imag > 0: - data = u'%r+%rj' % (data.real, data.imag) - else: - data = u'%r%rj' % (data.real, data.imag) - return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data) - -OSafeRepresenter.add_representer(complex, - OSafeRepresenter.represent_complex) - -class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): - """ - This is a modification of the YAML Safe Dumper to use our own Safe - Representer that supports complex numbers. - """ - def __init__(self, stream, - default_style=None, default_flow_style=None, - canonical=None, indent=None, width=None, - allow_unicode=None, line_break=None, - encoding=None, explicit_start=None, explicit_end=None, - version=None, tags=None): - Emitter.__init__(self, stream, canonical=canonical, - indent=indent, width=width, - allow_unicode=allow_unicode, line_break=line_break) - Serializer.__init__(self, encoding=encoding, - explicit_start=explicit_start, explicit_end=explicit_end, - version=version, tags=tags) - OSafeRepresenter.__init__(self, default_style=default_style, - default_flow_style=default_flow_style) - Resolver.__init__(self) - +class NoTestIDSpecified(Exception): + pass def safe_dump(data, stream=None, **kw): """ @@ -86,42 +49,93 @@ def safe_dump(data, stream=None, **kw): """ return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw) -class OONIBReporter(object): - def __init__(self, backend_url): - from twisted.web.client import Agent - from twisted.internet import reactor +(a)defer.inlineCallbacks +def getTestDetails(options): + client_geodata = {} + + if config.privacy.includeip or \ + config.privacy.includeasn or \ + config.privacy.includecountry or \ + config.privacy.includecity: + log.msg("Running geo IP lookup via check.torproject.org") + client_ip = yield geodata.myIP() + client_location = geodata.IPToLocation(client_ip) + else: + client_ip = "127.0.0.1" + + if config.privacy.includeip: + client_geodata['ip'] = client_ip + else: + client_geodata['ip'] = "127.0.0.1" + + client_geodata['asn'] = None + client_geodata['city'] = None + client_geodata['countrycode'] = None + + if config.privacy.includeasn: + client_geodata['asn'] = client_location['asn'] + + if config.privacy.includecity: + client_geodata['city'] = client_location['city'] + + if config.privacy.includecountry: + client_geodata['countrycode'] = client_location['countrycode'] + + test_details = {'start_time': otime.utcTimeNow(), + 'probe_asn': client_geodata['asn'], + 'probe_cc': client_geodata['countrycode'], + 'probe_ip': client_geodata['ip'], + 'test_name': options['name'], + 'test_version': options['version'], + } + defer.returnValue(test_details) + + +class OReporter(object): + def createReport(options): + """ + Override this with your own logic to implement tests. + """ + raise NotImplemented - self.agent = Agent(reactor) - self.backend_url = backend_url + def writeReportEntry(self, entry): + """ + Takes as input an entry and writes a report for it. + """ + raise NotImplemented - def _newReportCreated(self, data): - log.debug("Got this as result: %s" % data) - return data + def finish(): + pass - def _processResponseBody(self, response, body_cb): - log.debug("Got response %s" % response) - done = defer.Deferred() - response.deliverBody(BodyReceiver(done)) - done.addCallback(body_cb) - return done + def testDone(self, test, test_name): + test_report = dict(test.report) - def newReport(self, test_name, test_version): - url = self.backend_url + '/new' - software_version = '0.0.1' + # XXX the scapy test has an example of how + # to do this properly. + if isinstance(test.input, packet.Packet): + test_input = repr(test.input) + else: + test_input = test.input - request = {'software_name': 'ooni-probe', - 'software_version': software_version, - 'test_name': test_name, 'test_version': test_version, - 'progress': 0} + test_started = test._start_time + test_runtime = test_started - time.time() - log.debug("Creating report via url %s" % url) - bodyProducer = StringProducer(json.dumps(request)) - d = self.agent.request("POST", url, bodyProducer=bodyProducer) - d.addCallback(self._processResponseBody, self._newReportCreated) - return d + report = {'input': test_input, + 'test_name': test_name, + 'test_started': test_started, + 'report': test_report} + self.writeReportEntry(report) + def allDone(self): + log.debug("allDone: Finished running all tests") + self.finish() + try: + reactor.stop() + except: + pass + return None -class YamlReporter(object): +class YAMLReporter(OReporter): """ These are useful functions for reporting to YAML format. """ @@ -145,91 +159,85 @@ class YamlReporter(object): self._write(safe_dump(entry)) self._write('...\n') - def finish(self): - self._stream.close() - -class OReporter(YamlReporter): - """ - This is a reporter factory. It emits new instances of Reports. It is also - responsible for writing the OONI Report headers. - """ - def writeTestsReport(self, tests): - for test in tests.values(): - self.writeReportEntry(test) - @defer.inlineCallbacks - def writeReportHeader(self, options): - self.firstrun = False + def createReport(self, options): self._writeln("###########################################") self._writeln("# OONI Probe Report for %s test" % options['name']) self._writeln("# %s" % otime.prettyDateNow()) self._writeln("###########################################") - client_geodata = {} + test_details = yield getTestDetails(options) - if config.privacy.includeip or \ - config.privacy.includeasn or \ - config.privacy.includecountry or \ - config.privacy.includecity: - log.msg("Running geo IP lookup via check.torproject.org") - client_ip = yield geodata.myIP() - client_location = geodata.IPToLocation(client_ip) - else: - client_ip = "127.0.0.1" + self.writeReportEntry(test_details) - if config.privacy.includeip: - client_geodata['ip'] = client_ip - else: - client_geodata['ip'] = "127.0.0.1" + def finish(self): + self._stream.close() - client_geodata['asn'] = None - client_geodata['city'] = None - client_geodata['countrycode'] = None +class OONIBReporter(object): + def __init__(self, backend_url): + from twisted.web.client import Agent + from twisted.internet import reactor + self.agent = Agent(reactor) + self.backend_url = backend_url - if config.privacy.includeasn: - client_geodata['asn'] = client_location['asn'] + def _newReportCreated(self, data): + log.debug("newReportCreated %s" % data) + return data - if config.privacy.includecity: - client_geodata['city'] = client_location['city'] + def _processResponseBody(self, response, body_cb): + log.debug("processResponseBody %s" % response) + done = defer.Deferred() + response.deliverBody(BodyReceiver(done)) + done.addCallback(body_cb) + return done - if config.privacy.includecountry: - client_geodata['countrycode'] = client_location['countrycode'] + def createReport(self, test_name, + test_version, report_header): + url = self.backend_url + '/new' + software_version = '0.0.1' - test_details = {'start_time': otime.utcTimeNow(), - 'probe_asn': client_geodata['asn'], - 'probe_cc': client_geodata['countrycode'], - 'probe_ip': client_geodata['ip'], - 'test_name': options['name'], - 'test_version': options['version'], - } - self.writeReportEntry(test_details) + request = {'software_name': 'ooni-probe', + 'software_version': software_version, + 'test_name': test_name, + 'test_version': test_version, + 'progress': 0, + 'content': report_header + } + def gotDetails(test_details): + log.debug("Creating report via url %s" % url) + + bodyProducer = StringProducer(json.dumps(request)) + d = self.agent.request("POST", url, + bodyProducer=bodyProducer) + d.addCallback(self._processResponseBody, + self._newReportCreated) + return d + + d = getTestDetails(options) + d.addCallback(gotDetails) + return d - def testDone(self, test, test_name): - test_report = dict(test.report) + def writeReportEntry(self, entry, test_id=None): + if not test_id: + log.err("Write report entry on OONIB requires test id") + raise NoTestIDSpecified - # XXX the scapy test has an example of how - # to do this properly. - if isinstance(test.input, packet.Packet): - test_input = repr(test.input) - else: - test_input = test.input + report = '---\n' + report += safe_dump(entry) + report += '...\n' - test_started = test._start_time - test_runtime = test_started - time.time() + url = self.backend_url + '/new' - report = {'input': test_input, - 'test_name': test_name, - 'test_started': test_started, - 'report': test_report} - self.writeReportEntry(report) + request = {'test_id': test_id, + 'content': report} + + bodyProducer = StringProducer(json.dumps(request)) + d = self.agent.request("PUT", url, + bodyProducer=bodyProducer) + + d.addCallback(self._processResponseBody, + self._newReportCreated) + return d - def allDone(self): - log.debug("allDone: Finished running all tests") - self.finish() - try: - reactor.stop() - except: - pass - return None diff --git a/ooni/runner.py b/ooni/runner.py index 1affb15..07882cf 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -194,7 +194,8 @@ def runTestWithInputUnit(test_class, for test_input in input_unit: log.debug("IU: %s" % test_input) try: - d = runTestWithInput(test_class, test_method, test_input, oreporter) + d = runTestWithInput(test_class, + test_method, test_input, oreporter) except Exception, e: print e log.debug("here y0") @@ -223,10 +224,11 @@ def runTestCases(test_cases, options, test_inputs = [None] reportFile = open(yamloo_filename, 'w+') - oreporter = reporter.OReporter(reportFile) + oreporter = reporter.YAMLReporter(reportFile) + input_unit_factory = InputUnitFactory(test_inputs) - yield oreporter.writeReportHeader(options) + yield oreporter.createReport(options) # This deferred list is a deferred list of deferred lists # it is used to store all the deferreds of the tests that # are run diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py index 4eef366..8912bbb 100644 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@ -8,6 +8,11 @@ # :authors: Arturo Filastò # :licence: see LICENSE +from yaml.representer import * +from yaml.emitter import * +from yaml.serializer import * +from yaml.resolver import * + import copy_reg def patched_reduce_ex(self, proto): @@ -61,3 +66,45 @@ def patched_reduce_ex(self, proto): return copy_reg._reconstructor, args, dict else: return copy_reg._reconstructor, args + +class OSafeRepresenter(SafeRepresenter): + """ + This is a custom YAML representer that allows us to represent reports + safely. + It extends the SafeRepresenter to be able to also represent complex numbers + """ + def represent_complex(self, data): + if data.imag == 0.0: + data = u'%r' % data.real + elif data.real == 0.0: + data = u'%rj' % data.imag + elif data.imag > 0: + data = u'%r+%rj' % (data.real, data.imag) + else: + data = u'%r%rj' % (data.real, data.imag) + return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data) + +OSafeRepresenter.add_representer(complex, + OSafeRepresenter.represent_complex) + +class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): + """ + This is a modification of the YAML Safe Dumper to use our own Safe + Representer that supports complex numbers. + """ + def __init__(self, stream, + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + OSafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style) + Resolver.__init__(self) +

1 0

[ooni-probe/master] Implement basic reporting to flat files containing the report ID.
by art＠torproject.org 10 Nov '12

10 Nov '12

commit 3baf68b4df709c039d8c3d856891afebd6077949 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 11 00:40:50 2012 +0100 Implement basic reporting to flat files containing the report ID. * XXX the location of such files is not configurable, they will leave tons of crap in your cwd. --- oonib/report/api.py | 123 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 files changed, 113 insertions(+), 10 deletions(-) diff --git a/oonib/report/api.py b/oonib/report/api.py index 1a4746a..489cc25 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -9,6 +9,8 @@ This is the async pcap reporting system. It requires the client to have created import random import string import json +import re +from ooni.utils import randomStr from twisted.internet import reactor, defer @@ -21,12 +23,118 @@ backend_version = '0.0.1' def updateReport(report_id, content): print "Report ID: %s" % report_id print "Content: %s" % content - return {'backend_version': backend_version, 'report_id': report_id} -class NewReportHandler(web.RequestHandler): +def generateReportID(): + return randomStr(100) + +class MissingField(Exception): + pass + +class InvalidRequestField(Exception): + pass + +def parseNewReportRequest(request): + """ + Here we parse a new report request. + """ + version_string = re.compile("[0-9A-Za-z_\-\.]+$") + name = re.compile("[a-zA-Z0-9_\- ]+$") + expected_request = {'software_name': name, + 'software_version': version_string, + 'test_name': name, + 'test_version': version_string, + 'progress': re.compile("[0-9]+$") + } + parsed_request = json.loads(request) + for k, regexp in expected_request.items(): + try: + value_to_check = parsed_request[k] + except KeyError: + raise MissingField(k) + print "Matching %s with %s | %s" % (regexp, value_to_check, k) + if re.match(regexp, str(value_to_check)): + continue + else: + raise InvalidRequestField(k) + return parsed_request + +class NewReportHandlerFile(web.RequestHandler): + """ + Responsible for creating and updating reports by writing to flat file. + """ + def post(self): + """ + Creates a new report with the input + + * Request + + {'software_name': 'XXX', + 'software_version': 'XXX', + 'test_name': 'XXX', + 'test_version': 'XXX', + 'progress': 'XXX', + 'content': 'XXX' + } + + Optional: + 'test_helper': 'XXX' + 'client_ip': 'XXX' + + * Response + + {'backend_version': 'XXX', 'report_id': 'XXX'} + + """ + # XXX here we should validate and sanitize the request + try: + report_data = parseNewReportRequest(self.request.body) + except InvalidRequestField, e: + raise web.HTTPError(400, "Invalid Request Field %s" % e) + except MissingField, e: + raise web.HTTPError(400, "Missing Request Field %s" % e) + + print "Parsed this data %s" % report_data + software_name = report_data['software_name'] + software_version = report_data['software_version'] + test_name = report_data['test_name'] + test_version = report_data['test_version'] + content = report_data['content'] + + report_id = generateReportID() + + #report_filename = '_'.join((report_id, + # report_data['software_name'], + # report_data['software_version'], + # report_data['test_name'], + # report_data['test_version'])) + + report_filename = report_id + report_filename += '.yamloo' + + response = {'backend_version': backend_version, + 'report_id': report_id + } + + fp = open(report_filename, 'w+') + fp.write(report_data['content']) + fp.close() + self.write(response) + + def put(self): + """ + Update an already existing report. + + {'report_id': 'XXX', + 'content': 'XXX' + } + """ + pass + +class NewReportHandlerDB(web.RequestHandler): """ - Responsible for creating and updating reports. + Responsible for creating and updating reports via database. + XXX this is not yet fully implemented. """ @web.asynchronous @@ -55,16 +163,11 @@ class NewReportHandler(web.RequestHandler): """ parsed_request = json.loads(self.request.body) - # XXX here we should validate and sanitize the request - report_data = parsed_request - new_report = models.Report() - print "Got %s as request" % parsed_request result = yield new_report.new(report_data) - self.write(result) self.finish() @@ -85,7 +188,7 @@ class PCAPReportHandler(web.RequestHandler): def post(self): pass -spec = [(r"/report/new", NewReportHandler), +spec = [(r"/report/new", NewReportHandlerFile), (r"/report/pcap", PCAPReportHandler)] -reportingBackend = web.Application(spec) +reportingBackend = web.Application(spec, debug=True)

1 0

[translation/vidalia_alpha_completed] Update translations for vidalia_alpha_completed
by translation＠torproject.org 10 Nov '12

10 Nov '12

commit 006a6df69cb1983f263006b6945decbe6076076d Author: Translation commit bot <translation(a)torproject.org> Date: Sat Nov 10 21:45:21 2012 +0000 Update translations for vidalia_alpha_completed --- es/vidalia_es.po | 26 +++++++++++++------------- 1 files changed, 13 insertions(+), 13 deletions(-) diff --git a/es/vidalia_es.po b/es/vidalia_es.po index 43f22c1..7f3fcdb 100644 --- a/es/vidalia_es.po +++ b/es/vidalia_es.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-03-21 17:46+0000\n" -"PO-Revision-Date: 2012-11-10 21:14+0000\n" +"PO-Revision-Date: 2012-11-10 21:33+0000\n" "Last-Translator: strel <strelnic(a)gmail.com>\n" "Language-Team: Spanish (http://www.transifex.com/projects/p/torproject/language/es/)\n" "MIME-Version: 1.0\n" @@ -1215,7 +1215,7 @@ msgstr "Zambia" msgctxt "CountryInfo" msgid "Zimbabwe" -msgstr "Zimbabwe" +msgstr "Zimbabue" msgctxt "CountryInfo" msgid "Zaire" @@ -1239,7 +1239,7 @@ msgstr "Bahrein" msgctxt "CountryInfo" msgid "Benin" -msgstr "Benin" +msgstr "Benín" msgctxt "CountryInfo" msgid "Ethiopia" @@ -1247,7 +1247,7 @@ msgstr "Etiopía" msgctxt "CountryInfo" msgid "Fiji" -msgstr "Fiji" +msgstr "Fiyi" msgctxt "CountryInfo" msgid "Finland" @@ -1307,11 +1307,11 @@ msgstr "Maldivas" msgctxt "CountryInfo" msgid "Mexico" -msgstr "Méjico" +msgstr "México" msgctxt "CountryInfo" msgid "Myanmar" -msgstr "Birmania" +msgstr "Myanmar (Birmania)" msgctxt "CountryInfo" msgid "Taiwan" @@ -1319,11 +1319,11 @@ msgstr "Taiwán" msgctxt "CrashReportDialog" msgid "Submit a Crash Report" -msgstr "Presentar un informe de error" +msgstr "Enviar un Informe de Error" msgctxt "CrashReportDialog" msgid "Vidalia encountered an error and needed to close" -msgstr "Vidalia ha encontrado un error y es necesario para cerrar" +msgstr "Vidalia ha encontrado un error y fue necesario cerrar" msgctxt "CrashReportDialog" msgid "Restart Vidalia" @@ -1335,13 +1335,13 @@ msgstr "No reiniciar" msgctxt "CrashReportDialog" msgid "Unable to restart Vidalia" -msgstr "No se puede reiniciar Vidalia" +msgstr "No se pudo reiniciar Vidalia" msgctxt "CrashReportDialog" msgid "" "We were unable to automatically restart Vidalia. Please restart Vidalia " "manually." -msgstr "No hemos podido reiniciar automáticamente Vidalia. Por favor, reinicia manualmente." +msgstr "No fue posible reiniciar automáticamente Vidalia. Por favor, reinicie manualmente." msgctxt "CrashReportDialog" msgid "Please fill a ticket in:" @@ -1392,7 +1392,7 @@ msgstr "Seleccione la ruta de Tor" msgctxt "GeneralPage" msgid "You must specify the name of your Tor executable." -msgstr "Debe especificar el nombre del ejecutable de Tor." +msgstr "Debe especificar el nombre de su ejecutable de Tor." msgctxt "GeneralPage" msgid "Start Vidalia when my system starts" @@ -1440,11 +1440,11 @@ msgstr "%1 GB" msgctxt "GraphFrame" msgid "Recv:" -msgstr "Recib." +msgstr "Reci:" msgctxt "GraphFrame" msgid "Sent:" -msgstr "Enviados:" +msgstr "Envi:" msgctxt "HelpBrowser" msgid "Supplied XML file is not a valid Contents document."

1 0

[translation/vidalia_alpha] Update translations for vidalia_alpha
by translation＠torproject.org 10 Nov '12

10 Nov '12

commit 6b787ac9f6ae4d3287ae55bf9a867384f07eedaa Author: Translation commit bot <translation(a)torproject.org> Date: Sat Nov 10 21:45:18 2012 +0000 Update translations for vidalia_alpha --- es/vidalia_es.po | 26 +++++++++++++------------- 1 files changed, 13 insertions(+), 13 deletions(-) diff --git a/es/vidalia_es.po b/es/vidalia_es.po index 43f22c1..7f3fcdb 100644 --- a/es/vidalia_es.po +++ b/es/vidalia_es.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-03-21 17:46+0000\n" -"PO-Revision-Date: 2012-11-10 21:14+0000\n" +"PO-Revision-Date: 2012-11-10 21:33+0000\n" "Last-Translator: strel <strelnic(a)gmail.com>\n" "Language-Team: Spanish (http://www.transifex.com/projects/p/torproject/language/es/)\n" "MIME-Version: 1.0\n" @@ -1215,7 +1215,7 @@ msgstr "Zambia" msgctxt "CountryInfo" msgid "Zimbabwe" -msgstr "Zimbabwe" +msgstr "Zimbabue" msgctxt "CountryInfo" msgid "Zaire" @@ -1239,7 +1239,7 @@ msgstr "Bahrein" msgctxt "CountryInfo" msgid "Benin" -msgstr "Benin" +msgstr "Benín" msgctxt "CountryInfo" msgid "Ethiopia" @@ -1247,7 +1247,7 @@ msgstr "Etiopía" msgctxt "CountryInfo" msgid "Fiji" -msgstr "Fiji" +msgstr "Fiyi" msgctxt "CountryInfo" msgid "Finland" @@ -1307,11 +1307,11 @@ msgstr "Maldivas" msgctxt "CountryInfo" msgid "Mexico" -msgstr "Méjico" +msgstr "México" msgctxt "CountryInfo" msgid "Myanmar" -msgstr "Birmania" +msgstr "Myanmar (Birmania)" msgctxt "CountryInfo" msgid "Taiwan" @@ -1319,11 +1319,11 @@ msgstr "Taiwán" msgctxt "CrashReportDialog" msgid "Submit a Crash Report" -msgstr "Presentar un informe de error" +msgstr "Enviar un Informe de Error" msgctxt "CrashReportDialog" msgid "Vidalia encountered an error and needed to close" -msgstr "Vidalia ha encontrado un error y es necesario para cerrar" +msgstr "Vidalia ha encontrado un error y fue necesario cerrar" msgctxt "CrashReportDialog" msgid "Restart Vidalia" @@ -1335,13 +1335,13 @@ msgstr "No reiniciar" msgctxt "CrashReportDialog" msgid "Unable to restart Vidalia" -msgstr "No se puede reiniciar Vidalia" +msgstr "No se pudo reiniciar Vidalia" msgctxt "CrashReportDialog" msgid "" "We were unable to automatically restart Vidalia. Please restart Vidalia " "manually." -msgstr "No hemos podido reiniciar automáticamente Vidalia. Por favor, reinicia manualmente." +msgstr "No fue posible reiniciar automáticamente Vidalia. Por favor, reinicie manualmente." msgctxt "CrashReportDialog" msgid "Please fill a ticket in:" @@ -1392,7 +1392,7 @@ msgstr "Seleccione la ruta de Tor" msgctxt "GeneralPage" msgid "You must specify the name of your Tor executable." -msgstr "Debe especificar el nombre del ejecutable de Tor." +msgstr "Debe especificar el nombre de su ejecutable de Tor." msgctxt "GeneralPage" msgid "Start Vidalia when my system starts" @@ -1440,11 +1440,11 @@ msgstr "%1 GB" msgctxt "GraphFrame" msgid "Recv:" -msgstr "Recib." +msgstr "Reci:" msgctxt "GraphFrame" msgid "Sent:" -msgstr "Enviados:" +msgstr "Envi:" msgctxt "HelpBrowser" msgid "Supplied XML file is not a valid Contents document."

1 0

[translation/vidalia_alpha_completed] Update translations for vidalia_alpha_completed
by translation＠torproject.org 10 Nov '12

10 Nov '12

commit 10840a851dc2caaa28ac67a83177706cbc19e860 Author: Translation commit bot <translation(a)torproject.org> Date: Sat Nov 10 21:15:23 2012 +0000 Update translations for vidalia_alpha_completed --- es/vidalia_es.po | 26 +++++++++++++------------- 1 files changed, 13 insertions(+), 13 deletions(-) diff --git a/es/vidalia_es.po b/es/vidalia_es.po index 1d2e38c..43f22c1 100644 --- a/es/vidalia_es.po +++ b/es/vidalia_es.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-03-21 17:46+0000\n" -"PO-Revision-Date: 2012-11-10 20:14+0000\n" +"PO-Revision-Date: 2012-11-10 21:14+0000\n" "Last-Translator: strel <strelnic(a)gmail.com>\n" "Language-Team: Spanish (http://www.transifex.com/projects/p/torproject/language/es/)\n" "MIME-Version: 1.0\n" @@ -803,7 +803,7 @@ msgstr "Jordania" msgctxt "CountryInfo" msgid "Kazakhstan" -msgstr "Kazajstán" +msgstr "Kazajistán" msgctxt "CountryInfo" msgid "Kenya" @@ -819,7 +819,7 @@ msgstr "Kuwait" msgctxt "CountryInfo" msgid "Kyrgyzstan" -msgstr "Kirguistán" +msgstr "Kirguizistán" msgctxt "CountryInfo" msgid "Laos" @@ -835,7 +835,7 @@ msgstr "Líbano" msgctxt "CountryInfo" msgid "Lesotho" -msgstr "Lesotho" +msgstr "Lesoto" msgctxt "CountryInfo" msgid "Liberia" @@ -871,7 +871,7 @@ msgstr "Malasia" msgctxt "CountryInfo" msgid "Mali" -msgstr "Mali" +msgstr "Malí" msgctxt "CountryInfo" msgid "Malta" @@ -947,7 +947,7 @@ msgstr "Níger" msgctxt "CountryInfo" msgid "Nigeria" -msgstr "Níger" +msgstr "Nigeria" msgctxt "CountryInfo" msgid "Norway" @@ -975,7 +975,7 @@ msgstr "Panamá" msgctxt "CountryInfo" msgid "Papua New Guinea" -msgstr "Papúa y Nueva Guinea" +msgstr "Papúa Nueva Guinea" msgctxt "CountryInfo" msgid "Paraguay" @@ -999,7 +999,7 @@ msgstr "Portugal" msgctxt "CountryInfo" msgid "Qatar" -msgstr "Qatar" +msgstr "Catar (Qatar)" msgctxt "CountryInfo" msgid "Romania" @@ -1023,7 +1023,7 @@ msgstr "Santa Lucía" msgctxt "CountryInfo" msgid "Saint Vincent & the Grenadines" -msgstr "San Vicente y las Granadinas" +msgstr "San Vicente y Granadinas" msgctxt "CountryInfo" msgid "Samoa" @@ -1035,7 +1035,7 @@ msgstr "San Marino" msgctxt "CountryInfo" msgid "Sao Tome & Principe" -msgstr "Sao Tomé y Príncipe" +msgstr "Santo Tomé y Príncipe" msgctxt "CountryInfo" msgid "Saudi Arabia" @@ -1115,7 +1115,7 @@ msgstr "Siria" msgctxt "CountryInfo" msgid "Tajikistan" -msgstr "Tazhdikistán" +msgstr "Tayikistán" msgctxt "CountryInfo" msgid "Tanzania" @@ -1127,7 +1127,7 @@ msgstr "Tailandia" msgctxt "CountryInfo" msgid "Timor-Leste (East Timor)" -msgstr "Timor Leste" +msgstr "Timor Oriental" msgctxt "CountryInfo" msgid "Togo" @@ -1207,7 +1207,7 @@ msgstr "Sahara Occidental" msgctxt "CountryInfo" msgid "Yemen" -msgstr "Yemén" +msgstr "Yemen" msgctxt "CountryInfo" msgid "Zambia"

1 0

[translation/vidalia_alpha] Update translations for vidalia_alpha
by translation＠torproject.org 10 Nov '12

10 Nov '12

commit 5e4204dced086a6c2e4ea78da9124b33ff21b13e Author: Translation commit bot <translation(a)torproject.org> Date: Sat Nov 10 21:15:20 2012 +0000 Update translations for vidalia_alpha --- es/vidalia_es.po | 26 +++++++++++++------------- 1 files changed, 13 insertions(+), 13 deletions(-) diff --git a/es/vidalia_es.po b/es/vidalia_es.po index 1d2e38c..43f22c1 100644 --- a/es/vidalia_es.po +++ b/es/vidalia_es.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-03-21 17:46+0000\n" -"PO-Revision-Date: 2012-11-10 20:14+0000\n" +"PO-Revision-Date: 2012-11-10 21:14+0000\n" "Last-Translator: strel <strelnic(a)gmail.com>\n" "Language-Team: Spanish (http://www.transifex.com/projects/p/torproject/language/es/)\n" "MIME-Version: 1.0\n" @@ -803,7 +803,7 @@ msgstr "Jordania" msgctxt "CountryInfo" msgid "Kazakhstan" -msgstr "Kazajstán" +msgstr "Kazajistán" msgctxt "CountryInfo" msgid "Kenya" @@ -819,7 +819,7 @@ msgstr "Kuwait" msgctxt "CountryInfo" msgid "Kyrgyzstan" -msgstr "Kirguistán" +msgstr "Kirguizistán" msgctxt "CountryInfo" msgid "Laos" @@ -835,7 +835,7 @@ msgstr "Líbano" msgctxt "CountryInfo" msgid "Lesotho" -msgstr "Lesotho" +msgstr "Lesoto" msgctxt "CountryInfo" msgid "Liberia" @@ -871,7 +871,7 @@ msgstr "Malasia" msgctxt "CountryInfo" msgid "Mali" -msgstr "Mali" +msgstr "Malí" msgctxt "CountryInfo" msgid "Malta" @@ -947,7 +947,7 @@ msgstr "Níger" msgctxt "CountryInfo" msgid "Nigeria" -msgstr "Níger" +msgstr "Nigeria" msgctxt "CountryInfo" msgid "Norway" @@ -975,7 +975,7 @@ msgstr "Panamá" msgctxt "CountryInfo" msgid "Papua New Guinea" -msgstr "Papúa y Nueva Guinea" +msgstr "Papúa Nueva Guinea" msgctxt "CountryInfo" msgid "Paraguay" @@ -999,7 +999,7 @@ msgstr "Portugal" msgctxt "CountryInfo" msgid "Qatar" -msgstr "Qatar" +msgstr "Catar (Qatar)" msgctxt "CountryInfo" msgid "Romania" @@ -1023,7 +1023,7 @@ msgstr "Santa Lucía" msgctxt "CountryInfo" msgid "Saint Vincent & the Grenadines" -msgstr "San Vicente y las Granadinas" +msgstr "San Vicente y Granadinas" msgctxt "CountryInfo" msgid "Samoa" @@ -1035,7 +1035,7 @@ msgstr "San Marino" msgctxt "CountryInfo" msgid "Sao Tome & Principe" -msgstr "Sao Tomé y Príncipe" +msgstr "Santo Tomé y Príncipe" msgctxt "CountryInfo" msgid "Saudi Arabia" @@ -1115,7 +1115,7 @@ msgstr "Siria" msgctxt "CountryInfo" msgid "Tajikistan" -msgstr "Tazhdikistán" +msgstr "Tayikistán" msgctxt "CountryInfo" msgid "Tanzania" @@ -1127,7 +1127,7 @@ msgstr "Tailandia" msgctxt "CountryInfo" msgid "Timor-Leste (East Timor)" -msgstr "Timor Leste" +msgstr "Timor Oriental" msgctxt "CountryInfo" msgid "Togo" @@ -1207,7 +1207,7 @@ msgstr "Sahara Occidental" msgctxt "CountryInfo" msgid "Yemen" -msgstr "Yemén" +msgstr "Yemen" msgctxt "CountryInfo" msgid "Zambia"

1 0

[ooni-probe/master] Improve HTTP Host test to work with the HTTP Requests test backend
by art＠torproject.org 10 Nov '12

10 Nov '12

commit b142d1fa8aeda530e0b155e5fc5aae377f523f76 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 10 21:11:54 2012 +0100 Improve HTTP Host test to work with the HTTP Requests test backend --- nettests/core/http_host.py | 63 +++++++++++++++++++++++++++++++++++++++----- ooni/templates/httpt.py | 15 +++++++--- 2 files changed, 66 insertions(+), 12 deletions(-) diff --git a/nettests/core/http_host.py b/nettests/core/http_host.py index b87594d..0e73f82 100644 --- a/nettests/core/http_host.py +++ b/nettests/core/http_host.py @@ -9,14 +9,17 @@ # :authors: Arturo Filastò # :licence: see LICENSE +import json from twisted.python import usage + +from ooni.utils import log from ooni.templates import httpt class UsageOptions(usage.Options): - optParameters = [ - ['backend', 'b', 'http://ooni.nu/test/', 'Test backend to use'] - ] - + optParameters = [['backend', 'b', 'http://127.0.0.1:1234', + 'URL of the test backend to use'], + ['content', 'c', None, + 'The file to read from containing the content of a block page']] class HTTPHost(httpt.HTTPTest): """ @@ -25,11 +28,13 @@ class HTTPHost(httpt.HTTPTest): """ name = "HTTP Host" author = "Arturo Filastò" - version = 0.1 + version = "0.2" usageOptions = UsageOptions - inputFile = ['urls', 'f', None, 'Urls file'] + inputFile = ['file', 'f', None, 'List of hostnames to test for censorship'] + + requiredOptions = ['backend'] def test_send_host_header(self): """ @@ -42,14 +47,58 @@ class HTTPHost(httpt.HTTPTest): headers["Host"] = [self.input] return self.doRequest(self.localOptions['backend'], headers=headers) + def check_for_censorship(self, body): + """ + If we have specified what a censorship page looks like here we will + check if the page we are looking at matches it. + + XXX this is not tested, though it is basically what was used to detect + censorship in the palestine case. + """ + if self.localOptions['content']: + self.report['censored'] = True + + censorship_page = open(self.localOptions['content']) + response_page = iter(body.split("\n")) + + for censorship_line in censorship_page.xreadlines(): + response_line = response_page.next() + if response_line != censorship_line: + self.report['censored'] = False + break + + censorship_page.close() + def processResponseBody(self, body): """ XXX this is to be filled in with either a domclass based classified or with a rule that will allow to detect that the body of the result is that of a censored site. """ - if 'not censored' in body: + # If we don't see a json array we know that something is wrong for + # sure + if not body.startswith("{"): + self.report['transparent_http_proxy'] = True + self.check_for_censorship(body) + return + try: + content = json.loads(body) + except: + log.debug("The json does not parse, this is not what we expected") + self.report['trans_http_proxy'] = True + self.check_for_censorship(body) + return + + # We base the determination of the presence of a transparent HTTP + # proxy on the basis of the response containing the json that is to be + # returned by a HTTP Request Test Helper + if 'request_method' in content and \ + 'request_uri' in content and \ + 'request_headers' in content: + log.debug("Found the keys I expected in %s" % content) self.report['trans_http_proxy'] = False else: + log.debug("Did not find the keys I expected in %s" % content) self.report['trans_http_proxy'] = True + self.check_for_censorship(body) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 96b7cc8..4f3be3d 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -14,6 +14,7 @@ from twisted.internet.ssl import ClientContextFactory from twisted.web.client import Agent from twisted.internet import reactor +from twisted.internet.error import ConnectionRefusedError from twisted.web._newclient import Request from twisted.web.http_headers import Headers @@ -29,9 +30,11 @@ class HTTPTest(NetTestCase): The main functions to look at are processResponseBody and processResponseHeader that are invoked once the headers have been received and once the request body has been received. + + XXX all of this requires some refactoring. """ name = "HTTP Test" - version = 0.1 + version = "0.1.1" randomizeUA = True followRedirects = False @@ -127,9 +130,10 @@ class HTTPTest(NetTestCase): d = self.build_request(url, method, headers, body) - def errback(data): - log.err("Error in test %s" % data) - self.report["error"] = data + def errback(failure): + failure.trap(ConnectionRefusedError) + log.err("Connection refused. The backend may be down") + self.report["failure"] = str(failure.value) def finished(data): return @@ -186,7 +190,8 @@ class HTTPTest(NetTestCase): finished = defer.Deferred() response.deliverBody(BodyReceiver(finished)) - finished.addCallback(self._processResponseBody, body_processor) + finished.addCallback(self._processResponseBody, + body_processor) return finished

1 0

[ooni-probe/master] Do some cleaning up
by art＠torproject.org 10 Nov '12

10 Nov '12

commit bae19b558fcfa618778dea948e292c393ab49ed8 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Nov 10 21:40:22 2012 +0100 Do some cleaning up * Remove Readme.md file * Update TODO with lists of tickets * Remove no longer used unittests --- README.md | 108 ++++++++++++++---------------------------- TODO | 105 +++------------------------------------- tests/assets/ipports.txt | 2 - tests/assets/urllist.txt | 3 - tests/legacy/test_plugins.py | 40 --------------- tests/legacy/test_tests.py | 45 ----------------- tests/test_worker.py | 29 ----------- 7 files changed, 43 insertions(+), 289 deletions(-) diff --git a/README.md b/README.md index 4837275..137386a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ooni-probe - Open Observatory of Network Interference +# ooniprobe - Open Observatory of Network Interference "The Net interprets censorship as damage and routes around it." - John Gilmore; TIME magazine (6 December 1993) @@ -15,103 +15,65 @@ To run OONI-probe without having to install it you must tell python that it can import modules from the root of ooni-probe, as well as initialize the included submodules. -From the root directory of the repo (.../ooni-probe/), initialize the submodules by doing: +## Getting started - $ git submodule init && git submodule update +Requirements: -Next, you will need to tell Python that OONI is part of its path: + * Git: http://git-scm.com/book/en/Getting-Started-Installing-Git + * Python >= 2.6: http://www.python.org/download/releases/ + * pip: http://www.pip-installer.org/en/latest/ - $ export PYTHONPATH=$PYTHONPATH:`pwd` +On debian based systems these can be installed with: -Then to see what tests are available: + apt-get install git-core python python-pip python-dev - $ cd ooni - $ python ooniprobe.py +The python dependencies required for running ooniprobe are: -If you see some errors see INSTALL to install the missing dependencies. + * Twisted + * Scapy >= 2.2.0 + * txtorcon -To list the help for a specific test: +They can be installed from the requirements.txt with: - $ python ooniprobe.py httpt --help + pip install -r requirements.txt -## Virtualenv way (Recommended) +You are highly recommended to do so from inside of a virtual environment, since +pip does not download the packages via SSL and you will need to install it +system wide. - virtualenv2 ENV/ - source ENV/bin/activate - pip install twisted Scapy pyyaml +This will require you to have installed virtualenv. -The setup.py script of pyOpenSSL has some issues and it is therefore -recommended to install it system wide with: + apt-get install python-virtualenv - sudo pip install pyOpenSSL +To create a new virtual environment do -Note: be sure to do this over a secure network since pip does not by default -use SSL... + virtualenv env -To install the most up to date scapy version (requires mercurial): +Then install OONI with: - pip install hg+http://hg.secdev.org/scapy + pip install -r requirements.txt +## Running some tests -# More details +To see the possible command line options run: -With the belief that unfettered access to information is a intrinsic human right, -OONI seeks to observe levels of surveillance, censorship, and network discrimination -in order for people worldwide to have a clearer understanding of the ways in -which their access to information is filtered. + ./bin/ooniprobe --help -The end goal of OONI is to collect data which can show an accurate -topology of network interference and censorship. Through this topology, it will be -possible to see what the internet looks like from nearly any location, including -what sites are censored, or have been tampered with, and by whom. We're calling -it filternet. +For interesting tests to run look in the nettests/core/ directory. -OONI uses open methodologies and the data will be provided in raw -format to allow any researcher to indipendently draw their conclusions -from the results OONI tests. +To run a test you can do so with: -There are currently projects aimed at measuring censorship in one -way or another but they either use non open methodologies or their -tools are not open sources. OONI aims at filling up this gap by -creating the first open source framework for developing network -tests and collecting data on censorship. + ./bin/ooniprobe -o report_file_name path/to/test.py -OONI revolves around three major concepts: Assets, Tests and -Reports. +Normally tests take options, you can see them with: -## Assets + ./bin/ooniprobe -o report_file_name path/to/test.py --help -Assets are the inputs used inside Tests to detect censorship events. -These can be URL lists, keywords, ip addresses, packets or any kind -of set of data. -In the python specific implementation this is represented as a python -iterable object. This means that the Testing framework will be able -to iterate through every element in the Asset. +## Configuration -## Tests +By default ooniprobe will not include personal identifying information in the +test result, nor create a pcap file. This behavior can be personalized by +editing your ooniprobe.conf configuration file. -This is the core of OONI. These are the actual tests that will be run -using as input (if an input is required) the Assets. -Tests can be summarized as an experiment and a control. The control -represents the expected result and the experiment is the network operation -being performed on the live network. If the experiment does not match up -with the control then a censorship event had occured. -OONI probe provides some useful functionality to the application developer -that may be useful when developing censorship detection tests. For example -it is possible to make a request over the Tor network easily or use a fast -and flexible non-blocking HTTP client implementation. - -## Reports - -This is the data that is collected from the test. OONI probe provides a -flexible means of storing results and uploading this data to a remote -server or a flat file. - -The Test developer should include in the report as much data as possible -and can contain raw packet dumps as well as structured synthetic results. - -In future on top of ooni-probe Reports it will be possible to develop -flexible post-processing tools to allow data-visualization guru's to -properly visualize and contextualize the resulting data. diff --git a/TODO b/TODO index b8524f3..6a989fd 100644 --- a/TODO +++ b/TODO @@ -1,108 +1,19 @@ -This is a list of things to be done on ooni-probe. +You can find things that need to be fixed by running from the root of the project: -Once you have completed something you should add a brief note to this file -stating what you have done under the item. If you discover needed tasks, feel -free to add them, but also keep in mind that OONI is mostly using the Tor Trac -instance, and the main ticket for OONI which all tests should be organized -under is here: + grep -rn XXX * - https://trac.torproject.org/projects/tor/ticket/5869 +# The full list of tickets related to OONI can be found by doing this query on trac: -New things to test ------------------- + https://trac.torproject.org/projects/tor/query?status=accepted&status=assig… -* Some of the tests still give errors. This is top priority. +# Things related to the OONIB can be found with this query: -* The submit-patch script is still a bit janky, and could use - some sprucing up. + https://trac.torproject.org/projects/tor/query?status=accepted&status=assig… -* The Makefile that I made to serve a quick check for whether or not the tests - are broken could use some more tests added to it. One thing is that some - tests take certain types of input files, others take none at all, and - nowhere is this presented to someone trying to run a test. So, the informing - users/testers bit can be worked on, and the testing. Obviously we're going - to want something more robust that a 20 LOC Makefile pretty fast. +# Tests that still need to be implemented can be found with: -Finalization of API design --------------------------- + https://trac.torproject.org/projects/tor/query?status=accepted&status=assig… -* The nettest.TestCase should have an interface. - I know that there is a push away from using zope.interfaces, but I think - it is actually *highly* necessary for ensuring that subclasses implement - the required functions, and also that they do not improperly override - necessary functions, for them to run. - Personally, I am quite annoyed when I subclass a class from Twisted and - override a public method, and it breaks things (when nothing in their - documentation informed me that it would break things) and I have to spend - half an hour digging through their code to figure out precisely what is - needed externally from the function I'm overriding. Others should not have - to do this with our code. -* The nettest.TestCase should have a twisted.python.usage.Options subclass and - interface as well, even if the instantiation of that subclass is handled by - the ooni.oonicli or the ooni.runner. There is more functionality to - usage.Options that we should expose than merely "optParameters", for - instance the "coerceOptions" parameter validation methods, or the - "postOptions" configuration. - -New things to develop ---------------------- - -These are either components specific to the new refactor of ooni -or that we haven't yet figured out how they should work. - -* Finish implementing the backend collection code. - - o PCAP READER/WRITER: - This should be quite simple...see scapy.all.wrpcap and - scapy.all.rdpcap. However, we have been warned by other projects that - this does *not* scale well. For example, see: - https://github.com/isislovecruft/switzerland/blob/master/switzerland/client… - Which is a circular ring buffer specifically for libpcap, to avoid kernel - buffer overflows due to a high number of incoming packets. I expect this - to only be an issue on substantially high-bandwidth nodes...though that - is what we'll be dealing with when we deploy on Mlab. - - o PCAP UPLOADER: - This also sounds simple, and is, until you begin to deal with things like - persistence. What we really need is rsync, written in python, or at least - some cross-platform implementation. I (Isis speaking) am the current - maintainer of pyrsync, BUT DO NOT USE PYRSYNC. It is only an - implementation of the rsync *algorithm* for diffs, it is not rsync the - program. Also, it is BROKEN AND I DO NOT MAINTAIN IT. If you want to - maintain it, please take it off my hands. - -* Useability: - - o UNITTESTS. Pronto. - - o DOCUMENTATION. If you found something that confused you, or still - confuses you, and you couldn't find the answer within fifteen seconds, - then that thing is not well documented. Make it better, or at least mark - it with an "XXX document me!" tag. - -* Persistence: - - o We need some type of scheduler/cron thing which will background the tests - so that they don't take up a terminal, and can be configured to run - certain tests at timed intervals. - - o The Reporter will probably need to be updated to handle knowing when *a - test* has completed, but that the scheduler is still running. - -Migrate code from old ---------------------- - -Migrate all the interesting parts of the old code to the new. This is mostly -finished, but there still are things in the /old-to-be-ported directory which -might be of use. At this point, because we have gone through several version -of the API design, many of them are entirely unusable, and merely the general -idea remains. - -It's important to make the new code asych and based on Twisted. It should -respect the design goals of the new ooni-probe model. Also, importing new, -non-standard libraries should be discussed first, if the new test is to be -used in the core of OONI (packaging scapy and twisted already makes our -codebase quite large). diff --git a/tests/assets/ipports.txt b/tests/assets/ipports.txt deleted file mode 100644 index ade757f..0000000 --- a/tests/assets/ipports.txt +++ /dev/null @@ -1,2 +0,0 @@ -127.0.0.1:80 -8.8.8.8:53 diff --git a/tests/assets/urllist.txt b/tests/assets/urllist.txt deleted file mode 100644 index dc38e22..0000000 --- a/tests/assets/urllist.txt +++ /dev/null @@ -1,3 +0,0 @@ -http://google.com/ -http://127.0.0.1/ -http://dio.it/ diff --git a/tests/legacy/test_plugins.py b/tests/legacy/test_plugins.py deleted file mode 100644 index 26bee51..0000000 --- a/tests/legacy/test_plugins.py +++ /dev/null @@ -1,40 +0,0 @@ -from twisted.internet import defer, reactor -from twisted.trial import unittest - -from ooni.ooniprobe import retrieve_plugoo, runTest, Options -from ooni.plugoo import work, tests - -def asset_file(filename): - import os - file_dir = os.path.normpath(os.path.join(__file__, '..')) - return os.path.join(file_dir, 'assets', filename) - -class PluginsTestCase(unittest.TestCase): - def test_plugin_blocking(self): - suboptions = {'asset': asset_file('urllist.txt')} - runTest('blocking', suboptions, Options(), reactor) - return - - def test_plugin_tcpconnect(self): - suboptions = {'asset': asset_file('ipports.txt')} - runTest('tcpconnect', suboptions, Options(), reactor) - return - - - def test_plugin_captivep(self): - runTest('blocking', None, Options(), reactor) - return - - - def test_plugin_httphost(self): - suboptions = {'asset': asset_file('urllist.txt')} - runTest('httphost', suboptions, Options(), reactor) - return - - - def test_plugin_httpt(self): - suboptions = {'urls': asset_file('urllist.txt')} - runTest('httpt', suboptions, Options(), reactor) - return - - diff --git a/tests/legacy/test_tests.py b/tests/legacy/test_tests.py deleted file mode 100644 index dc89b98..0000000 --- a/tests/legacy/test_tests.py +++ /dev/null @@ -1,45 +0,0 @@ -from twisted.internet import defer -from twisted.trial import unittest - -from ooni.plugoo import work, tests - -class TestsTestCase(unittest.TestCase): - def setUp(self): - class dummyReport: - def __call__(self, *args, **kw): - pass - self.dummyreport = dummyReport() - self.callbackResults = None - self.errbackResults = None - - def _callback(self, *args, **kw): - #print args, kw - self.callbackResults = args, kw - - def _errback(self, *args, **kw): - pass - - @defer.inlineCallbacks - def test_fallThrough(self): - """ - This tests to make sure that what is returned from the experiment - method falls all the way through to control and finish. - """ - test_dict = {"hello": "world"} - class DummyTest(tests.OONITest): - blocking = False - def experiment(self, args): - def bla(a): - print a - return test_dict - d2 = defer.Deferred() - d2.addCallback(bla) - from twisted.internet import reactor - reactor.callLater(0.1, d2.callback, None) - return d2 - - test = DummyTest(None, None, self.dummyreport) - yield test.startTest(None).addCallback(self._callback) - self.assertEqual(self.callbackResults[0][0]['return_value'], test_dict) - return - diff --git a/tests/test_worker.py b/tests/test_worker.py deleted file mode 100644 index 3222f65..0000000 --- a/tests/test_worker.py +++ /dev/null @@ -1,29 +0,0 @@ -from twisted.trial import unittest - -from ooni.plugoo import work, tests - -class WorkerTestCase(unittest.TestCase): - def testWorkGenerator(self): - class DummyTest: - assets = {} - dummytest = DummyTest() - asset = [] - for i in range(10): - asset.append(i) - dummytest.assets['asset'] = asset - wgen = work.WorkGenerator(dummytest) - - for j, x in enumerate(wgen): - pass - self.assertEqual(i, j) - - def testNoAssets(self): - class DummyTest: - assets = {'asset': None} - dummytest = DummyTest() - wgen = work.WorkGenerator(dummytest) - i = 0 - for j in wgen: - i += 1 - self.assertEqual(i, 1) -

1 0