- tor-commits - lists.torproject.org

[ooni-probe/master] Implement basic collector for ooniprobe reports
by art＠torproject.org 11 Nov '12

11 Nov '12

commit 5544ee71c001d64aecc5a729724af53c8ec2f246 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 11 18:15:54 2012 +0100 Implement basic collector for ooniprobe reports * Reports can be submitted over the network via http to a remote collector * Implement the backend component of the collector that writes submitted reports to flat files, following the report_id naming convention. * XXX add support for connecting to the collector via Tor Hidden Services --- nettests/core/http_host.py | 2 +- ooni/oonicli.py | 2 + ooni/reporter.py | 176 ++++++++++++++++++++------------------------ ooni/runner.py | 19 +++-- ooni/utils/net.py | 2 +- oonib/config.py | 6 +- oonib/report/api.py | 44 ++++++++++- 7 files changed, 138 insertions(+), 113 deletions(-) diff --git a/nettests/core/http_host.py b/nettests/core/http_host.py index 0e73f82..662cc40 100644 --- a/nettests/core/http_host.py +++ b/nettests/core/http_host.py @@ -16,7 +16,7 @@ from ooni.utils import log from ooni.templates import httpt class UsageOptions(usage.Options): - optParameters = [['backend', 'b', 'http://127.0.0.1:1234', + optParameters = [['backend', 'b', 'http://127.0.0.1:57001', 'URL of the test backend to use'], ['content', 'c', None, 'The file to read from containing the content of a block page']] diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 9287174..9473ad8 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -40,6 +40,8 @@ class Options(usage.Options, app.ReactorSelectionMixin): 'Report deferred creation and callback stack traces'],] optParameters = [["reportfile", "o", None, "report file name"], + ["collector", "c", None, + "Address of the collector of test results. (example: http://127.0.0.1:8888)"], ["logfile", "l", None, "log file name"], ["pcapfile", "p", None, "pcap file name"]] diff --git a/ooni/reporter.py b/ooni/reporter.py index 7147b35..05ea94a 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -106,6 +106,8 @@ class OReporter(object): pass def testDone(self, test, test_name): + log.debug("Finished running %s" % test_name) + log.debug("Writing report") test_report = dict(test.report) if isinstance(test.input, packet.Packet): @@ -120,14 +122,15 @@ class OReporter(object): 'test_name': test_name, 'test_started': test_started, 'report': test_report} - self.writeReportEntry(report) + return self.writeReportEntry(report) def allDone(self): log.debug("allDone: Finished running all tests") - self.finish() try: + log.debug("Stopping the reactor") reactor.stop() except: + log.debug("Unable to stop the reactor") pass return None @@ -151,6 +154,7 @@ class YAMLReporter(OReporter): untilConcludes(self._stream.flush) def writeReportEntry(self, entry): + log.debug("Writing report with YAML reporter") self._write('---\n') self._write(safe_dump(entry)) self._write('...\n') @@ -169,88 +173,60 @@ class YAMLReporter(OReporter): def finish(self): self._stream.close() -class OONIBReporter(object): + +class OONIBReportUpdateFailed(Exception): + pass + +class OONIBReportCreationFailed(Exception): + pass + +class OONIBTestDetailsLookupFailed(Exception): + pass + +class OONIBReporter(OReporter): def __init__(self, backend_url): from twisted.web.client import Agent from twisted.internet import reactor self.agent = Agent(reactor) self.backend_url = backend_url - def _newReportCreated(self, data): - log.debug("newReportCreated %s" % data) - return data - - def _processResponseBody(self, response, body_cb): - log.debug("processResponseBody %s" % response) - done = defer.Deferred() - response.deliverBody(BodyReceiver(done)) - done.addCallback(body_cb) - return done - - def createReport(self, test_name, - test_version, report_header): - url = self.backend_url + '/new' - software_version = '0.0.1' - - request = {'software_name': 'ooni-probe', - 'software_version': software_version, - 'test_name': test_name, - 'test_version': test_version, - 'progress': 0, - 'content': report_header - } - def gotDetails(test_details): - log.debug("Creating report via url %s" % url) - - bodyProducer = StringProducer(json.dumps(request)) - d = self.agent.request("POST", url, - bodyProducer=bodyProducer) - d.addCallback(self._processResponseBody, - self._newReportCreated) - return d - - d = getTestDetails(options) - d.addCallback(gotDetails) - return d - - def writeReportEntry(self, entry, test_id=None): - if not test_id: - log.err("Write report entry on OONIB requires test id") - raise NoTestIDSpecified + @defer.inlineCallbacks + def writeReportEntry(self, entry): + log.debug("Writing report with OONIB reporter") + content = '---\n' + content += safe_dump(entry) + content += '...\n' - report = '---\n' - report += safe_dump(entry) - report += '...\n' + url = self.backend_url + '/report/new' - url = self.backend_url + '/new' + request = {'report_id': self.report_id, + 'content': content} - request = {'test_id': test_id, - 'content': report} + log.debug("Updating report with id %s" % self.report_id) + request_json = json.dumps(request) + log.debug("Sending %s" % request_json) bodyProducer = StringProducer(json.dumps(request)) - d = self.agent.request("PUT", url, - bodyProducer=bodyProducer) - - d.addCallback(self._processResponseBody, - self._newReportCreated) - return d - + log.debug("Creating report via url %s" % url) + try: + response = yield self.agent.request("PUT", url, + bodyProducer=bodyProducer) + except: + # XXX we must trap this in the runner and make sure to report the data later. + raise OONIBReportUpdateFailed -class OONIBReporter(OReporter): - def __init__(self, backend_url): - from twisted.web.client import Agent - from twisted.internet import reactor - self.agent = Agent(reactor) - self.backend_url = backend_url + #parsed_response = json.loads(backend_response) + #self.report_id = parsed_response['report_id'] + #self.backend_version = parsed_response['backend_version'] + #log.debug("Created report with id %s" % parsed_response['report_id']) - def _processResponseBody(self, *arg, **kw): - #done = defer.Deferred() - #response.deliverBody(BodyReceiver(done)) - #done.addCallback(self._newReportCreated) - #return done + @defer.inlineCallbacks def createReport(self, options): + """ + Creates a report on the oonib collector. + """ test_name = options['name'] test_version = options['version'] @@ -258,33 +234,41 @@ class OONIBReporter(OReporter): url = self.backend_url + '/report/new' software_version = '0.0.1' - def gotDetails(test_details): - content = '---\n' - content += safe_dump(test_details) - content += '...\n' + test_details = yield getTestDetails(options) + + content = '---\n' + content += safe_dump(test_details) + content += '...\n' - request = {'software_name': 'ooniprobe', - 'software_version': software_version, - 'test_name': test_name, - 'test_version': test_version, - 'progress': 0, - 'content': content - } - log.debug("Creating report via url %s" % url) - request_json = json.dumps(request) - log.debug("Sending %s" % request_json) - - def bothCalls(*arg, **kw): - print arg, kw - - body_producer = StringProducer(request_json) - d = self.agent.request("POST", url, None, - body_producer) - d.addBoth(self._processResponseBody) - return d - - d = getTestDetails(options) - d.addCallback(gotDetails) - # XXX handle errors - return d + request = {'software_name': 'ooniprobe', + 'software_version': software_version, + 'test_name': test_name, + 'test_version': test_version, + 'progress': 0, + 'content': content + } + log.debug("Creating report via url %s" % url) + request_json = json.dumps(request) + log.debug("Sending %s" % request_json) + + bodyProducer = StringProducer(json.dumps(request)) + log.debug("Creating report via url %s" % url) + + try: + response = yield self.agent.request("POST", url, + bodyProducer=bodyProducer) + except: + raise OONIBReportCreationFailed + + # This is a little trix to allow us to unspool the response. We create + # a deferred and call yield on it. + response_body = defer.Deferred() + response.deliverBody(BodyReceiver(response_body)) + + backend_response = yield response_body + + parsed_response = json.loads(backend_response) + self.report_id = parsed_response['report_id'] + self.backend_version = parsed_response['backend_version'] + log.debug("Created report with id %s" % parsed_response['report_id']) diff --git a/ooni/runner.py b/ooni/runner.py index b92ad7e..1aac145 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -153,8 +153,10 @@ def loadTestsAndOptions(classes, cmd_line_options): def runTestWithInput(test_class, test_method, test_input, oreporter): log.debug("Running %s with %s" % (test_method, test_input)) + def test_done(result, test_instance, test_name): - oreporter.testDone(test_instance, test_name) + log.debug("runTestWithInput: concluded %s" % test_name) + return oreporter.testDone(test_instance, test_name) def test_error(error, test_instance, test_name): log.err("%s\n" % error) @@ -221,22 +223,21 @@ def runTestCases(test_cases, options, log.msg("Could not find inputs!") log.msg("options[0] = %s" % first) test_inputs = [None] - + reportFile = open(yamloo_filename, 'w+') - #oreporter = reporter.YAMLReporter(reportFile) - oreporter = reporter.OONIBReporter('http://127.0.0.1:8888') + + if cmd_line_options['collector']: + oreporter = reporter.OONIBReporter(cmd_line_options['collector']) + else: + oreporter = reporter.YAMLReporter(reportFile) input_unit_factory = InputUnitFactory(test_inputs) log.debug("Creating report") - yield oreporter.createReport(options) - - oreporter = reporter.YAMLReporter(reportFile) - - input_unit_factory = InputUnitFactory(test_inputs) yield oreporter.createReport(options) + # This deferred list is a deferred list of deferred lists # it is used to store all the deferreds of the tests that # are run diff --git a/ooni/utils/net.py b/ooni/utils/net.py index d43261a..3ddba61 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -7,7 +7,7 @@ import sys from zope.interface import implements -from twisted.internet import protocol +from twisted.internet import protocol, defer from twisted.internet import threads, reactor from twisted.web.iweb import IBodyProducer diff --git a/oonib/config.py b/oonib/config.py index dc2be2f..b34a8ae 100644 --- a/oonib/config.py +++ b/oonib/config.py @@ -18,11 +18,13 @@ main.db_threadpool_size = 10 helpers = Storage() helpers.http_return_request = Storage() -helpers.http_return_request.port = 57001 +helpers.http_return_request.port = 57001 +# XXX this actually needs to be the advertised Server HTTP header of our web +# server helpers.http_return_request.server_version = "Apache" helpers.tcp_echo = Storage() -helpers.tcp_echo.port = 57002 +helpers.tcp_echo.port = 57002 helpers.daphn3 = Storage() helpers.daphn3.yaml_file = "/path/to/data/oonib/daphn3.yaml" diff --git a/oonib/report/api.py b/oonib/report/api.py index 489cc25..bd409b0 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -59,6 +59,28 @@ def parseNewReportRequest(request): raise InvalidRequestField(k) return parsed_request +def parseUpdateReportRequest(request): + # XXX this and the function above can probably be refactored into something + # more compact. There is quite a bit of code duplication going on here. + + report_id_regexp = re.compile("[a-zA-Z0-9]+$") + + # XXX here we are actually parsing a json object that could be quite big. + # If we want this to scale properly we only want to look at the test_id + # field. + # We are also keeping in memory multiple copies of the same object. A lot + # of optimization can be done. + parsed_request = json.loads(request) + try: + report_id = parsed_request['report_id'] + except KeyError: + raise MissingField('report_id') + + if not re.match(report_id_regexp, report_id): + raise InvalidRequestField('report_id') + + return parsed_request + class NewReportHandlerFile(web.RequestHandler): """ Responsible for creating and updating reports by writing to flat file. @@ -129,7 +151,20 @@ class NewReportHandlerFile(web.RequestHandler): 'content': 'XXX' } """ - pass + parsed_request = parseUpdateReportRequest(self.request.body) + report_id = parsed_request['report_id'] + print "Got this request %s" % parsed_request + + report_filename = report_id + report_filename += '.yamloo' + try: + with open(report_filename, 'a+') as f: + # XXX this could be quite big. We should probably use the + # twisted.internet.fdesc module + print parsed_request['content'] + f.write(parsed_request['content']) + except IOError as e: + web.HTTPError(404, "Report not found") class NewReportHandlerDB(web.RequestHandler): """ @@ -188,7 +223,8 @@ class PCAPReportHandler(web.RequestHandler): def post(self): pass -spec = [(r"/report/new", NewReportHandlerFile), - (r"/report/pcap", PCAPReportHandler)] +reportingBackendAPI = [(r"/report/new", NewReportHandlerFile), + (r"/report/pcap", PCAPReportHandler) +] -reportingBackend = web.Application(spec, debug=True) +reportingBackend = web.Application(reportingBackendAPI, debug=True)

1 0

[translation/vidalia_alpha_completed] Update translations for vidalia_alpha_completed
by translation＠torproject.org 11 Nov '12

11 Nov '12

commit 00ce276c9feeede1b6bf675ecbf1bababb8870ee Author: Translation commit bot <translation(a)torproject.org> Date: Sun Nov 11 12:15:20 2012 +0000 Update translations for vidalia_alpha_completed --- es/vidalia_es.po | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/es/vidalia_es.po b/es/vidalia_es.po index ba51e72..34bd7aa 100644 --- a/es/vidalia_es.po +++ b/es/vidalia_es.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-03-21 17:46+0000\n" -"PO-Revision-Date: 2012-11-11 03:25+0000\n" +"PO-Revision-Date: 2012-11-11 12:14+0000\n" "Last-Translator: strel <strelnic(a)gmail.com>\n" "Language-Team: Spanish (http://www.transifex.com/projects/p/torproject/language/es/)\n" "MIME-Version: 1.0\n" @@ -2069,7 +2069,7 @@ msgid "" "You are currently running a relay. Terminating your relay will interrupt any open connections from clients.\n" "\n" "Would you like to shutdown gracefully and give clients time to find a new relay?" -msgstr "Actualmente está ejecutando un repetidor (relay). Detenerlo interrumpiría cualquier conexión abierta desde los clientes.\n\n¿Desea cerrar con un periodo de gracia para dar tiempo de encontrar un nuevo repetidor a los clientes?" +msgstr "Actualmente está ejecutando un repetidor (relay). Detenerlo interrumpiría cualquier conexión abierta desde los clientes.\n\n¿Desea cerrar con un periodo de gracia para dar tiempo a los clientes de encontrar un nuevo repetidor (relay)?" msgctxt "MainWindow" msgid ""

1 0

[translation/vidalia_alpha] Update translations for vidalia_alpha
by translation＠torproject.org 11 Nov '12

11 Nov '12

commit d3ce9e4b06cfed96caa657a14630917b0b18589b Author: Translation commit bot <translation(a)torproject.org> Date: Sun Nov 11 12:15:18 2012 +0000 Update translations for vidalia_alpha --- es/vidalia_es.po | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/es/vidalia_es.po b/es/vidalia_es.po index ba51e72..34bd7aa 100644 --- a/es/vidalia_es.po +++ b/es/vidalia_es.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-03-21 17:46+0000\n" -"PO-Revision-Date: 2012-11-11 03:25+0000\n" +"PO-Revision-Date: 2012-11-11 12:14+0000\n" "Last-Translator: strel <strelnic(a)gmail.com>\n" "Language-Team: Spanish (http://www.transifex.com/projects/p/torproject/language/es/)\n" "MIME-Version: 1.0\n" @@ -2069,7 +2069,7 @@ msgid "" "You are currently running a relay. Terminating your relay will interrupt any open connections from clients.\n" "\n" "Would you like to shutdown gracefully and give clients time to find a new relay?" -msgstr "Actualmente está ejecutando un repetidor (relay). Detenerlo interrumpiría cualquier conexión abierta desde los clientes.\n\n¿Desea cerrar con un periodo de gracia para dar tiempo de encontrar un nuevo repetidor a los clientes?" +msgstr "Actualmente está ejecutando un repetidor (relay). Detenerlo interrumpiría cualquier conexión abierta desde los clientes.\n\n¿Desea cerrar con un periodo de gracia para dar tiempo a los clientes de encontrar un nuevo repetidor (relay)?" msgctxt "MainWindow" msgid ""

1 0

[ooni-probe/master] Clean up filenames of generated reports
by art＠torproject.org 11 Nov '12

11 Nov '12

commit afa72fbc40dddcc1d5ea93003f3dba1aff48e791 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 11 13:09:39 2012 +0100 Clean up filenames of generated reports * They now follow the format $testName_report_$timestamp.yamloo --- ooni/config.py | 9 +++++---- ooni/oonicli.py | 8 ++++++-- ooni/runner.py | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ooni/config.py b/ooni/config.py index a2ce917..29255c1 100644 --- a/ooni/config.py +++ b/ooni/config.py @@ -17,16 +17,17 @@ def get_root_path(): root = os.path.abspath(root) return root -def oreport_filenames(): +def oreport_filenames(file_name): """ returns the filenames for the pcap file and the yamloo report returns yamloo_filename, pcap_filename """ - base_filename = "%s_"+otime.timestamp()+".%s" - yamloo_filename = base_filename % ("report", "yamloo") - pcap_filename = base_filename % ("packets", "pcap") + test_name = '.'.join(file_name.split(".")[:-1]) + base_filename = "%s_%s_"+otime.timestamp()+".%s" + yamloo_filename = base_filename % (test_name, "report", "yamloo") + pcap_filename = base_filename % (test_name, "packets", "pcap") return yamloo_filename, pcap_filename config_file = os.path.join(get_root_path(), 'ooniprobe.conf') diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 25da8c3..9287174 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -93,7 +93,12 @@ def run(): if cmd_line_options['debug-stacktraces']: defer.setDebugging(True) - yamloo_filename, pcap_filename = config.oreport_filenames() + log.start(cmd_line_options['logfile']) + + test_file_name = os.path.basename(cmd_line_options['test']) + log.debug("Running script %s" % test_file_name) + + yamloo_filename, pcap_filename = config.oreport_filenames(test_file_name) if cmd_line_options['reportfile']: yamloo_filename = cmd_line_options['reportfile'] @@ -108,7 +113,6 @@ def run(): log.msg("Renaming it to %s" % pcap_filename+'.old') os.rename(pcap_filename, pcap_filename+'.old') - log.start(cmd_line_options['logfile']) classes = runner.findTestClassesFromConfig(cmd_line_options) test_cases, options = runner.loadTestsAndOptions(classes, cmd_line_options) if config.privacy.includepcap: diff --git a/ooni/runner.py b/ooni/runner.py index c6679c0..23ff898 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -221,7 +221,7 @@ def runTestCases(test_cases, options, log.msg("Could not find inputs!") log.msg("options[0] = %s" % first) test_inputs = [None] - + reportFile = open(yamloo_filename, 'w+') oreporter = reporter.YAMLReporter(reportFile)

1 0

[ooni-probe/master] Add software version to the report
by art＠torproject.org 11 Nov '12

11 Nov '12

commit a4708eaad29e275dc67b80ad767b3e17cc63abb8 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 11 11:51:46 2012 +0100 Add software version to the report * Cleanup some debug messages * Remove unused ooni-probe.conf config file --- ooni/__init__.py | 4 ++ ooni/inputunit.py | 2 +- ooni/ooni-probe.conf | 100 -------------------------------------------------- ooni/reporter.py | 6 ++- ooni/runner.py | 3 +- 5 files changed, 11 insertions(+), 104 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index 16b20ca..2763b16 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,3 +1,4 @@ +# -*- encoding: utf-8 -*- from . import config from . import inputunit from . import kit @@ -12,3 +13,6 @@ from . import utils __all__ = ['config', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', 'runner', 'templates', 'utils'] + +__author__ = "Arturo Filastò" +__version__ = "0.0.7.1-alpha" diff --git a/ooni/inputunit.py b/ooni/inputunit.py index 484631b..65f6d91 100644 --- a/ooni/inputunit.py +++ b/ooni/inputunit.py @@ -6,7 +6,7 @@ # units. Input units are how the inputs to be fed to tests are # split up into. # -# :authors: Arturo Filastò, Isis Lovecruft +# :authors: Arturo Filastò # :license: see included LICENSE file diff --git a/ooni/ooni-probe.conf b/ooni/ooni-probe.conf deleted file mode 100644 index 0c68fe0..0000000 --- a/ooni/ooni-probe.conf +++ /dev/null @@ -1,100 +0,0 @@ -# ooni-probe -# -# These are the global configuration parameters necessary to -# make ooni-probe work -[main] -reportdir = reports/ -logfile = ooniprobe.log -assetdir = assets/ -testdir = oonitests/ - -loglevel = DEBUG -consoleloglevel = DEBUG -proxyaddress = 127.0.0.1:9050 - -# The following configurations are for searching for PlanetLab -# nodes, adding them to a slice, and PlanetLab general API -# authentication: -pl_username = yourusername -pl_password = yourpassword - -# These are configurations specific to the tests that should be -# run by ooni-probe -[tests] -run = dnstamper -### DNS testing related config parameters - -# This is the list of hostnames that must be looked up -dns_experiment = top-1m.txt - -# This is the dns servers to be tested -dns_experiment_dns = dns_servers.txt - -# This is the control known good DNS server -dns_control_server = 91.191.136.152 - -# Specify whether the dnstamper test should attempt to remove -# GeoIP-based false positives by doing a reverse DNS resolve -# on positive results. -dns_reverse_lookup = true - -### captiveportal testing configuration parameters - -# This is an optional list of user defined captive portal tests, -# one per line, with each line in the format: -# experiment_url, control_result, control_code -# where experiment_url is the test page to retrieve, -# control_result is some unique text found on the test page, -# and control_code is the expected HTTP status code. -captive_portal = captive_portal_tests.txt - -# The default User Agent that ooni-probe should send for -# HTTP requests (pretend we're a Windows box running FF10): -default_ua = Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.2) Gecko/20100101 Firefox/10.0.2 - -# Enable vendor tests for captive portals: -do_captive_portal_vendor_tests = true - -# Enable DNS-based vendor tests for captive portals: -do_captive_portal_vendor_dns_tests = true - -# Enable checking of DNS requests for tampering: -check_dns_requests = true - -### traceroute testing related config parameters - -# This is the list of ips to traceroute to -traceroute = example_exp_list.txt - -# This is the list of ports that should be used -# src_x,src_y,src_z|dst_x,dst_y,dst_z -traceroute_ports = 0,53,80,123,443|0,53,80,123,443 - -# The protocol to be used in the scan -traceroute_proto = UDP, TCP, ICMP - -### keyword injection related tests - -# List of keywords -keywords = keywordlist.txt - -# hosts -keywords_hosts = hostslist.txt - -# Methods to be used for testing -keyword_method = http,telnet - -### Tor bridge testing - -tor_bridges = bridgetests.txt -tor_bridges_timeout = 40 - -[report] -file = report.log -timestamp = true -#ssh = 127.0.0.1:22 -#ssh_user = theusername -#ssh_password = thepassword -#ssh_keyfile = ~/.ssh/mykey_rsa -#ssh_rpath = ~/ooni-probe/ -#tcp = "127.0.0.1:9088" diff --git a/ooni/reporter.py b/ooni/reporter.py index b625603..eea44c7 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -51,6 +51,8 @@ def safe_dump(data, stream=None, **kw): @defer.inlineCallbacks def getTestDetails(options): + from ooni import __version__ as software_version + client_geodata = {} if config.privacy.includeip or \ @@ -87,9 +89,11 @@ def getTestDetails(options): 'probe_ip': client_geodata['ip'], 'test_name': options['name'], 'test_version': options['version'], + 'software_name': 'ooniprobe', + 'software_version': software_version } defer.returnValue(test_details) - + class OReporter(object): def createReport(options): diff --git a/ooni/runner.py b/ooni/runner.py index 07882cf..c6679c0 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -192,13 +192,12 @@ def runTestWithInputUnit(test_class, dl = [] log.debug("input unit %s" % input_unit) for test_input in input_unit: - log.debug("IU: %s" % test_input) + log.debug("running with input: %s" % test_input) try: d = runTestWithInput(test_class, test_method, test_input, oreporter) except Exception, e: print e - log.debug("here y0") dl.append(d) return defer.DeferredList(dl)

1 0

[ooni-probe/master] Add not about the fact that tests should return a deferred.
by art＠torproject.org 11 Nov '12

11 Nov '12

commit 9235c7d2d769c70971f1622ee40f3588467aaa6b Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 11 12:15:46 2012 +0100 Add not about the fact that tests should return a deferred. --- ooni/nettest.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 896ae2a..62b3657 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -81,6 +81,9 @@ class NetTestCase(object): required for proper running of a test. * localOptions: contains the parsed command line arguments. + + Quirks: + Every class that is prefixed with test *must* return a twisted.internet.defer.Deferred. """ name = "I Did Not Change The Name" author = "Jane Doe <foo(a)example.com>"

1 0

[ooni-probe/master] Implement ICMP traceroute test
by art＠torproject.org 11 Nov '12

11 Nov '12

commit 8ede385b70a9d717572b95b2d5ab87085bb82cd7 Author: Arturo Filastò <art(a)fuffa.org> Date: Sun Nov 11 12:55:17 2012 +0100 Implement ICMP traceroute test * parametrize max_ttl and timeout * make max_ttl and timeout be included in the report --- nettests/core/traceroute.py | 70 ++++++++++++++++++++++++++++++++++++------ ooni/templates/scapyt.py | 3 +- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/nettests/core/traceroute.py b/nettests/core/traceroute.py index 23f124a..4317a0e 100644 --- a/nettests/core/traceroute.py +++ b/nettests/core/traceroute.py @@ -14,50 +14,100 @@ from scapy.all import * class UsageOptions(usage.Options): optParameters = [ ['backend', 'b', '8.8.8.8', 'Test backend to use'], - ['timeout', 't', 5, 'The timeout for the traceroute test'] + ['timeout', 't', 5, 'The timeout for the traceroute test'], + ['maxttl', 'm', 30, 'The maximum value of ttl to set on packets'] ] class TracerouteTest(scapyt.BaseScapyTest): name = "Multi Protocol Traceroute Test" author = "Arturo Filastò" - version = 0.1 + version = "0.1.1" usageOptions = UsageOptions dst_ports = [22, 23, 80, 123, 443] + def max_ttl_and_timeout(self): + max_ttl = int(self.localOptions['maxttl']) + timeout = int(self.localOptions['timeout']) + self.report['max_ttl'] = max_ttl + self.report['timeout'] = timeout + return max_ttl, timeout def test_tcp_traceroute(self): + """ + Does a traceroute to the destination by sending TCP SYN packets + with TTLs from 1 until max_ttl. + """ def finished(packets, port): - log.debug("Finished running TCP traceroute test" % port) + log.debug("Finished running TCP traceroute test on port %s" % port) answered, unanswered = packets self.report['hops_'+str(port)] = [] for snd, rcv in answered: - report = {'ttl': snd.ttl, 'address': rcv.src} + report = {'ttl': snd.ttl, + 'address': rcv.src, + 'rtt': rcv.time - snd.time + } + log.debug("%s: %s" % (port, report)) self.report['hops_'+str(port)].append(report) dl = [] + max_ttl, timeout = self.max_ttl_and_timeout() for port in self.dst_ports: packets = IP(dst=self.localOptions['backend'], - ttl=(1,25),id=RandShort())/TCP(flags=0x2, dport=port) - d = self.sr(packets, timeout=5) + ttl=(1,max_ttl),id=RandShort())/TCP(flags=0x2, dport=port) + d = self.sr(packets, timeout=timeout) d.addCallback(finished, port) dl.append(d) return defer.DeferredList(dl) def test_udp_traceroute(self): + """ + Does a traceroute to the destination by sending UDP packets with empty + payloads with TTLs from 1 until max_ttl. + """ def finished(packets, port): - log.debug("Finished running udp traceroute test") + log.debug("Finished running UDP traceroute test on port %s" % port) answered, unanswered = packets self.report['hops_'+str(port)] = [] for snd, rcv in answered: - report = {'ttl': snd.ttl, 'address': rcv.src} + report = {'ttl': snd.ttl, + 'address': rcv.src, + 'rtt': rcv.time - snd.time + } + log.debug("%s: %s" % (port, report)) self.report['hops_'+str(port)].append(report) dl = [] + max_ttl, timeout = self.max_ttl_and_timeout() for port in self.dst_ports: packets = IP(dst=self.localOptions['backend'], - ttl=(1,25),id=RandShort())/UDP(dport=port) - d = self.sr(packets, timeout=5) + ttl=(1,max_ttl),id=RandShort())/UDP(dport=port) + d = self.sr(packets, timeout=timeout) d.addCallback(finished, port) dl.append(d) return defer.DeferredList(dl) + def test_icmp_traceroute(self): + """ + Does a traceroute to the destination by sending ICMP echo request + packets with TTLs from 1 until max_ttl. + """ + def finished(packets): + log.debug("Finished running ICMP traceroute test") + answered, unanswered = packets + self.report['hops'] = [] + for snd, rcv in answered: + report = {'ttl': snd.ttl, + 'address': rcv.src, + 'rtt': rcv.time - snd.time + } + log.debug("%s" % (report)) + self.report['hops'].append(report) + dl = [] + max_ttl, timeout = self.max_ttl_and_timeout() + packets = IP(dst=self.localOptions['backend'], + ttl=(1,max_ttl),id=RandShort())/ICMP() + d = self.sr(packets, timeout=timeout) + d.addCallback(finished) + return d + + diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index c443610..feaf720 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -48,12 +48,11 @@ class BaseScapyTest(NetTestCase): at layer 3. """ def finished(packets): - log.debug("Got this bullshit") answered, unanswered = packets self.report['answered_packets'] = [] self.report['sent_packets'] = [] for snd, rcv in answered: - log.debug("Writing report %s" % snd) + log.debug("Writing report for scapy test") pkt_report_r = createPacketReport(rcv) pkt_report_s = createPacketReport(snd) self.report['answered_packets'].append(pkt_report_r)

1 0

[stem/master] Adding link for stem's development wiki
by atagar＠torproject.org 11 Nov '12

11 Nov '12

commit 505b5a6f82222f4d16e1e1cdac96c66096a52bf1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Nov 10 23:17:31 2012 -0800 Adding link for stem's development wiki We don't have anything linking stem's front page to the development wiki. In general this is ok, the dev wiki is only of interest to people wanting to hack *on* stem, not use it. However, we should have at least one link to let new contibutors find it on their own. --- docs/index.rst | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 7dbde74..f0a6b67 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,6 +41,7 @@ Stem is a python controller library for `Tor <https://www.torproject.org/>`_. Li Subsections =========== +* `Development Wiki <https://trac.torproject.org/projects/tor/wiki/doc/stem>`_ * `Contact <http://www.atagar.com/contact/>`_ * :ref:`genindex` * :ref:`modindex`

1 0

r25872: {website} Adding TorBirdy to the projects table Projects table entry p (website/trunk/getinvolved/en)
by Damian Johnson 11 Nov '12

11 Nov '12

Author: atagar Date: 2012-11-11 07:12:42 +0000 (Sun, 11 Nov 2012) New Revision: 25872 Modified: website/trunk/getinvolved/en/volunteer.wml Log: Adding TorBirdy to the projects table Projects table entry provided by Sukhbir. Modified: website/trunk/getinvolved/en/volunteer.wml =================================================================== --- website/trunk/getinvolved/en/volunteer.wml 2012-11-11 04:00:10 UTC (rev 25871) +++ website/trunk/getinvolved/en/volunteer.wml 2012-11-11 07:12:42 UTC (rev 25872) @@ -166,6 +166,14 @@ </tr> <tr> + <td><a href="#project-torbirdy">TorBirdy</a></td> + <td>Browser Add-on</td> + <td>JavaScript</td> + <td>Heavy</td> + <td>Sukhbir (sukhe), ioerror</td> + </tr> + + <tr class="alt"> <td><a href="#project-obfsproxy">Obfsproxy</a></td> <td>Client Add-on</td> <td>C</td> @@ -173,7 +181,7 @@ <td>nickm, asn</td> </tr> - <tr class="alt"> + <tr> <td>*<a href="#project-thandy">Thandy</a></td> <td>Updater</td> <td>Python</td> @@ -181,7 +189,7 @@ <td>chiiph, Erinn, nickm</td> </tr> - <tr> + <tr class="alt"> <td>*<a href="#project-ooni-probe">Ooni Probe</a></td> <td>Scanner</td> <td>Python</td> @@ -189,7 +197,7 @@ <td>hellais, ioerror</td> </tr> - <tr class="alt"> + <tr> <td><a href="#project-shadow">Shadow</a></td> <td>Experimentation</td> <td>C, Python</td> @@ -197,7 +205,7 @@ <td>robgjansen</td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-torctl">TorCtl</a></td> <td>Library</td> <td>Python</td> @@ -205,7 +213,7 @@ <td>aagbsn</td> </tr> - <tr class="alt"> + <tr> <td>*<a href="#project-stem">Stem</a></td> <td>Library</td> <td>Python</td> @@ -213,7 +221,7 @@ <td>atagar, neena</td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-metrics">Metrics</a></td> <td>Client Service</td> <td>Java</td> @@ -221,7 +229,7 @@ <td>karsten</td> </tr> - <tr class="alt"> + <tr> <td><a href="#project-atlas">Atlas</a></td> <td>Client Service</td> <td>JavaScript</td> @@ -229,7 +237,7 @@ <td>hellais</td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-torstatus">TorStatus</a></td> <td>Client Service</td> <td>Python, Django</td> @@ -237,7 +245,7 @@ <td></td> </tr> - <tr class="alt"> + <tr> <td><a href="#project-compass">Compass</a></td> <td>Client Service</td> <td>Python</td> @@ -245,7 +253,7 @@ <td>gsathya, karsten</td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-weather">Weather</a></td> <td>Client Service</td> <td>Python</td> @@ -253,7 +261,7 @@ <td>kaner</td> </tr> - <tr class="alt"> + <tr> <td><a href="#project-gettor">GetTor</a></td> <td>Client Service</td> <td>Python</td> @@ -261,7 +269,7 @@ <td>kaner</td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-torcheck">TorCheck</a></td> <td>Client Service</td> <td>Python, Perl</td> @@ -269,7 +277,7 @@ <td></td> </tr> - <tr class="alt"> + <tr> <td><a href="#project-onionoo">Onionoo</a></td> <td>Backend Service</td> <td>Java, Python</td> @@ -277,7 +285,7 @@ <td>karsten, gsathya</td> </tr> - <tr> + <tr class="alt"> <td><a href="#project-bridgedb">BridgeDB</a></td> <td>Backend Service</td> <td>Python</td> @@ -285,7 +293,7 @@ <td>kaner, nickm</td> </tr> - <tr class="alt"> + <tr> <td><a href="#project-torflow">TorFlow</a></td> <td>Backend Service</td> <td>Python</td> @@ -293,7 +301,7 @@ <td>aagbsn</td> </tr> - <tr> + <tr class="alt"> <td>*<a href="#project-torbel">TorBEL</a></td> <td>Backend Service</td> <td>Python</td> @@ -482,6 +490,21 @@ as well</a>. </p> + <a id="project-torbirdy"></a> + <h3>TorBirdy (<a + href="https://github.com/ioerror/torbirdy">code</a>, <a + href="https://trac.torproject.org/projects/tor/wiki/torbirdy/dev">bug + tracker</a>)</h3> + + <p> + TorBirdy is Torbutton for Thunderbird and related Mozilla mail clients. + </p> + + <p> + <b>Project Ideas:</b><br /> + <i><a href="#improveTorbirdy">Improving TorBirdy</a></i><br /> + </p> + <a id="project-obfsproxy"></a> <h3><a href="https://gitweb.torproject.org/obfsproxy.git/tree/HEAD:/doc">Obfsproxy</a> (<a href="https://gitweb.torproject.org/obfsproxy.git">code</a>, <a

1 0

[flashproxy/master] Use the pt_cmethods_done function in another place.
by dcf＠torproject.org 11 Nov '12

11 Nov '12

commit 823596e0dded69827c3faae2add9e71a6f142b80 Author: David Fifield <david(a)bamsoftware.com> Date: Sat Nov 10 21:37:57 2012 -0800 Use the pt_cmethods_done function in another place. --- flashproxy-client | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/flashproxy-client b/flashproxy-client index fded7a0..6a81968 100755 --- a/flashproxy-client +++ b/flashproxy-client @@ -1004,7 +1004,7 @@ option.\ client_transports = pt_get_client_transports(["websocket"]) if not client_transports: - pt_line("CMETHODS", "DONE") + pt_cmethods_done() sys.exit(1) def pt_cmethod(method_name, addr):

1 0