- tor-commits - lists.torproject.org

[ooni-probe/master] Improve the appearance of stdout output
by art＠torproject.org 07 Dec '12

07 Dec '12

commit 48da99388163157f8c371b0207958d8804f7d574 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 7 03:52:35 2012 +0100 Improve the appearance of stdout output * No longer use the python logging interface * Include only the timestamp and component name in the ooniprobe log --- ooni/reporter.py | 3 ++- ooni/runner.py | 3 +-- ooni/utils/log.py | 33 ++++++++++++++++++--------------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 602b687..12e4dc8 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -116,7 +116,7 @@ def getTestDetails(options): config.privacy.includeasn or \ config.privacy.includecountry or \ config.privacy.includecity: - log.msg("Running geo IP lookup via check.torproject.org") + log.msg("We will include some geo data in the report") client_geodata = geodata.IPToLocation(config.probe_ip) if config.privacy.includeip: @@ -132,6 +132,7 @@ def getTestDetails(options): # XXX this regexp should probably go inside of geodata client_geodata['asn'] = \ re.search('AS\d+', client_geodata['asn']).group(0) + log.msg("Your AS number is: %s" % client_geodata['asn']) if client_geodata and not config.privacy.includecity: client_geodata['city'] = None diff --git a/ooni/runner.py b/ooni/runner.py index 2936a99..53e4e55 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -171,7 +171,7 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, tests_report = {} def test_done(result, test_instance, test_name): - log.msg("Successfully finished running %s" % test_name) + log.msg("Finished running %s" % test_name) log.debug("Deferred callback result: %s" % result) tests_report[test_name] = dict(test_instance.report) if not oonib_reporter: @@ -356,7 +356,6 @@ def updateProgressMeters(test_filename, input_unit_factory, """ Update the progress meters for keeping track of test state. """ - log.msg("Setting up progress meters") if not config.state.test_filename: config.state[test_filename] = Storage() diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 9ab8880..0740c10 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -1,9 +1,10 @@ import sys import os -import traceback import logging +import traceback from twisted.python import log as txlog +from twisted.python import util from twisted.python.failure import Failure from twisted.python.logfile import DailyLogFile @@ -14,6 +15,15 @@ from ooni import config ## IPv6 destination warnings": logging.getLogger("scapy.runtime").setLevel(logging.ERROR) +class LogWithNoPrefix(txlog.FileLogObserver): + def emit(self, eventDict): + text = txlog.textFromEventDict(eventDict) + if text is None: + return + + util.untilConcludes(self.write, "%s\n" % text) + util.untilConcludes(self.flush) # Hoorj! + def start(logfile=None, application_name="ooniprobe"): daily_logfile = None @@ -27,29 +37,22 @@ def start(logfile=None, application_name="ooniprobe"): txlog.msg("Starting %s on %s (%s UTC)" % (application_name, otime.prettyDateNow(), otime.utcPrettyDateNow())) - logging.basicConfig() - python_logging = txlog.PythonLoggingObserver(application_name) - - if config.advanced.debug: - python_logging.logger.setLevel(logging.DEBUG) - else: - python_logging.logger.setLevel(logging.INFO) - - txlog.startLoggingWithObserver(python_logging.emit) + txlog.startLoggingWithObserver(LogWithNoPrefix(sys.stdout).emit) txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit) def stop(): - txlog.msg("Stopping OONI") + print "Stopping OONI" def msg(msg, *arg, **kw): - txlog.msg(msg, logLevel=logging.INFO, *arg, **kw) + print "%s" % msg def debug(msg, *arg, **kw): - txlog.msg(msg, logLevel=logging.DEBUG, *arg, **kw) + if config.advanced.debug: + print "[D] %s" % msg def err(msg, *arg, **kw): - txlog.err("Error: " + str(msg), logLevel=logging.ERROR, *arg, **kw) + print "[!] %s" % msg def exception(error): """ @@ -71,7 +74,7 @@ class LoggerFactory(object): def start(self, application): # XXX parametrize this - start('/tmp/oonib.log', "OONIB") + start('oonib.log', "OONIB") def stop(self): txlog.msg("Stopping OONIB")

1 0

[ooni-probe/master] Update parser to the changes in the report format
by art＠torproject.org 07 Dec '12

07 Dec '12

commit fe25e0d68647af689c4015f1728cd7dd2d48b7ee Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 7 03:10:29 2012 +0100 Update parser to the changes in the report format --- scripts/example_parser.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/scripts/example_parser.py b/scripts/example_parser.py index 6c0dfab..050312b 100644 --- a/scripts/example_parser.py +++ b/scripts/example_parser.py @@ -15,7 +15,7 @@ print "Test name: %s" % report_header['test_name'] print "Test version: %s" % report_header['test_version'] for report_entry in yamloo: - print "Test: %s" % report_entry['test'] + print "Test: %s" % report_entry['test_name'] print "Input: %s" % report_entry['input'] print "Report: %s" % report_entry['report']

1 0

[ooni-probe/master] Add a very simple example on how to securely parse the ooniprobe reports
by art＠torproject.org 07 Dec '12

07 Dec '12

commit 83b0a8e394f281ac41abe0be52c7560e285e0967 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 7 03:08:08 2012 +0100 Add a very simple example on how to securely parse the ooniprobe reports --- scripts/example_parser.py | 22 ++++++++++++++++++++++ 1 files changed, 22 insertions(+), 0 deletions(-) diff --git a/scripts/example_parser.py b/scripts/example_parser.py new file mode 100644 index 0000000..6c0dfab --- /dev/null +++ b/scripts/example_parser.py @@ -0,0 +1,22 @@ +# This is an example of how to parse ooniprobe reports + +import yaml +import sys +print "Opening %s" % sys.argv[1] +f = open(sys.argv[1]) +yamloo = yaml.safe_load_all(f) + +report_header = yamloo.next() +print "ASN: %s" % report_header['probe_asn'] +print "CC: %s" % report_header['probe_cc'] +print "IP: %s" % report_header['probe_ip'] +print "Start Time: %s" % report_header['start_time'] +print "Test name: %s" % report_header['test_name'] +print "Test version: %s" % report_header['test_version'] + +for report_entry in yamloo: + print "Test: %s" % report_entry['test'] + print "Input: %s" % report_entry['input'] + print "Report: %s" % report_entry['report'] + +f.close()

1 0

[ooni-probe/master] Tor2webMode takes a boolean 0|1
by art＠torproject.org 07 Dec '12

07 Dec '12

commit 2425894859b7d8cf578a9dc9568adfdfc4626c73 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 7 02:07:31 2012 +0100 Tor2webMode takes a boolean 0|1 --- oonib/runner.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index 44cfe9c..407f10d 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -41,7 +41,7 @@ def startTor(): torconfig = txtorcon.TorConfig() torconfig.SocksPort = 9055 if config.main.tor2webmode: - torconfig.Tor2webMode = True + torconfig.Tor2webMode = 1 torconfig.save() d = txtorcon.launch_tor(torconfig, reactor, tor_binary=config.main.tor_binary,

1 0

[ooni-probe/master] Set the default ASN to 0
by art＠torproject.org 07 Dec '12

07 Dec '12

commit 06072dd64fa6544b980a841c73179fd96f8c35e9 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 7 02:30:39 2012 +0100 Set the default ASN to 0 * Should be good as per http://tools.ietf.org/html/draft-ietf-idr-as0-06 --- ooni/reporter.py | 2 +- oonib/report/file_collector.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 460b584..602b687 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -127,7 +127,7 @@ def getTestDetails(options): # Here we unset all the client geodata if the option to not include then # has been specified if client_geodata and not config.privacy.includeasn: - client_geodata['asn'] = None + client_geodata['asn'] = 'AS0' else: # XXX this regexp should probably go inside of geodata client_geodata['asn'] = \ diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index 636576b..6d5584c 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -56,6 +56,9 @@ def parseNewReportRequest(request): } parsed_request = json.loads(request) + if not parsed_request['probe_asn']: + parsed_request['probe_asn'] = 'AS0' + for k, regexp in expected_request.items(): try: value_to_check = parsed_request[k]

1 0

[ooni-probe/master] Add config option for enabling tor2web mode on oonib
by art＠torproject.org 07 Dec '12

07 Dec '12

commit 93467bb98dc20ff08dceb65c03e1f85275888b85 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 7 02:03:37 2012 +0100 Add config option for enabling tor2web mode on oonib --- oonib/config.py | 6 ++++++ oonib/runner.py | 2 ++ 2 files changed, 8 insertions(+), 0 deletions(-) diff --git a/oonib/config.py b/oonib/config.py index cdee45a..8598e00 100644 --- a/oonib/config.py +++ b/oonib/config.py @@ -24,6 +24,12 @@ main.db_threadpool_size = 10 #main.tor_binary = '/usr/sbin/tor' main.tor_binary = '/usr/local/bin/tor' +# This requires compiling Tor with tor2web mode enabled +# BEWARE!! THIS PROVIDES NO ANONYMITY!! +# ONLY DO IT IF YOU KNOW WHAT YOU ARE DOING!! +# HOSTING A COLLECTOR WITH TOR2WEB MODE GIVES YOU NO ANONYMITY!! +main.tor2webmode = True + helpers = Storage() helpers.http_return_request = Storage() diff --git a/oonib/runner.py b/oonib/runner.py index b13c8b2..44cfe9c 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -40,6 +40,8 @@ def startTor(): torconfig = txtorcon.TorConfig() torconfig.SocksPort = 9055 + if config.main.tor2webmode: + torconfig.Tor2webMode = True torconfig.save() d = txtorcon.launch_tor(torconfig, reactor, tor_binary=config.main.tor_binary,

1 0

[ooni-probe/master] Thiss
by art＠torproject.org 07 Dec '12

07 Dec '12

commit 45e0364a9b3815916fbbc0c876a7ec32b3f36f56 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 7 01:52:04 2012 +0100 Thiss --- oonib/runner.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index e406ce0..b13c8b2 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -42,6 +42,7 @@ def startTor(): torconfig.SocksPort = 9055 torconfig.save() d = txtorcon.launch_tor(torconfig, reactor, + tor_binary=config.main.tor_binary, progress_updates=updates) d.addCallback(setupCollector) d.addErrback(txSetupFailed)

1 0

[ooni-probe/master] Fix bugs in oonib reporter
by art＠torproject.org 07 Dec '12

07 Dec '12

commit 69c5d3a014212ce0c0899e256531fdfe3d179604 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 7 01:51:11 2012 +0100 Fix bugs in oonib reporter * Add support for setting the tor binary path in oonib/config.py --- oonib/config.py | 4 +++ oonib/report/api.py | 6 +++- oonib/report/file_collector.py | 48 +++++++++++++++++++++++++++++++-------- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/oonib/config.py b/oonib/config.py index 2736cd0..cdee45a 100644 --- a/oonib/config.py +++ b/oonib/config.py @@ -7,6 +7,8 @@ def get_root_path(): root = os.path.abspath(root) return root +backend_version = '0.0.1' + # XXX convert this to something that is a proper config file main = Storage() @@ -19,6 +21,8 @@ main.tor_datadir = os.path.join(get_root_path(), 'oonib', 'data', 'tor') main.database_uri = "sqlite:"+get_root_path()+"oonib_test_db.db" main.db_threadpool_size = 10 +#main.tor_binary = '/usr/sbin/tor' +main.tor_binary = '/usr/local/bin/tor' helpers = Storage() diff --git a/oonib/report/api.py b/oonib/report/api.py index 14a4bc1..b3d529d 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -1,9 +1,10 @@ """ -/new +/report /pcap -This is the async pcap reporting system. It requires the client to have created a report already, but can work independently from test progress. +This is the async pcap reporting system. It requires the client to have created +a report already, but can work independently from test progress. """ import random @@ -18,6 +19,7 @@ from cyclone import web from ooni import otime from ooni.utils import randomStr + from oonib import models, config from oonib.report import file_collector diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index 584a11d..636576b 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -6,10 +6,38 @@ import os from twisted.internet import fdesc -from oonib.report import MissingField, InvalidRequestField +from cyclone import web + from ooni.utils import randomStr +from ooni import otime + +from oonib.report import MissingField, InvalidRequestField + +from oonib import config + +def parseUpdateReportRequest(request): + #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") + + # this is the regexp for the reports that include the timestamp + report_id_regexp = re.compile("[a-zA-Z0-9_\-]+$") + + # XXX here we are actually parsing a json object that could be quite big. + # If we want this to scale properly we only want to look at the test_id + # field. + # We are also keeping in memory multiple copies of the same object. A lot + # of optimization can be done. + parsed_request = json.loads(request) + try: + report_id = parsed_request['report_id'] + except KeyError: + raise MissingField('report_id') + + if not re.match(report_id_regexp, report_id): + raise InvalidRequestField('report_id') + + return parsed_request + -from cyclone import web def parseNewReportRequest(request): """ @@ -99,7 +127,7 @@ class NewReportHandlerFile(web.RequestHandler): software_version = report_data['software_version'] test_name = report_data['test_name'] test_version = report_data['test_version'] - probe_asn = report_data['test_version'] + probe_asn = report_data['probe_asn'] content = report_data['content'] if not probe_asn: @@ -113,7 +141,7 @@ class NewReportHandlerFile(web.RequestHandler): # random nonce report_filename = os.path.join(config.main.report_dir, report_id) - response = {'backend_version': backend_version, + response = {'backend_version': config.backend_version, 'report_id': report_id } @@ -122,10 +150,10 @@ class NewReportHandlerFile(web.RequestHandler): self.write(response) - def writeToReport(report_filename, data): + def writeToReport(self, report_filename, data): with open(report_filename, 'w+') as fd: - fdesc.setNonBlocking(fd) - fdesc.writeToFD(data) + fdesc.setNonBlocking(fd.fileno()) + fdesc.writeToFD(fd.fileno(), data) def put(self): """ @@ -146,11 +174,11 @@ class NewReportHandlerFile(web.RequestHandler): self.updateReport(report_filename, parsed_request['content']) - def updateReport(report_filename, data): + def updateReport(self, report_filename, data): try: with open(report_filename, 'a+') as fd: - fdesc.setNonBlocking(fd) - fdesc.writeToFD(data) + fdesc.setNonBlocking(fd.fileno()) + fdesc.writeToFD(fd.fileno(), data) except IOError as e: web.HTTPError(404, "Report not found")

1 0

[ooni-probe/master] Update the reporting path in the ooniprobe
by art＠torproject.org 07 Dec '12

07 Dec '12

commit 8dbd73d2e72ce45c9cd5677d1c24fba393cb21d7 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 7 01:49:33 2012 +0100 Update the reporting path in the ooniprobe --- ooni/reporter.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 6ab32e8..460b584 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -269,7 +269,7 @@ class OONIBReporter(OReporter): content += safe_dump(entry) content += '...\n' - url = self.backend_url + '/report/new' + url = self.backend_url + '/report' request = {'report_id': self.report_id, 'content': content} @@ -294,7 +294,7 @@ class OONIBReporter(OReporter): """ Creates a report on the oonib collector. """ - url = self.backend_url + '/report/new' + url = self.backend_url + '/report' try: test_details = getTestDetails(options)

1 0

[ooni-probe/master] Use saner variable names
by art＠torproject.org 07 Dec '12

07 Dec '12

commit e552bb2ca2e5dff8e6a9ae1989eabeb3749dd5b1 Author: Flavio Amieiro <amieiro.flavio(a)gmail.com> Date: Thu Nov 29 11:38:35 2012 -0200 Use saner variable names --- setup.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 0112d95..1b8ecc3 100644 --- a/setup.py +++ b/setup.py @@ -14,15 +14,15 @@ def get_requirements(): # satisfy the dependency using that url (as long as it is in standard sdist # format, a single .py file or an egg). pypi_packages = [] - external_links = [] + dependency_links = [] for package_desc in requirements: if '#egg=' in package_desc: - external_links.append(package_desc) + dependency_links.append(package_desc) pypi_packages.append(package_desc.split('#egg=')[-1]) else: pypi_packages.append(package_desc) - return pypi_packages, external_links + return pypi_packages, dependency_links install_requires, dependency_links = get_requirements()

1 0