September 2013 - tor-commits - lists.torproject.org

[oonib/master] Return the correct format
by art＠torproject.org 11 Sep '13

11 Sep '13

commit bacb8cc652916859757438ab330fafa695ec72ad Author: aagbsn <aagbsn(a)extc.org> Date: Tue Sep 10 14:26:23 2013 +0200 Return the correct format --- oonib/bouncer/handlers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py index d33dcfe..c6b249d 100644 --- a/oonib/bouncer/handlers.py +++ b/oonib/bouncer/handlers.py @@ -94,14 +94,15 @@ class Bouncer(object): # If we can, try to pick the same collector. choices = self.getHelperAddresses(helper_name) for item in response.values(): - if item['collector'] in choices.keys(): - choice = choices[item['collector']] + c = item['collector'] + h = choices[c] + if c in choices.keys(): break # Or default to a random selection else: c,h = random.choice(choices.items()) - choice = {'collector': c, 'address': h} - response[helper_name] = choice + + response[helper_name] = {'collector': c, 'address': h} except e.TestHelperNotFound: response = {'error': 'test-helper-not-found'}

1 0

[oonib/master] Merge pull request #16 from TheTorProject/fix/simplify_filterHelperAddresses
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 1dd52502d756ca560982737a62a40b2a460f16e1 Merge: fb49373 bacb8cc Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Sep 10 08:28:45 2013 -0700 Merge pull request #16 from TheTorProject/fix/simplify_filterHelperAddresses Simplify filterHelperAddresses oonib/bouncer/handlers.py | 68 +++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 36 deletions(-)

1 0

[oonib/master] Parametrise the DNS test helper remote resolver.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 639e23d7341c4a12a3cf5b901d1479e2f4fee38b Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Sep 10 19:52:33 2013 +0200 Parametrise the DNS test helper remote resolver. --- oonib.conf.example | 1 + oonib/testhelpers/dns_helpers.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/oonib.conf.example b/oonib.conf.example index 323b2af..c25d405 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -50,6 +50,7 @@ helpers: address: null udp_port: 57004 tcp_port: 57005 + resolver_address: '8.8.8.8:53' ssl: address: null diff --git a/oonib/testhelpers/dns_helpers.py b/oonib/testhelpers/dns_helpers.py index dc9c772..8959215 100644 --- a/oonib/testhelpers/dns_helpers.py +++ b/oonib/testhelpers/dns_helpers.py @@ -3,11 +3,20 @@ from twisted.internet import reactor from twisted.names import dns from twisted.names import client, server +from oonib import config + class DNSTestHelper(server.DNSServerFactory): def __init__(self, authorities = None, caches = None, clients = None, verbose = 0): - resolver = client.Resolver(servers=[('8.8.8.8', 53)]) + try: + host, port = config.helpers.dns.split(':') + port = int(port) + # XXX remove this when we have configuration file versioning. + # https://github.com/TheTorProject/ooni-probe/issues/190 + except: + host, port = '8.8.8.8', 53 + resolver = client.Resolver(servers=[(host, port)]) server.DNSServerFactory.__init__(self, authorities = authorities, caches = caches, clients = [resolver], verbose = verbose)

1 0

[oonib/master] Remove dependency from Storm
by art＠torproject.org 11 Sep '13

11 Sep '13

commit fd52ee29a712f23b2e6c82922a27f848c0ffd98b Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jun 8 11:48:32 2013 +0200 Remove dependency from Storm We are currently not using the db so there is no need for this. --- oonib/__init__.py | 17 ++----- oonib/db/__init__.py | 30 ------------ oonib/db/tables.py | 123 ----------------------------------------------- oonib/models.py | 129 -------------------------------------------------- oonib/oonibackend.py | 3 -- oonib/report/api.py | 55 +-------------------- requirements.txt | 2 - 7 files changed, 4 insertions(+), 355 deletions(-) diff --git a/oonib/__init__.py b/oonib/__init__.py index 30e446a..c374f53 100644 --- a/oonib/__init__.py +++ b/oonib/__init__.py @@ -6,18 +6,14 @@ In here we shall keep track of all variables and objects that should be instantiated only once and be common to pieces of GLBackend code. """ -__all__ = ['database', 'db_threadpool', 'Storage', 'randomStr'] -from twisted.python.threadpool import ThreadPool +__version__ = '0.9.1' + +__all__ = ['Storage', 'randomStr'] -from storm.uri import URI -from storm.twisted.transact import Transactor -from storm.databases.sqlite import SQLite import string import random -__version__ = '0.9.1' - class Storage(dict): """ A Storage object is like a dictionary except `obj.foo` can be used @@ -69,10 +65,3 @@ def randomStr(length, num=True): if num: chars += string.digits return ''.join(random.choice(chars) for x in range(length)) - -from oonib import config - -database = SQLite(URI(config.main.database_uri)) -db_threadpool = ThreadPool(0, config.main.db_threadpool_size) -db_threadpool.start() -transactor = Transactor(db_threadpool) diff --git a/oonib/db/__init__.py b/oonib/db/__init__.py deleted file mode 100644 index 2839747..0000000 --- a/oonib/db/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -__all__ = ['createTables', 'database', 'transactor'] - -from twisted.python.threadpool import ThreadPool -from twisted.internet.defer import inlineCallbacks, returnValue, Deferred - -from storm.locals import Store -from storm.uri import URI -from storm.databases.sqlite import SQLite - -from oonib import database, transactor -from oonib import log - -@inlineCallbacks -def createTables(): - """ - XXX this is to be refactored and only exists for experimentation. - """ - from oonib.db import models - for model_name in models.__all__: - try: - model = getattr(m, model_name) - except Exception, e: - log.err("Error in db initting") - log.err(e) - try: - log.debug("Creating %s" % model) - yield tables.runCreateTable(model, transactor, database) - except Exception, e: - log.debug(str(e)) - diff --git a/oonib/db/tables.py b/oonib/db/tables.py deleted file mode 100644 index 908a295..0000000 --- a/oonib/db/tables.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -from twisted.internet.defer import inlineCallbacks - -from storm.locals import Store -from storm.properties import PropertyColumn -from storm.exceptions import StormError - -from storm.variables import BoolVariable, DateTimeVariable, DateVariable -from storm.variables import DecimalVariable, EnumVariable -from storm.variables import FloatVariable, IntVariable, RawStrVariable -from storm.variables import UnicodeVariable, JSONVariable, PickleVariable - -def variableToSQLite(var_type): - """ - We take as input a storm.variable and we output the SQLite string it - represents. - """ - sqlite_type = "VARCHAR" - if isinstance(var_type, BoolVariable): - sqlite_type = "INTEGER" - elif isinstance(var_type, DateTimeVariable): - pass - sqlite_type = "" - elif isinstance(var_type, DateVariable): - pass - elif isinstance(var_type, DecimalVariable): - pass - elif isinstance(var_type, EnumVariable): - sqlite_type = "BLOB" - elif isinstance(var_type, FloatVariable): - sqlite_type = "REAL" - elif isinstance(var_type, IntVariable): - sqlite_type = "INTEGER" - elif isinstance(var_type, RawStrVariable): - sqlite_type = "BLOB" - elif isinstance(var_type, UnicodeVariable): - pass - elif isinstance(var_type, JSONVariable): - sqlite_type = "BLOB" - elif isinstance(var_type, PickleVariable): - sqlite_type = "BLOB" - return "%s" % sqlite_type - -def varsToParametersSQLite(variables, primary_keys): - """ - Takes as input a list of variables (convered to SQLite syntax and in the - form of strings) and primary_keys. - Outputs these variables converted into paramter syntax for SQLites. - - ex. - variables: ["var1 INTEGER", "var2 BOOL", "var3 INTEGER"] - primary_keys: ["var1"] - - output: "(var1 INTEGER, var2 BOOL, var3 INTEGER PRIMARY KEY (var1))" - """ - params = "(" - for var in variables[:-1]: - params += "%s %s, " % var - if len(primary_keys) > 0: - params += "%s %s, " % variables[-1] - params += "PRIMARY KEY (" - for key in primary_keys[:-1]: - params += "%s, " % key - params += "%s))" % primary_keys[-1] - else: - params += "%s %s)" % variables[-1] - return params - -def generateCreateQuery(model): - """ - This takes as input a Storm model and outputs the creation query for it. - """ - query = "CREATE TABLE "+ model.__storm_table__ + " " - - variables = [] - primary_keys = [] - - for attr in dir(model): - a = getattr(model, attr) - if isinstance(a, PropertyColumn): - var_stype = a.variable_factory() - var_type = variableToSQLite(var_stype) - name = a.name - variables.append((name, var_type)) - if a.primary: - primary_keys.append(name) - - query += varsToParametersSQLite(variables, primary_keys) - return query - -def createTable(model, transactor, database): - """ - Create the table for the specified model. - Specification of a transactor and database is useful in unittesting. - """ - if not transactor: - from oonib.db import transactor - if not database: - from oonib.db import database - store = Store(database) - create_query = generateCreateQuery(model) - try: - store.execute(create_query) - # XXX trap the specific error that is raised when the table exists - except StormError, e: - print "Failed to create table!" - print e - store.close() - store.commit() - store.close() - -@inlineCallbacks -def runCreateTable(model, transactor=None, database=None): - """ - Runs the table creation query wrapped in a transaction. - Transactions run in a separate thread. - """ - yield transactor.run(createTable, model, transactor, database) - diff --git a/oonib/models.py b/oonib/models.py deleted file mode 100644 index 60c0472..0000000 --- a/oonib/models.py +++ /dev/null @@ -1,129 +0,0 @@ -__all__ = ['Report', 'TestHelperTMP'] -from storm.twisted.transact import transact -from storm.locals import * - -from oonib import randomStr -from oonib import transactor - -def generateReportID(): - """ - Generates a report ID for usage in the database backed oonib collector. - - XXX note how this function is different from the one in report/api.py - """ - report_id = randomStr(100) - return report_id - -class OModel(object): - - transactor = transactor - - def getStore(self): - return Store(database) - - @transact - def create(self, query): - store = Store(database) - store.execute(query) - store.commit() - - @transact - def save(self): - store = getStore() - store.add(self) - store.commit() - -class Report(OModel): - """ - This represents an OONI Report as stored in the database. - - report_id: this is generated by the backend and is used by the client to - reference a previous report and append to it. It should be - treated as a shared secret between the probe and backend. - - software_name: this indicates the name of the software performing the test - (this will default to ooniprobe) - - software_version: this is the version number of the software running the - test. - - test_name: the name of the test on which the report is being created. - - test_version: indicates the version of the test - - progress: what is the current progress of the report. This allows clients - to report event partial reports up to a certain percentage of - progress. Once the report is complete progress will be 100. - - content: what is the content of the report. If the current progress is less - than 100 we should append to the YAML data structure that is - currently stored in such field. - - XXX this is currently not used. - """ - __storm_table__ = 'reports' - - createQuery = "CREATE TABLE " + __storm_table__ +\ - "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\ - "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\ - "progress VARCHAR, content VARCHAR)" - - - id = Int(primary=True) - - report_id = Unicode() - - software_name = Unicode() - software_version = Unicode() - test_name = Unicode() - test_version = Unicode() - progress = Int() - - content = Unicode() - - @transact - def new(report): - store = self.getStore() - - print "Creating new report %s" % report - - report_id = generateReportID() - new_report = models.Report() - new_report.report_id = unicode(report_id) - - new_report.software_name = report['software_name'] - new_report.software_version = report['software_version'] - new_report.test_name = report['test_name'] - new_report.test_version = report['test_version'] - new_report.progress = report['progress'] - - if 'content' in report: - new_report.content = report['content'] - - print "Report: %s" % report - - store.add(new_report) - try: - store.commit() - except: - store.close() - - defer.returnValue({'backend_version': backend_version, 'report_id': - report_id}) - - -class TestHelperTMP(OModel): - __storm_table__ = 'testhelpertmp' - - createQuery = "CREATE TABLE " + __storm_table__ +\ - "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\ - " client_ip VARCHAR, creation_time VARCHAR)" - - id = Int(primary=True) - - report_id = Unicode() - - test_helper = Unicode() - client_ip = Unicode() - - creation_time = Date() diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 0eef588..afd961f 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -20,7 +20,6 @@ from oonib.testhelpers import http_helpers, tcp_helpers from oonib import log -from oonib import db_threadpool from oonib import config if config.main.uid and config.main.gid: @@ -73,5 +72,3 @@ if config.helpers.http_return_request.port: int(config.helpers.http_return_request.port), http_helpers.HTTPReturnJSONHeadersHelper()) http_return_request_helper.setServiceParent(serviceCollection) - -reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop) diff --git a/oonib/report/api.py b/oonib/report/api.py index d01f47c..7747a1e 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -20,7 +20,7 @@ from cyclone import web from oonib import otime from oonib import randomStr -from oonib import models, config +from oonib import config from oonib.report import file_collector def parseUpdateReportRequest(request): @@ -45,59 +45,6 @@ def parseUpdateReportRequest(request): return parsed_request - -class NewReportHandlerDB(web.RequestHandler): - """ - Responsible for creating and updating reports via database. - XXX this is not yet fully implemented. - """ - - @web.asynchronous - @defer.inlineCallbacks - def post(self): - """ - Creates a new report with the input to the database. - XXX this is not yet implemented. - - * Request - - {'software_name': 'XXX', - 'software_version': 'XXX', - 'test_name': 'XXX', - 'test_version': 'XXX', - 'progress': 'XXX', - 'content': 'XXX' - } - - Optional: - 'test_helper': 'XXX' - 'client_ip': 'XXX' - - * Response - - {'backend_version': 'XXX', 'report_id': 'XXX'} - - """ - report_data = json.loads(self.request.body) - new_report = models.Report() - log.debug("Got this request %s" % report_data) - result = yield new_report.new(report_data) - self.write(result) - self.finish() - - def put(self): - """ - Update an already existing report with the database. - - XXX this is not yet implemented. - - {'report_id': 'XXX', - 'content': 'XXX' - } - """ - pass - - reportingBackendAPI = [ (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile), (r"/report", file_collector.NewReportHandlerFile), diff --git a/requirements.txt b/requirements.txt index b0d7f75..08fd114 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,8 +32,6 @@ pygeoip>=0.2.6 ## https://people.torproject.org/~ioerror/src/mirrors/ooniprobe/scapy-02-25-20… transaction>=1.4.1 -#https://pypi.python.org/packages/source/t/transaction/transaction-1.4.1.zip#md5=8db2680bc0f999219861a67b8f335a88#egg=transaction -storm>=0.19 #https://pypi.python.org/packages/source/s/storm/storm-0.19.tar.gz#md5=61d1ee4cd2a08639ab917e43fa2c9265#egg=storm txtorcon>=0.7 pyOpenSSL>=0.13

1 0

[oonib/master] Only print request payload if log level is set to debug
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 7ff1aaea9cd5a06ff2e036a396a596590861f801 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jun 8 15:59:31 2013 +0200 Only print request payload if log level is set to debug --- oonib.conf.example | 2 +- oonib/report/file_collector.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oonib.conf.example b/oonib.conf.example index 694296f..b6924f1 100644 --- a/oonib.conf.example +++ b/oonib.conf.example @@ -20,7 +20,7 @@ main: uuid: Null no_save: true profile: Null - debug: Null + debug: false stale_time: 3600 helpers: diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index 193f7fd..dab913b 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -189,7 +189,7 @@ class NewReportHandlerFile(web.RequestHandler): report_id = parsed_request['report_id'] - print "Got this request %s" % parsed_request + log.debug("Got this request %s" % parsed_request) report_filename = os.path.join(config.main.report_dir, report_id)

1 0

[oonib/master] Properly sanitize user input
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 6eb53ba096b0f8254ad06dc94cde264eec490185 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Jun 17 13:08:23 2013 +0200 Properly sanitize user input --- oonib/report/file_collector.py | 51 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index dab913b..490ed77 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -17,7 +17,7 @@ from oonib import otime from oonib.report import MissingField, InvalidRequestField -from oonib import config +from oonib import config, log def parseUpdateReportRequest(request): #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") @@ -75,6 +75,41 @@ def parseNewReportRequest(request): return parsed_request +class InvalidReportHeader(Exception): + pass + +class MissingReportHeaderKey(InvalidReportHeader): + pass + +def validate_report_header(report_header): + required_keys = ['probe_asn', 'probe_cc', 'probe_ip', 'software_name', + 'software_version', 'test_name', 'test_version'] + for key in required_keys: + if key not in report_header: + raise MissingReportHeaderKey(key) + + if report_header['probe_asn'] is None: + report_header['probe_asn'] = 'AS0' + + if not re.match('AS[0-9]+$', report_header['probe_asn']): + raise InvalidReportHeader('probe_asn') + + # If no country is known, set it to be ZZ (user assigned value in ISO 3166) + if report_header['probe_cc'] is None: + report_header['probe_cc'] = 'ZZ' + + if not re.match('[a-zA-Z]{2}$', report_header['probe_cc']): + raise InvalidReportHeader('probe_cc') + + if not re.match('[a-z_\-]+$', report_header['test_name']): + raise InvalidReportHeader('test_name') + + + if not re.match('([0-9]+\.)+[0-9]+$', report_header['test_version']): + raise InvalidReportHeader('test_version') + + return report_header + def get_report_path(report_id): return os.path.join(config.main.report_dir, report_id) @@ -145,8 +180,18 @@ class NewReportHandlerFile(web.RequestHandler): probe_asn = report_data['probe_asn'] content = yaml.safe_load(report_data['content']) content['backend_version'] = config.backend_version - serialized_content = yaml.dump(content) - content = "---\n" + serialized_content + '...\n' + + try: + report_header = validate_report_header(content) + + except MissingReportHeaderKey, key: + raise web.HTTPError(406, "Missing report header key %s" % key) + + except InvalidReportHeader, key: + raise web.HTTPError(406, "Invalid report header %s" % key) + + report_header = yaml.dump(report_header) + content = "---\n" + report_header + '...\n' if not probe_asn: probe_asn = "AS0"

1 0

[oonib/master] Import future before version
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 0470288eddc058f379edfe8c4cea5029418d2316 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jun 8 11:42:20 2013 +0200 Import future before version --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b89f9f8..0659d48 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ -from oonib import __version__ from __future__ import with_statement + +from oonib import __version__ from setuptools import setup, find_packages def get_requirements():

1 0

[oonib/master] adds libsqlite3-dev installation to Vagrantfile
by art＠torproject.org 11 Sep '13

11 Sep '13

commit 3d578ab383802081142062b1abf37e0b2c643e8f Author: Johannes Fürmann <johannes(a)weltraumpflege.org> Date: Fri Jun 21 17:08:49 2013 +0200 adds libsqlite3-dev installation to Vagrantfile --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 01e401a..33b89ba 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -111,7 +111,7 @@ Vagrant.configure("2") do |config| end $setup_script = <<SCRIPT -apt-get -y install curl python-setuptools python-dev +apt-get -y install curl python-setuptools python-dev libsqlite3-dev echo "Updating to the latest version of PIP" cd /tmp/

1 0

[oonib/master] Fixes a bug where logfile config option was unused.
by art＠torproject.org 11 Sep '13

11 Sep '13

commit ab1fed3d3e9207ad9b48f1f3e1ae971af6e8426b Author: aagbsn <aagbsn(a)extc.org> Date: Mon Jul 1 17:28:55 2013 +0200 Fixes a bug where logfile config option was unused. --- oonib/log.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/oonib/log.py b/oonib/log.py index 6a35ecf..b28b5bf 100644 --- a/oonib/log.py +++ b/oonib/log.py @@ -32,10 +32,12 @@ class LogWithNoPrefix(txlog.FileLogObserver): util.untilConcludes(self.write, "%s\n" % text) util.untilConcludes(self.flush) # Hoorj! -def start(logfile=None, application_name="oonib"): +def start(application_name="oonib"): daily_logfile = None - if not logfile: + if not config.main.logfile: + logfile = 'oonib.log' + else: logfile = config.main.logfile log_folder = os.path.dirname(logfile) @@ -85,8 +87,7 @@ class LoggerFactory(object): pass def start(self, application): - # XXX parametrize this - start('oonib.log', "OONIB") + start("OONIB") def stop(self): txlog.msg("Stopping OONIB")

1 0

[oonib/master] Merge pull request #11 from waaaaargh/master
by art＠torproject.org 11 Sep '13

11 Sep '13

commit daaef7623871feb66ef52439abc17e26bf7b7ef2 Merge: 6eb53ba 3d578ab Author: Arturo Filastò <hellais(a)gmail.com> Date: Fri Jun 21 08:21:59 2013 -0700 Merge pull request #11 from waaaaargh/master Repairs Vagrantfile Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0