June 2013 - tor-commits - lists.torproject.org

[oonib/master] Respect the nodaemon and chroot config options
by art＠torproject.org 07 Jun '13

07 Jun '13

commit 89fd0ae28a7789813ad315e8e9f4546b49461717 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 20:04:48 2013 +0200 Respect the nodaemon and chroot config options --- bin/oonib | 9 +++++++-- oonib/oonibackend.py | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bin/oonib b/bin/oonib index cda5bc0..b82373d 100755 --- a/bin/oonib +++ b/bin/oonib @@ -13,8 +13,13 @@ sys.path.insert(0, os.path.abspath(os.getcwd())) from oonib import runner, config from oonib.oonibackend import application -# Uncomment this line to daemonize -#sys.argv[1:] = ['-y', backend_script] +if config.main.chroot: + sys.argv.append('--chroot') + sys.argv.append(config.chroot) + +if not config.main.nodaemon: + sys.argv.append('-y') + sys.argv.append(backend_script) def runApp(config): runner.OBaseRunner(config).run() diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index 9e3e9cb..0eef588 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -23,9 +23,9 @@ from oonib import log from oonib import db_threadpool from oonib import config -if config.uid and config.gid: - application = service.Application('oonibackend', uid=config.uid, - gid=config.gid) +if config.main.uid and config.main.gid: + application = service.Application('oonibackend', uid=config.main.uid, + gid=config.main.gid) else: application = service.Application('oonibackend')

1 0

[oonib/master] Create tmp directory for tor only after having shed privs
by art＠torproject.org 07 Jun '13

07 Jun '13

commit 7bf37f8571f66665bc6bb5683149baca791c0d4e Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 20:21:37 2013 +0200 Create tmp directory for tor only after having shed privs --- oonib/runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index c3b57b0..81fef38 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -33,9 +33,6 @@ class OBaseRunner(object): _repo_dir = os.path.join(os.getcwd().split('ooni-backend')[0], 'ooni-backend') tempfile.tempdir = os.path.join(_repo_dir, 'tmp') -if not os.path.isdir(tempfile.gettempdir()): - os.makedirs(tempfile.gettempdir()) -_temp_dir = tempfile.mkdtemp() def txSetupFailed(failure): log.err("Setup failed") @@ -46,6 +43,10 @@ def setupCollector(tor_process_protocol): print("Exposed collector Tor hidden service on httpo://%s" % port.onion_uri) + if not os.path.isdir(tempfile.gettempdir()): + os.makedirs(tempfile.gettempdir()) + _temp_dir = tempfile.mkdtemp() + if config.main.tor_datadir is None: log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") log.msg("Creating tmp directory in current directory for datadir.")

1 0

[oonib/master] Priviledges are already dropped when we start Tor
by art＠torproject.org 07 Jun '13

07 Jun '13

commit 00a4d84d95c32f118b102d8b13a3fb3fe00772b4 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 21:10:41 2013 +0200 Priviledges are already dropped when we start Tor --- oonib/runner.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index e66aab1..a6b479f 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -72,8 +72,6 @@ def startTor(): torconfig = TorConfig() torconfig.SocksPort = config.main.socks_port - if config.main.uid: - torconfig.User = config.main.uid if config.main.tor2webmode: torconfig.Tor2webMode = 1 torconfig.CircuitBuildTimeout = 60

1 0

[oonib/master] Fix typo in daemonization mode
by art＠torproject.org 07 Jun '13

07 Jun '13

commit 4e04459004eeb9bfb2804046b6895c37a7c0c5fa Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 21:07:36 2013 +0200 Fix typo in daemonization mode --- bin/oonib | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/oonib b/bin/oonib index b82373d..554d3bb 100755 --- a/bin/oonib +++ b/bin/oonib @@ -19,7 +19,6 @@ if config.main.chroot: if not config.main.nodaemon: sys.argv.append('-y') - sys.argv.append(backend_script) def runApp(config): runner.OBaseRunner(config).run()

1 0

[oonib/master] When specifying UID also drop privs for tor
by art＠torproject.org 07 Jun '13

07 Jun '13

commit 8ebc4dee6a7cbdef9e95e4c2fb8aa25a29adf50f Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 20:52:25 2013 +0200 When specifying UID also drop privs for tor --- oonib/runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oonib/runner.py b/oonib/runner.py index 81fef38..e66aab1 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -32,7 +32,6 @@ class OBaseRunner(object): _repo_dir = os.path.join(os.getcwd().split('ooni-backend')[0], 'ooni-backend') -tempfile.tempdir = os.path.join(_repo_dir, 'tmp') def txSetupFailed(failure): log.err("Setup failed") @@ -43,6 +42,7 @@ def setupCollector(tor_process_protocol): print("Exposed collector Tor hidden service on httpo://%s" % port.onion_uri) + tempfile.tempdir = os.path.join(_repo_dir, 'tmp') if not os.path.isdir(tempfile.gettempdir()): os.makedirs(tempfile.gettempdir()) _temp_dir = tempfile.mkdtemp() @@ -72,6 +72,8 @@ def startTor(): torconfig = TorConfig() torconfig.SocksPort = config.main.socks_port + if config.main.uid: + torconfig.User = config.main.uid if config.main.tor2webmode: torconfig.Tor2webMode = 1 torconfig.CircuitBuildTimeout = 60

1 0

[oonib/master] Move scripts/archive_reports.py to bin/archive_oonib_reports
by art＠torproject.org 07 Jun '13

07 Jun '13

commit 6e786ee25aaf9bf2f9abb93b2db3ec984d20f89c Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 21:46:16 2013 +0200 Move scripts/archive_reports.py to bin/archive_oonib_reports This will make it so that the built rpm will also install the archiving script. --- .gitignore | 1 - bin/archive_oonib_reports | 174 ++++++++++++++++++++++++++++++++++++++++++++ scripts/archive_reports.py | 174 -------------------------------------------- setup.py | 2 +- 4 files changed, 175 insertions(+), 176 deletions(-) diff --git a/.gitignore b/.gitignore index 7210250..628ccf5 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ dist build eggs parts -bin var sdist develop-eggs diff --git a/bin/archive_oonib_reports b/bin/archive_oonib_reports new file mode 100755 index 0000000..d8c1394 --- /dev/null +++ b/bin/archive_oonib_reports @@ -0,0 +1,174 @@ +#!/usr/bin/env python +import yaml +import sys +import glob +import fcntl +import os +import re +from ipaddr import IPAddress +from datetime import timedelta +from datetime import datetime +from oonib.otime import fromTimestamp, timestamp +from oonib.otime import InvalidTimestampFormat, utcDateNow +from oonib import log + +############################################################################### +# You can set some config options here # +############################################################################### +report_age = 1 # hours +report_archive_dir = '/var/spool/mlab_ooni/archived' +report_source_dir = '/var/spool/mlab_ooni' +valid_test_versions = ['0.1', '0.1.1', '0.4', '0.1.3'] +default_probe_cc = '??' +target_permission = 0444 +path_permission = 0755 +retry_attempts = 100 +############################################################################### + +now = utcDateNow() +delta = timedelta(hours=report_age) + +def filter_reports_by_age(report): + try: + ts,__,__ = os.path.basename(report).split('_') + if now - fromTimestamp(ts) > delta: + return True + except (InvalidTimestampFormat, ValueError): + return False + +class InvalidReportField(Exception): + pass + +def validate_fields(fields): + log.debug("Report fields are: %s" % fields) + + # check report version + if fields['test_version'] not in valid_test_versions: + raise InvalidReportField('test_version') + + # check report CC + #XXX: confirm what value we use for default CC and whether + # or not we should support > 2 character CC + if fields['probe_cc'] is None: + fields['probe_cc'] = default_probe_cc + if not re.match('[A-Z\?]{2,4}', fields['probe_cc'].upper()): + raise InvalidReportField('probe_cc') + + # check report ASN + if fields['probe_asn'] is None: + fields['probe_asn'] = 'AS0' + if not re.match('^AS[0-9]{1,10}', fields['probe_asn'].upper()): + raise InvalidReportField('probe_asn') + + # check report timestamp + try: + datetime_ts = datetime.fromtimestamp(fields['start_time']) + datetime_str = timestamp(datetime_ts) + except InvalidTimestampFormat: + raise InvalidReportField('start_time') + + # check report IP + try: + IPAddress(fields['probe_ip']) + except ValueError: + raise InvalidReportField('probe_ip') + + # all looks good! + +def get_report_header_fields(report_header): + required_fields = ['probe_asn', 'probe_cc', 'probe_ip', 'start_time', + 'test_name', 'test_version'] + try: + return dict([(k,report_header[k]) for k in required_fields ]) + except KeyError: + return None + +def get_test_name(fields): + test_name = fields['test_name'].lower().replace(' ', '_') + return test_name + +def get_target_or_fail(fields, report): + # set the target filename + reportFormatVersion = fields['test_version'] + CC = fields['probe_cc'] + # XXX: wouldn't hurt to check timestamp for sanity again? + dateInISO8601Format,__,__ = os.path.basename(report).split('_') + probeASNumber = fields['probe_asn'] + testName = get_test_name(fields) + + # make sure path reportFormatVersion/CC exists + path = os.path.abspath(report_archive_dir) + for component in [reportFormatVersion, CC]: + path = os.path.join(path, component) + if not os.path.isdir(path): + try: + os.mkdir(path, path_permission) + log.debug("mkdir path: %s" % path) + except OSError: + return None + + # if the target file already exists, try to find another filename + filename = "%s-%s-%s.yamloo" % (testName, dateInISO8601Format, probeASNumber) + target = os.path.join(path, filename) + + # try to get a unique filename. os.open as used below requires + # that the file not already exist + naming_attempts = 1 + while os.path.exists(target) and naming_attempts < retry_attempts: + filename = "%s-%s-%s.%d.yamloo" % (testName, dateInISO8601Format, + probeASNumber, naming_attempts) + target = os.path.join(path, filename) + naming_attempts = naming_attempts + 1 + + if naming_attempts >= retry_attempts: + log.err("Failed getting unique filename %d times; skipping" % i) + return None + return target + +# grab list of reports +reports = glob.glob(report_source_dir+'/*') +reports_to_archive = filter(filter_reports_by_age, reports) + +# iterate over the reports to archive +for report in reports_to_archive: + log.debug("Parsing report: %s" % report) + try: + #XXX: verify that os.fdopen works as expected + f = os.fdopen(os.open(report, os.O_RDONLY|os.O_EXCL|os.O_NONBLOCK)) + except IOError: + log.err("Unable to get exclusive lock on %s; skipping" % report) + continue + + # parse the header and validate it + yamloo = yaml.safe_load_all(f) + report_header = yamloo.next() + fields = get_report_header_fields(report_header) + try: + validate_fields(fields) + except InvalidReportField, field_name: + log.err("Report %s contains invalid field called %s" % (report, field_name)) + continue + except: + log.err("An unhandled error occurred while processing %s" % report) + continue + + # get a target filename or fail + target = get_target_or_fail(fields, report) + if not target: + continue + + log.debug("target: %s" % target) + + try: + #XXX: My system does not have os.O_EXLOCK. Verify this works as is. + g = os.fdopen(os.open(target, os.O_CREAT|os.O_EXCL|os.O_NONBLOCK)) + + os.rename(report, target) + os.chmod(target, target_permission) + f.close() + g.close() + + except IOError: + # unable to lock the file... still held open? + log.err("Failed to lock target file. Possible race condition!") + continue diff --git a/scripts/archive_reports.py b/scripts/archive_reports.py deleted file mode 100755 index d8c1394..0000000 --- a/scripts/archive_reports.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python -import yaml -import sys -import glob -import fcntl -import os -import re -from ipaddr import IPAddress -from datetime import timedelta -from datetime import datetime -from oonib.otime import fromTimestamp, timestamp -from oonib.otime import InvalidTimestampFormat, utcDateNow -from oonib import log - -############################################################################### -# You can set some config options here # -############################################################################### -report_age = 1 # hours -report_archive_dir = '/var/spool/mlab_ooni/archived' -report_source_dir = '/var/spool/mlab_ooni' -valid_test_versions = ['0.1', '0.1.1', '0.4', '0.1.3'] -default_probe_cc = '??' -target_permission = 0444 -path_permission = 0755 -retry_attempts = 100 -############################################################################### - -now = utcDateNow() -delta = timedelta(hours=report_age) - -def filter_reports_by_age(report): - try: - ts,__,__ = os.path.basename(report).split('_') - if now - fromTimestamp(ts) > delta: - return True - except (InvalidTimestampFormat, ValueError): - return False - -class InvalidReportField(Exception): - pass - -def validate_fields(fields): - log.debug("Report fields are: %s" % fields) - - # check report version - if fields['test_version'] not in valid_test_versions: - raise InvalidReportField('test_version') - - # check report CC - #XXX: confirm what value we use for default CC and whether - # or not we should support > 2 character CC - if fields['probe_cc'] is None: - fields['probe_cc'] = default_probe_cc - if not re.match('[A-Z\?]{2,4}', fields['probe_cc'].upper()): - raise InvalidReportField('probe_cc') - - # check report ASN - if fields['probe_asn'] is None: - fields['probe_asn'] = 'AS0' - if not re.match('^AS[0-9]{1,10}', fields['probe_asn'].upper()): - raise InvalidReportField('probe_asn') - - # check report timestamp - try: - datetime_ts = datetime.fromtimestamp(fields['start_time']) - datetime_str = timestamp(datetime_ts) - except InvalidTimestampFormat: - raise InvalidReportField('start_time') - - # check report IP - try: - IPAddress(fields['probe_ip']) - except ValueError: - raise InvalidReportField('probe_ip') - - # all looks good! - -def get_report_header_fields(report_header): - required_fields = ['probe_asn', 'probe_cc', 'probe_ip', 'start_time', - 'test_name', 'test_version'] - try: - return dict([(k,report_header[k]) for k in required_fields ]) - except KeyError: - return None - -def get_test_name(fields): - test_name = fields['test_name'].lower().replace(' ', '_') - return test_name - -def get_target_or_fail(fields, report): - # set the target filename - reportFormatVersion = fields['test_version'] - CC = fields['probe_cc'] - # XXX: wouldn't hurt to check timestamp for sanity again? - dateInISO8601Format,__,__ = os.path.basename(report).split('_') - probeASNumber = fields['probe_asn'] - testName = get_test_name(fields) - - # make sure path reportFormatVersion/CC exists - path = os.path.abspath(report_archive_dir) - for component in [reportFormatVersion, CC]: - path = os.path.join(path, component) - if not os.path.isdir(path): - try: - os.mkdir(path, path_permission) - log.debug("mkdir path: %s" % path) - except OSError: - return None - - # if the target file already exists, try to find another filename - filename = "%s-%s-%s.yamloo" % (testName, dateInISO8601Format, probeASNumber) - target = os.path.join(path, filename) - - # try to get a unique filename. os.open as used below requires - # that the file not already exist - naming_attempts = 1 - while os.path.exists(target) and naming_attempts < retry_attempts: - filename = "%s-%s-%s.%d.yamloo" % (testName, dateInISO8601Format, - probeASNumber, naming_attempts) - target = os.path.join(path, filename) - naming_attempts = naming_attempts + 1 - - if naming_attempts >= retry_attempts: - log.err("Failed getting unique filename %d times; skipping" % i) - return None - return target - -# grab list of reports -reports = glob.glob(report_source_dir+'/*') -reports_to_archive = filter(filter_reports_by_age, reports) - -# iterate over the reports to archive -for report in reports_to_archive: - log.debug("Parsing report: %s" % report) - try: - #XXX: verify that os.fdopen works as expected - f = os.fdopen(os.open(report, os.O_RDONLY|os.O_EXCL|os.O_NONBLOCK)) - except IOError: - log.err("Unable to get exclusive lock on %s; skipping" % report) - continue - - # parse the header and validate it - yamloo = yaml.safe_load_all(f) - report_header = yamloo.next() - fields = get_report_header_fields(report_header) - try: - validate_fields(fields) - except InvalidReportField, field_name: - log.err("Report %s contains invalid field called %s" % (report, field_name)) - continue - except: - log.err("An unhandled error occurred while processing %s" % report) - continue - - # get a target filename or fail - target = get_target_or_fail(fields, report) - if not target: - continue - - log.debug("target: %s" % target) - - try: - #XXX: My system does not have os.O_EXLOCK. Verify this works as is. - g = os.fdopen(os.open(target, os.O_CREAT|os.O_EXCL|os.O_NONBLOCK)) - - os.rename(report, target) - os.chmod(target, target_permission) - f.close() - g.close() - - except IOError: - # unable to lock the file... still held open? - log.err("Failed to lock target file. Possible race condition!") - continue diff --git a/setup.py b/setup.py index 45a7908..b3d119a 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ setup( url="https://ooni.torproject.org", license="LICENSE", description="OONI-Probe Backend", - scripts=["bin/oonib"], + scripts=["bin/oonib", "bin/archive_oonib_reports"], packages=find_packages(), install_requires=install_requires, dependency_links=dependency_links,

1 0

[oonib/master] Parse json data before assigning to the datastruct
by art＠torproject.org 07 Jun '13

07 Jun '13

commit 56c4577b4fe4b40957ecceb4f93a58eacc011b5b Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 21:36:51 2013 +0200 Parse json data before assigning to the datastruct --- oonib/report/file_collector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index 2f7ac48..8211fe9 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -131,8 +131,9 @@ class NewReportHandlerFile(web.RequestHandler): test_name = report_data['test_name'] test_version = report_data['test_version'] probe_asn = report_data['probe_asn'] - content = report_data['content'] + content = json.loads(report_data['content']) content['backend_version'] = config.backend_version + content = json.dumps(content) if not probe_asn: probe_asn = "AS0"

1 0

[oonib/master] Use YAML instead of JSON for encoding the content
by art＠torproject.org 07 Jun '13

07 Jun '13

commit f7e298d95c792965f481f5002e2724471e1be6e1 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 22:45:19 2013 +0200 Use YAML instead of JSON for encoding the content --- oonib/report/file_collector.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index 8211fe9..422c94b 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -1,5 +1,6 @@ import random import string +import yaml import json import re import os @@ -131,9 +132,10 @@ class NewReportHandlerFile(web.RequestHandler): test_name = report_data['test_name'] test_version = report_data['test_version'] probe_asn = report_data['probe_asn'] - content = json.loads(report_data['content']) + content = yaml.safe_load(report_data['content']) content['backend_version'] = config.backend_version - content = json.dumps(content) + serialized_content = yaml.dump(content) + content = "---\n" + serialized_content + '...\n' if not probe_asn: probe_asn = "AS0"

1 0

[oonib/master] Do not do insane validation of test_version.
by art＠torproject.org 07 Jun '13

07 Jun '13

commit 28223e9c6add53325ee38cec70857bea10178ff6 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jun 6 23:25:32 2013 +0200 Do not do insane validation of test_version. --- bin/archive_oonib_reports | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/archive_oonib_reports b/bin/archive_oonib_reports index d8c1394..0fe6c5f 100755 --- a/bin/archive_oonib_reports +++ b/bin/archive_oonib_reports @@ -18,7 +18,6 @@ from oonib import log report_age = 1 # hours report_archive_dir = '/var/spool/mlab_ooni/archived' report_source_dir = '/var/spool/mlab_ooni' -valid_test_versions = ['0.1', '0.1.1', '0.4', '0.1.3'] default_probe_cc = '??' target_permission = 0444 path_permission = 0755 @@ -43,7 +42,7 @@ def validate_fields(fields): log.debug("Report fields are: %s" % fields) # check report version - if fields['test_version'] not in valid_test_versions: + if 'test_version' not in fields: raise InvalidReportField('test_version') # check report CC

1 0

[oonib/master] Bump oonib version to 0.9.1
by art＠torproject.org 07 Jun '13

07 Jun '13

commit 5133865aad9e9294dc13cca30d6bd8937b9ee573 Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Jun 8 01:19:55 2013 +0200 Bump oonib version to 0.9.1 --- oonib/__init__.py | 2 +- setup.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/oonib/__init__.py b/oonib/__init__.py index a612c5a..30e446a 100644 --- a/oonib/__init__.py +++ b/oonib/__init__.py @@ -16,7 +16,7 @@ from storm.databases.sqlite import SQLite import string import random -__version__ = '0.9' +__version__ = '0.9.1' class Storage(dict): """ diff --git a/setup.py b/setup.py index b3d119a..b89f9f8 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +from oonib import __version__ from __future__ import with_statement from setuptools import setup, find_packages @@ -26,7 +27,7 @@ def get_requirements(): install_requires, dependency_links = get_requirements() setup( name="oonib", - version="0.9", + version=__version__, author="The Tor Project, Inc", url="https://ooni.torproject.org", license="LICENSE",

1 0