January 2014 - tor-commits - lists.torproject.org

[oonib/master] Add required package dependency libsqlite3-dev
by art＠torproject.org 31 Jan '14

31 Jan '14

commit 55113375714dcfe7edcda1b4ab0c1d16cde0b01c Author: anadahz <anadahz@github> Date: Sun Dec 15 18:07:28 2013 +0100 Add required package dependency libsqlite3-dev SQLite 3 development files are needed in order to successfully install ooni-backend. Error message while installing 'pip install -r requirements.txt' 'src/connection.h:33:21: fatal error: sqlite3.h: No such file or directory' --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b2d13b9..8504d05 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ There are a few dependencies which we recommend you get from your distribution's archives. ``` -sudo apt-get install build-essential python-dev python-setuptools openssl +sudo apt-get install build-essential python-dev python-setuptools openssl libsqlite3-dev ``` ### Tor

1 0

[oonib/master] Monkeypatch txtorcon version < 0.9.0 TCPHiddenServiceEndpoint
by art＠torproject.org 31 Jan '14

31 Jan '14

commit 8ff77ca17a987949285ba3de163c1a6e9ab6c10f Author: aagbsn <aagbsn(a)extc.org> Date: Fri Nov 22 12:13:13 2013 +0000 Monkeypatch txtorcon version < 0.9.0 TCPHiddenServiceEndpoint In txtorcon versions < 0.9.0, TCPHiddenServiceEndpoints uses the default TCP4ServerEndpoint, which listens on all addresses by default. --- oonib/runner.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/oonib/runner.py b/oonib/runner.py index 04e5310..3cd825b 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -28,6 +28,21 @@ from oonib import config from oonib import log +from txtorcon import __version__ as txtorcon_version +if txtorcon_version < '0.9.0': + """ + Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint + listens on all interfaces by default. + """ + def create_listener(self, proto): + self._update_onion(self.hiddenservice.dir) + self.tcp_endpoint = TCP4ServerEndpoint(self.reactor, self.listen_port, + interface='127.0.0.1') + d = self.tcp_endpoint.listen(self.protocolfactory) + d.addCallback(self._add_attributes).addErrback(self._retry_local_port) + return d + TCPHiddenServiceEndpoint._create_listener = create_listener + class OBaseRunner(object): pass

1 0

[oonib/master] bump to v1.0.0-rc5
by art＠torproject.org 31 Jan '14

31 Jan '14

commit 46788d7231bf628c1aa50b022fd6157d9ac86491 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Sep 12 17:52:37 2013 +0200 bump to v1.0.0-rc5 --- oonib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oonib/__init__.py b/oonib/__init__.py index b5ae531..32c4c83 100644 --- a/oonib/__init__.py +++ b/oonib/__init__.py @@ -3,7 +3,7 @@ In here we shall keep track of all variables and objects that should be instantiated only once and be common to pieces of GLBackend code. """ -__version__ = '1.0.0-rc4' +__version__ = '1.0.0-rc5' __all__ = ['Storage', 'randomStr']

1 0

[oonib/master] Fix bug in exception handling code.
by art＠torproject.org 31 Jan '14

31 Jan '14

commit b77c577d283b8797f17369ff34edf478fc4a13fb Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Dec 26 22:56:06 2013 +0100 Fix bug in exception handling code. --- oonib/report/handlers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index fd008dd..0c727ef 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -181,10 +181,10 @@ class NewReportHandlerFile(OONIBHandler): # XXX here we should validate and sanitize the request try: report_data = parseNewReportRequest(self.request.body) - except InvalidRequestField, e: - raise e.InvalidRequestField(e) - except MissingField, e: - raise e.MissingRequestField(e) + except InvalidRequestField, exc: + raise e.InvalidRequestField(exc) + except MissingField, exc: + raise e.MissingRequestField(exc) log.debug("Parsed this data %s" % report_data)

1 0

[oonib/master] Merge pull request #26 from TheTorProject/fix/exc
by art＠torproject.org 31 Jan '14

31 Jan '14

commit 53980723d2d4368ba23dd278900e2aa6310f6cab Merge: 46788d7 02e95e2 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Thu Jan 2 07:04:10 2014 -0800 Merge pull request #26 from TheTorProject/fix/exc Fix bug in exception handling code. oonib/report/handlers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)

1 0

[oonib/master] Fix other bugs in ooni-backend
by art＠torproject.org 31 Jan '14

31 Jan '14

commit 02e95e2c55165fe566414affd15033d3ad68ca7c Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Dec 27 11:56:10 2013 +0100 Fix other bugs in ooni-backend --- oonib/report/handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 82f38b3..65f0e7c 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -213,7 +213,7 @@ class NewReportHandlerFile(OONIBHandler): raise e.MissingReportHeaderKey(key) except InvalidReportHeader, key: - raise e.InvalidReportHeaderKey(key) + raise e.InvalidReportHeader(key) report_header = yaml.dump(report_header) content = "---\n" + report_header + '...\n' @@ -286,7 +286,7 @@ class NewReportHandlerFile(OONIBHandler): with open(report_filename, 'a+') as fd: fdesc.setNonBlocking(fd.fileno()) fdesc.writeToFD(fd.fileno(), data) - except IOError as e: + except IOError as exc: e.OONIBError(404, "Report not found") class ReportNotFound(Exception):

1 0

[oonib/master] Merge pull request #25 from anadahz/readme_fix
by art＠torproject.org 31 Jan '14

31 Jan '14

commit 46e5158c8a272d680ee1173f3ee2936f7f08d4e0 Merge: dfb3a1c 5511337 Author: Arturo Filastò <hellais(a)users.noreply.github.com> Date: Thu Jan 2 07:09:28 2014 -0800 Merge pull request #25 from anadahz/readme_fix Add required package dependency libsqlite3-dev README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[oonib/master] Update the default bouncer with some real addresses
by art＠torproject.org 31 Jan '14

31 Jan '14

commit dfb3a1c56d0f896a5997dc9917dbf246d20e543d Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jan 2 16:05:54 2014 +0100 Update the default bouncer with some real addresses --- data/bouncer.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/data/bouncer.yaml b/data/bouncer.yaml index 7709e40..9ab2dc0 100644 --- a/data/bouncer.yaml +++ b/data/bouncer.yaml @@ -1,5 +1,3 @@ collector: - httpo://nkvphnp3p6agi5qq.onion: - test-helper: {dns: 93.95.227.200, http-return-json-headers: 'http://93.95.227.200'} - httpo://nkvphnp3p6agi5q1.onion: - test-helper: {dns: 93.22.227.200, http-return-json-headers: 'http://93.95.227.220'} + httpo://ihiderha53f36lsd.onion: + test-helper: {dns: '93.95.227.200:57004', http-return-json-headers: 'http://93.95.227.200', ssl: 'https://93.95.227.20', tcp-echo: '213.138.109.232', traceroute: '93.95.227.200'}

1 0

[oonib/master] Fix bug in cleanup of runner
by art＠torproject.org 31 Jan '14

31 Jan '14

commit 293a1df577e0f5073a85f066f0f4546762e203b3 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Jan 23 00:35:09 2014 +0100 Fix bug in cleanup of runner --- oonib/runner.py | 106 ++++++++++++++++++++++++++----------------------------- 1 file changed, 50 insertions(+), 56 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index 04e5310..8b98dc2 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -31,56 +31,6 @@ from oonib import log class OBaseRunner(object): pass -_repo_dir = os.path.join(os.getcwd().split('ooni-backend')[0], 'ooni-backend') - -def txSetupFailed(failure): - log.err("Setup failed") - log.exception(failure) - -def setupHSEndpoint(tor_process_protocol, torconfig, endpoint): - endpointName = endpoint.settings['name'] - def setup_complete(port): - print("Exposed %s Tor hidden service on httpo://%s" % (endpointName, - port.onion_uri)) - - public_port = 80 - hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, - data_dir=os.path.join(torconfig.DataDirectory, endpointName)) - d = hs_endpoint.listen(endpoint) - d.addCallback(setup_complete) - d.addErrback(txSetupFailed) - return d - -def startTor(torconfig): - def updates(prog, tag, summary): - print("%d%%: %s" % (prog, summary)) - - tempfile.tempdir = os.path.join(_repo_dir, 'tmp') - if not os.path.isdir(tempfile.gettempdir()): - os.makedirs(tempfile.gettempdir()) - _temp_dir = tempfile.mkdtemp() - - torconfig.SocksPort = config.main.socks_port - if config.main.tor2webmode: - torconfig.Tor2webMode = 1 - torconfig.CircuitBuildTimeout = 60 - if config.main.tor_datadir is None: - log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") - log.msg("Creating tmp directory in current directory for datadir.") - log.debug("Using %s" % _temp_dir) - datadir = _temp_dir - else: - datadir = config.main.tor_datadir - torconfig.DataDirectory = datadir - torconfig.save() - if config.main.tor_binary is not None: - d = launch_tor(torconfig, reactor, - tor_binary=config.main.tor_binary, - progress_updates=updates) - else: - d = launch_tor(torconfig, reactor, progress_updates=updates) - return d - if platformType == "win32": from twisted.scripts._twistw import WindowsApplicationRunner @@ -92,6 +42,50 @@ if platformType == "win32": else: from twisted.scripts._twistd_unix import UnixApplicationRunner class OBaseRunner(UnixApplicationRunner): + temporary_data_dir = None + + def txSetupFailed(self, failure): + log.err("Setup failed") + log.exception(failure) + + def setupHSEndpoint(self, tor_process_protocol, torconfig, endpoint): + endpointName = endpoint.settings['name'] + def setup_complete(port): + print("Exposed %s Tor hidden service on httpo://%s" % (endpointName, + port.onion_uri)) + + public_port = 80 + hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, + data_dir=os.path.join(torconfig.DataDirectory, endpointName)) + d = hs_endpoint.listen(endpoint) + d.addCallback(setup_complete) + d.addErrback(self.txSetupFailed) + return d + + def startTor(self, torconfig): + def updates(prog, tag, summary): + print("%d%%: %s" % (prog, summary)) + + torconfig.SocksPort = config.main.socks_port + if config.main.tor2webmode: + torconfig.Tor2webMode = 1 + torconfig.CircuitBuildTimeout = 60 + if config.main.tor_datadir is None: + self.temporary_data_dir = tempfile.mkdtemp() + log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") + log.warn("Using %s" % self.temporary_data_dir) + torconfig.DataDirectory = self.temporary_data_dir + else: + torconfig.DataDirectory = config.main.tor_datadir + torconfig.save() + if config.main.tor_binary is not None: + d = launch_tor(torconfig, reactor, + tor_binary=config.main.tor_binary, + progress_updates=updates) + else: + d = launch_tor(torconfig, reactor, progress_updates=updates) + return d + def postApplication(self): """After the application is created, start the application and run the reactor. After the reactor stops, clean up PID files and such. @@ -101,20 +95,20 @@ else: # twisted/scripts/_twistd_unix.py 12.2.0 if config.main.tor_hidden_service: torconfig = TorConfig() - d = startTor(torconfig) - d.addCallback(setupHSEndpoint, torconfig, ooniBackend) + d = self.startTor(torconfig) + d.addCallback(self.setupHSEndpoint, torconfig, ooniBackend) if ooniBouncer: - d.addCallback(setupHSEndpoint, torconfig, ooniBouncer) + d.addCallback(self.setupHSEndpoint, torconfig, ooniBouncer) else: if ooniBouncer: reactor.listenTCP(8888, ooniBouncer, interface="127.0.0.1") reactor.listenTCP(8889, ooniBackend, interface="127.0.0.1") self.startReactor(None, self.oldstdout, self.oldstderr) self.removePID(self.config['pidfile']) - if os.path.exists(tempfile.gettempdir()): + if self.temporary_data_dir: log.msg("Removing temporary directory: %s" - % tempfile.gettempdir()) - rmtree(tempfile.gettempdir(), onerror=log.err) + % self.temporary_data_dir) + rmtree(self.temporary_data_dir, onerror=log.err) def createOrGetApplication(self): return oonibackend.application

1 0

[oonib/master] Fix memory leak inside of update report handler.
by art＠torproject.org 31 Jan '14

31 Jan '14

commit 4036ad3891f847389b22387afcdd600fff110ca4 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Jan 28 17:35:33 2014 +0100 Fix memory leak inside of update report handler. The issue was with the fact that we were not cancelling the delayed calls therefore they would stay hanging until the stale time was reached. This meant that when a lot of reporting was happening there would be N hanging delayed calls while there should only be one. We were also not deleting the items of the reports dict after we were done with them. This fixes: https://github.com/TheTorProject/ooni-backend/issues/28 --- oonib/report/api.py | 1 + oonib/report/handlers.py | 124 +++++++++++++++++++++++++++------------------- 2 files changed, 74 insertions(+), 51 deletions(-) diff --git a/oonib/report/api.py b/oonib/report/api.py index e8a4c0c..44f676c 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -2,6 +2,7 @@ from oonib.report import handlers reportAPI = [ (r"/report/([a-zA-Z0-9_\-]+)/close", handlers.CloseReportHandlerFile), + (r"/report/([a-zA-Z0-9_\-]+)", handlers.UpdateReportHandlerFile), (r"/report", handlers.NewReportHandlerFile), (r"/pcap", handlers.PCAPReportHandler), ] diff --git a/oonib/report/handlers.py b/oonib/report/handlers.py index 65f0e7c..a970f4d 100644 --- a/oonib/report/handlers.py +++ b/oonib/report/handlers.py @@ -20,6 +20,54 @@ class MissingField(Exception): class InvalidRequestField(Exception): pass +class Report(object): + def __init__(self, report_id): + self.report_id = report_id + self.delayed_call = None + + self.refresh() + + def refresh(self): + self.last_updated = time.time() + if self.delayed_call: + self.delayed_call.cancel() + self.delayed_call = reactor.callLater(config.main.stale_time, self.stale_check) + + def stale_check(self): + if (time.time() - self.last_updated) > config.main.stale_time: + try: + self.close() + except ReportNotFound: + pass + + def close(self): + def get_report_path(report_id): + return os.path.join(config.main.report_dir, report_id) + + report_filename = get_report_path(self.report_id) + try: + with open(report_filename) as fd: + yaml_data = ''.join(fd.readline() for _ in range(12)) + report_details = yaml.safe_load(yaml_data) + except IOError: + raise ReportNotFound + + timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time'])) + dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( + timestamp=timestamp, + **report_details) + + dst_path = os.path.join(config.main.archive_dir, + report_details['probe_cc']) + + if not os.path.isdir(dst_path): + os.mkdir(dst_path) + + dst_path = os.path.join(dst_path, dst_filename) + os.rename(report_filename, dst_path) + + del config.reports[report_id] + def parseUpdateReportRequest(request): #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") @@ -118,17 +166,24 @@ def validate_report_header(report_header): return report_header -def get_report_path(report_id): - return os.path.join(config.main.report_dir, report_id) +class UpdateReportMixin(object): + def updateReport(self, report_id, parsed_request): + + log.debug("Got this request %s" % parsed_request) + report_filename = os.path.join(config.main.report_dir, + report_id) + + config.reports[report_id].refresh() -def stale_check(report_id): - if (time.time() - config.reports[report_id]) > config.main.stale_time: try: - close_report(report_id) - except ReportNotFound: - pass + with open(report_filename, 'a+') as fd: + fdesc.setNonBlocking(fd.fileno()) + fdesc.writeToFD(fd.fileno(), parsed_request['content']) + except IOError as exc: + e.OONIBError(404, "Report not found") + self.write({}) -class NewReportHandlerFile(OONIBHandler): +class NewReportHandlerFile(OONIBHandler, UpdateReportMixin): """ Responsible for creating and updating reports by writing to flat file. """ @@ -202,7 +257,7 @@ class NewReportHandlerFile(OONIBHandler): except KeyError: raise e.InputHashNotProvided self.checkPolicy() - + content = yaml.safe_load(report_data['content']) content['backend_version'] = config.backend_version @@ -245,10 +300,8 @@ class NewReportHandlerFile(OONIBHandler): response['test_helper_address'] = config.helpers[requested_helper].address except KeyError: raise e.TestHelperNotFound - - config.reports[report_id] = time.time() - - reactor.callLater(config.main.stale_time, stale_check, report_id) + + config.reports[report_id] = Report(report_id) self.writeToReport(report_filename, content) @@ -269,52 +322,21 @@ class NewReportHandlerFile(OONIBHandler): } """ parsed_request = parseUpdateReportRequest(self.request.body) - report_id = parsed_request['report_id'] - log.debug("Got this request %s" % parsed_request) - report_filename = os.path.join(config.main.report_dir, - report_id) - - config.reports[report_id] = time.time() - reactor.callLater(config.main.stale_time, stale_check, report_id) - - self.updateReport(report_filename, parsed_request['content']) + self.updateReport(report_id, parsed_request) - def updateReport(self, report_filename, data): +class UpdateReportHandlerFile(OONIBHandler, UpdateReportMixin): + def post(self, report_id): try: - with open(report_filename, 'a+') as fd: - fdesc.setNonBlocking(fd.fileno()) - fdesc.writeToFD(fd.fileno(), data) - except IOError as exc: - e.OONIBError(404, "Report not found") + parsed_request = json.loads(self.request.body) + except ValueError: + raise e.InvalidRequest + self.updateReport(report_id, parsed_request) class ReportNotFound(Exception): pass -def close_report(report_id): - report_filename = get_report_path(report_id) - try: - with open(report_filename) as fd: - yaml_data = ''.join(fd.readline() for _ in range(12)) - report_details = yaml.safe_load(yaml_data) - except IOError: - raise ReportNotFound - - timestamp = otime.timestamp(datetime.fromtimestamp(report_details['start_time'])) - dst_filename = '{test_name}-{timestamp}-{probe_asn}-probe.yamloo'.format( - timestamp=timestamp, - **report_details) - - dst_path = os.path.join(config.main.archive_dir, - report_details['probe_cc']) - - if not os.path.isdir(dst_path): - os.mkdir(dst_path) - - dst_path = os.path.join(dst_path, dst_filename) - os.rename(report_filename, dst_path) - class CloseReportHandlerFile(OONIBHandler): def get(self): pass

1 0