May 2013 - tor-commits - lists.torproject.org

[oonib/master] Cleanup log/print statements and docstrings in oonib/runner.py.
by isis＠torproject.org 05 May '13

05 May '13

commit 568d05b4276d6a9cd0d0f4a8e93f4b84022f65f5 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 21 05:56:12 2013 +0000 Cleanup log/print statements and docstrings in oonib/runner.py. --- oonib/runner.py | 19 ++++++++++--------- 1 files changed, 10 insertions(+), 9 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index e7b3db9..4b2ebb5 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -6,6 +6,8 @@ In here we define a runner for the oonib backend system. """ +from __future__ import print_function + from twisted.internet import reactor from twisted.application import service, internet, app from twisted.python.runtime import platformType @@ -24,7 +26,8 @@ def txSetupFailed(failure): def setupCollector(tor_process_protocol): def setup_complete(port): - print "Exposed collector Tor hidden service on httpo://%s" % port.onion_uri + print("Exposed collector Tor hidden service on httpo://%s" + % port.onion_uri) torconfig = txtorcon.TorConfig(tor_process_protocol.tor_protocol) public_port = 80 @@ -39,7 +42,7 @@ def setupCollector(tor_process_protocol): def startTor(): def updates(prog, tag, summary): - print "%d%%: %s" % (prog, summary) + print("%d%%: %s" % (prog, summary)) torconfig = txtorcon.TorConfig() torconfig.SocksPort = 9055 @@ -60,16 +63,16 @@ if platformType == "win32": from twisted.scripts._twistw import WindowsApplicationRunner OBaseRunner = WindowsApplicationRunner - # XXX Current we don't support windows for the starting of Tor Hidden Service + # XXX Currently we don't support windows for starting a Tor Hidden Service + log.warn( + "Apologies! We don't support starting a Tor Hidden Service on Windows.") else: from twisted.scripts._twistd_unix import UnixApplicationRunner class OBaseRunner(UnixApplicationRunner): def postApplication(self): - """ - To be called after the application is created: start the - application and run the reactor. After the reactor stops, - clean up PID files and such. + """After the application is created, start the application and run + the reactor. After the reactor stops, clean up PID files and such. """ self.startApplication(self.application) # This is our addition. The rest is taken from @@ -82,5 +85,3 @@ else: return oonibackend.application OBaseRunner.loggerFactory = log.LoggerFactory - -

1 0

[oonib/master] Create a tempdir at the top of the repo directory for tor_datadir if not set.
by isis＠torproject.org 05 May '13

05 May '13

commit ad74b062e7e2e0f7ce433900e33ccc924ce05c6c Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 21 05:58:33 2013 +0000 Create a tempdir at the top of the repo directory for tor_datadir if not set. --- oonib/runner.py | 34 +++++++++++++++++++++++++++++++--- 1 files changed, 31 insertions(+), 3 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index 4b2ebb5..5b77b9a 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -8,6 +8,9 @@ In here we define a runner for the oonib backend system. from __future__ import print_function +import tempfile +import os + from twisted.internet import reactor from twisted.application import service, internet, app from twisted.python.runtime import platformType @@ -20,6 +23,17 @@ from oonib import oonibackend from oonib import config from oonib import log + +class OBaseRunner(object): + pass + + +_repo_dir = os.path.join(os.getcwd().split('ooni-backend')[0], 'ooni-backend') +tempfile.tempdir = os.path.join(_repo_dir, 'tmp') +if not os.path.isdir(tempfile.gettempdir()): + os.makedirs(tempfile.gettempdir()) +_temp_dir = tempfile.mkdtemp() + def txSetupFailed(failure): log.err("Setup failed") log.exception(failure) @@ -29,13 +43,22 @@ def setupCollector(tor_process_protocol): print("Exposed collector Tor hidden service on httpo://%s" % port.onion_uri) + if config.main.tor_datadir is None: + log.warn("Option 'tor_datadir' in oonib.conf is unspecified!") + log.msg("Creating tmp directory in current directory for datadir.") + log.debug("Using %s" % _temp_dir) + datadir = _temp_dir + else: + datadir = config.main.tor_datadir + torconfig = txtorcon.TorConfig(tor_process_protocol.tor_protocol) public_port = 80 # XXX there is currently a bug in txtorcon that prevents data_dir from # being passed properly. Details on the bug can be found here: # https://github.com/meejah/txtorcon/pull/22 hs_endpoint = txtorcon.TCPHiddenServiceEndpoint(reactor, torconfig, - public_port, data_dir=config.main.tor_datadir) + public_port, + data_dir=datadir) hidden_service = hs_endpoint.listen(reportingBackend) hidden_service.addCallback(setup_complete) hidden_service.addErrback(txSetupFailed) @@ -56,8 +79,6 @@ def startTor(): d.addCallback(setupCollector) d.addErrback(txSetupFailed) -class OBaseRunner(): - pass if platformType == "win32": from twisted.scripts._twistw import WindowsApplicationRunner @@ -80,6 +101,13 @@ else: startTor() self.startReactor(None, self.oldstdout, self.oldstderr) self.removePID(self.config['pidfile']) + if os.path.exists(tempfile.gettempdir()): + try: + log.msg("Removing temporary directory: %s" + % tempfile.gettempdir()) + os.removedirs(tempfile.gettempdir()) + except OSError as ose: + log.err(ose) def createOrGetApplication(self): return oonibackend.application

1 0

[oonib/master] Use the socks_port option from oonib.conf when starting Tor.
by isis＠torproject.org 05 May '13

05 May '13

commit be1d323ff1ece98892aff344ca541c365d84554f Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 21 05:59:41 2013 +0000 Use the socks_port option from oonib.conf when starting Tor. --- oonib/runner.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index 5b77b9a..db6d734 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -68,7 +68,7 @@ def startTor(): print("%d%%: %s" % (prog, summary)) torconfig = txtorcon.TorConfig() - torconfig.SocksPort = 9055 + torconfig.SocksPort = config.main.socks_port if config.main.tor2webmode: torconfig.Tor2webMode = 1 torconfig.CircuitBuildTimeout = 60

1 0

[oonib/master] Don't pass the tor_binary option to txtorcon if not set.
by isis＠torproject.org 05 May '13

05 May '13

commit 5a45d4a31da2ce5c147c914da1429a74d3f24428 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 21 06:06:13 2013 +0000 Don't pass the tor_binary option to txtorcon if not set. * txtorcon will handle finding the tor binary for us. --- oonib/runner.py | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index db6d734..2e1e6e0 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -73,9 +73,12 @@ def startTor(): torconfig.Tor2webMode = 1 torconfig.CircuitBuildTimeout = 60 torconfig.save() - d = txtorcon.launch_tor(torconfig, reactor, - tor_binary=config.main.tor_binary, - progress_updates=updates) + if config.main.tor_binary is not None: + d = txtorcon.launch_tor(torconfig, reactor, + tor_binary=config.main.tor_binary, + progress_updates=updates) + else: + d = txtorcon.launch_tor(torconfig, reactor, progress_updates=updates) d.addCallback(setupCollector) d.addErrback(txSetupFailed)

1 0

[oonib/master] Update the .gitignore.
by isis＠torproject.org 05 May '13

05 May '13

commit 851e91bde15123574f3e645b7c6e7bdf95516cae Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 21 07:05:12 2013 +0000 Update the .gitignore. --- .gitignore | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 64 insertions(+), 1 deletions(-) diff --git a/.gitignore b/.gitignore index 0d20b64..7210250 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,64 @@ -*.pyc +# Emacs temp files: +*~ +*.*~ +\#*\# +.\#* + +# Python binary and object files: +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +__pycache__ + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject] + +# PyCharm directory: +.idea/* + +# config.tor_datadir directory: +tmp/* + +# openssl certificates, keys, signing requests: +*.crt +*.csr +*.key + +# logs and private files: +*.log +*.private + +# from "python setup.py install --record installed-files.txt": +installed-files.txt + +# client config: +oonib.conf

1 0

[oonib/master] Fix the "directory not empty" error when removing the tmp tor_datadir.
by isis＠torproject.org 05 May '13

05 May '13

commit 0a99413af64562e039ce05a7a264565ecd3aedf4 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 21 08:00:03 2013 +0000 Fix the "directory not empty" error when removing the tmp tor_datadir. --- oonib/runner.py | 11 +++++------ 1 files changed, 5 insertions(+), 6 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index 4e783df..c3b57b0 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -11,6 +11,8 @@ from __future__ import print_function import tempfile import os +from shutil import rmtree + from twisted.internet import reactor from twisted.application import service, internet, app from twisted.python.runtime import platformType @@ -105,12 +107,9 @@ else: self.startReactor(None, self.oldstdout, self.oldstderr) self.removePID(self.config['pidfile']) if os.path.exists(tempfile.gettempdir()): - try: - log.msg("Removing temporary directory: %s" - % tempfile.gettempdir()) - os.removedirs(tempfile.gettempdir()) - except OSError as ose: - log.err(ose) + log.msg("Removing temporary directory: %s" + % tempfile.gettempdir()) + rmtree(tempfile.gettempdir(), onerror=log.err) def createOrGetApplication(self): return oonibackend.application

1 0

[oonib/master] Add script for killing the TravisCI spawned oonib after thirty seconds.
by isis＠torproject.org 05 May '13

05 May '13

commit 99d86cfb7a4944e57d179709168e5c7c7e9a64a3 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 21 07:25:37 2013 +0000 Add script for killing the TravisCI spawned oonib after thirty seconds. * ∃!isisⒶwintermute:~/code/torproject/ooni-backend ∴ date Sun Apr 21 07:25:24 UTC 2013 ∃!isisⒶwintermute:~/code/torproject/ooni-backend ∴ travis-lint Hooray, /home/isis/code/torproject/ooni-backend/.travis.yml seems to be solid! --- .travis.test.sh | 31 +++++++++++++++++++++++++++++++ .travis.yml | 2 +- 2 files changed, 32 insertions(+), 1 deletions(-) diff --git a/.travis.test.sh b/.travis.test.sh new file mode 100755 index 0000000..8af421d --- /dev/null +++ b/.travis.test.sh @@ -0,0 +1,31 @@ +#!/bin/bash +############################################################################## +# +# .travis.test.sh +# ------------------- +# Run $2 for $1 seconds and then kill the process. +# +# :authors: Isis Agora Lovecruft, 0x2cdb8b35 +# :date: 21 April 2013 +# :version: 0.0.1 +############################################################################## + +function killitwithfire () { + trap - ALRM + kill -ALRM $prog 2>/dev/null + kill $! 2>/dev/null && exit 124 +} + +function waitforit () { + trap "killitwithfire" ALRM + sleep $1 & wait + kill -ALRM $$ +} + +waitforit $1& prog=$! ; shift ; +trap "killitwithfire" ALRM INT +"$@" & wait $1 +RET=$? +kill -ALRM $prog +wait $prog +exit $RET diff --git a/.travis.yml b/.travis.yml index f347600..ac0bd29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,4 +22,4 @@ script: - openssl x509 -req -days 365 -in server.csr -signkey private.key -out certificate.crt - test -f "private.key.org" && rm -f private.key.org - test -f "server.csr" && rm -f server.csr - - bin/oonib + - chmod +x .travis.test.sh && ./.travis.test.sh 30 bin/oonib

1 0

[oonib/master] Everything exits 0, and then Travis exits with 1. #FML.
by isis＠torproject.org 05 May '13

05 May '13

commit 7b9bed9b744c1391d0db8c61b4d5b1d587333ca0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 21 09:02:33 2013 +0000 Everything exits 0, and then Travis exits with 1. #FML. --- .travis.test.sh | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/.travis.test.sh b/.travis.test.sh index 559b099..4ffe8d3 100755 --- a/.travis.test.sh +++ b/.travis.test.sh @@ -10,6 +10,8 @@ # :version: 0.0.1 ############################################################################## +set -vx -- + function killitwithfire () { trap - ALRM kill -ALRM $prog 2>/dev/null

1 0

[oonib/master] Only import txtorcon classes which are in use.
by isis＠torproject.org 05 May '13

05 May '13

commit 0fe80886f44886130ef4f2d457a2248bda110914 Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 21 07:59:27 2013 +0000 Only import txtorcon classes which are in use. --- oonib/runner.py | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) diff --git a/oonib/runner.py b/oonib/runner.py index 2e1e6e0..4e783df 100644 --- a/oonib/runner.py +++ b/oonib/runner.py @@ -15,7 +15,8 @@ from twisted.internet import reactor from twisted.application import service, internet, app from twisted.python.runtime import platformType -import txtorcon +from txtorcon import TCPHiddenServiceEndpoint, TorConfig +from txtorcon import launch_tor from oonib.report.api import reportingBackend @@ -51,14 +52,13 @@ def setupCollector(tor_process_protocol): else: datadir = config.main.tor_datadir - torconfig = txtorcon.TorConfig(tor_process_protocol.tor_protocol) + torconfig = TorConfig(tor_process_protocol.tor_protocol) public_port = 80 # XXX there is currently a bug in txtorcon that prevents data_dir from # being passed properly. Details on the bug can be found here: # https://github.com/meejah/txtorcon/pull/22 - hs_endpoint = txtorcon.TCPHiddenServiceEndpoint(reactor, torconfig, - public_port, - data_dir=datadir) + hs_endpoint = TCPHiddenServiceEndpoint(reactor, torconfig, public_port, + data_dir=datadir) hidden_service = hs_endpoint.listen(reportingBackend) hidden_service.addCallback(setup_complete) hidden_service.addErrback(txSetupFailed) @@ -67,18 +67,18 @@ def startTor(): def updates(prog, tag, summary): print("%d%%: %s" % (prog, summary)) - torconfig = txtorcon.TorConfig() + torconfig = TorConfig() torconfig.SocksPort = config.main.socks_port if config.main.tor2webmode: torconfig.Tor2webMode = 1 torconfig.CircuitBuildTimeout = 60 torconfig.save() if config.main.tor_binary is not None: - d = txtorcon.launch_tor(torconfig, reactor, - tor_binary=config.main.tor_binary, - progress_updates=updates) + d = launch_tor(torconfig, reactor, + tor_binary=config.main.tor_binary, + progress_updates=updates) else: - d = txtorcon.launch_tor(torconfig, reactor, progress_updates=updates) + d = launch_tor(torconfig, reactor, progress_updates=updates) d.addCallback(setupCollector) d.addErrback(txSetupFailed)

1 0

[oonib/master] Updated the README with better instructions. [ci skip]
by isis＠torproject.org 05 May '13

05 May '13

commit d67644556149f183850e81c8dfc74e04498f219b Author: Isis Lovecruft <isis(a)torproject.org> Date: Sun Apr 21 10:30:38 2013 +0000 Updated the README with better instructions. [ci skip] --- README.md | 113 ++++++++++++++++++++++++++++++++++++------------------------ 1 files changed, 68 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index f176bb4..90a3d1a 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,52 @@ # Dependencies and Installation + - cp oonib.conf.example oonib.conf + - chmod +x .travis.test.sh && ./.travis.test.sh 30 ./bin/oonib + - echo "Build successful." -We recommend that you use a python virtualenv. The recommended commands for -setting up an ooni-backend are: +## Distro dependencies (Debian) +There are a few dependencies which we recommend you get from your +distribution's archives. + + sudo apt-get install build-essential python-dev python-setuptools openssl + +### Tor +You will need a Tor binary on your system. For complete instructions, see also: - sudo apt-get install build-essential python-dev python-setuptools - # Grab the get-pip installer to make sure we have pip>=1.3.0 [1] + https://www.torproject.org/docs/tor-doc-unix.html.en + https://www.torproject.org/docs/rpms.html.en + +If you've already got Tor, or plan to compile it yourself from source, great! +You can skip this step. Otherwise, if you're installing Tor (or reinstalling), +you'll want to make sure to get our keyring package in Debian: + + echo "deb http://deb.torproject.org/torproject.org wheezy main" | \ + sudo tee -a /etc/apt/sources.list + gpg --keyserver keys.gnupg.net --recv 886DDD89 + gpg --export A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 | sudo apt-key add - + sudo apt-get update + sudo apt-get install deb.torproject.org-keyring tor tor-geoipdb + +### Pip (>=1.3.0) +We recommend using the Pip>=1.3.0 because it included several important +security and privacy related patches: + + * It forces the use of HTTPS for [PyPI](pypi.python.org). + * and checks package hash sums before installation, with support for hashes + more collision-resistant than MD5. + +The least painful way (that we know of) to install a newer Pip is to use Pip's +get-pip script: + + # Grab the get-pip installer to make sure we have pip>=1.3.0 curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py sudo python ./get-pip.py ## pip (>=1.3.0) is recommended for security reasons + # And make sure we're actually using the newer one: sudo update-alternatives --install /usr/bin/pip pip /usr/local/bin/pip 0 + +### Virtualenv +We recommend that you use a python virtualenv. The recommended commands for +setting up this up and installing are: + sudo pip install --upgrade virtualenv virtualenvwrapper # Setup the virtualenv directory: export WORKON_HOME=~/.virtualenvs && mkdir -p $WORKON_HOME @@ -15,16 +54,28 @@ setting up an ooni-backend are: # Clone ooni-backend: git clone https://github.com/TheTorProject/ooni-backend.git && cd ooni-backend # Create the virtualenv for ooni-backend... - mkvirtualenv -a $PWD -r requirements.txt --unzip-setuptools --setuptools --no-site-packages oonib + mkvirtualenv -a $PWD --unzip-setuptools --setuptools --no-site-packages oonib # ...and install ooni-backend (sudo is not necessary since we're in a virtualenv): + pip install -r requirements.txt --use-mirrors python setup.py install -[1] Note: pip>=1.3.0 is recommended because it included several security - patches, including using forcing the use of HTTPS and checking package hash - sums before installation. +# Running an OONI collector + +## Configure oonib -# Generate self signed certs for OONIB +Copy the example config file to ```oonib.conf```. + cp oonib.conf.example oonib.conf + +Then edit your configuration to fit your needs. The fields you should probably +end up changing are ```report_dir``` (the public web server directory where you +would like ooni-probe clients to be able to submit reports to, for example, if +the clients should submit POSTs to https://abcdef0123456789.onion/report then +this would simply be ```'report'```) and ```tor_datadir``` (where you would +like the spawned Tor process to keep its data). If you compiled Tor yourself, +you'll likely want to specify it for the ```tor_binary``` option. + +## Generate self signed certs for OONIB If you want to use the HTTPS test helper, you will need to create a certificate: openssl genrsa -des3 -out private.key 4096 @@ -34,13 +85,15 @@ If you want to use the HTTPS test helper, you will need to create a certificate: openssl rsa -in private.key.org -out private.key openssl x509 -req -days 365 -in server.csr -signkey private.key -out certificate.crt rm private.key.org + rm server.csr -Don't forget to update oonib/config.py options helpers.ssl.private_key and -helpers.ssl.certificate +If you decide to put your certificate and key somewhere else, don't forget to +update oonib.conf options ```helpers.ssl.private_key``` and +```helpers.ssl.certificate```! -# Redirect low ports with iptables - -The following iptables commands will map connections on low ports to those bound by oonib +## Redirect low ports with iptables +The following iptables commands will map connections on low ports to those +bound by oonib: # Map port 80 to config.helpers.http_return_request.port (default: 57001) iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 57001 @@ -51,35 +104,6 @@ The following iptables commands will map connections on low ports to those bound # Map port 53 tcp to config.helpers.dns.tcp_port (default: 57005) iptables -t nat -A PREROUTING -p tcp -m tcp --dport 53 -j REDIRECT --tor-ports -# Install Tor (Debian). - -You will need a Tor binary on your system. For complete instructions, see also: - - https://www.torproject.org/docs/tor-doc-unix.html.en - https://www.torproject.org/docs/rpms.html.en - -Add this line to your /etc/apt/sources.list, replacing <DISTRIBUTION> -where appropriate: - - deb http://deb.torproject.org/torproject.org <DISTRIBUTION> main - -Add the Tor Project gpg key to apt: - - gpg --keyserver keys.gnupg.net --recv 886DDD89 - gpg --export A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 | sudo apt-key add - - # Update apt and install the torproject keyring, tor, and geoipdb - apt-get update - apt-get install deb.torproject.org-keyring tor tor-geoipdb - -# Configure oonib - -Copy the example config file - - cp oonib.conf.example oonib.conf - -Then edit your configuration to fit your needs. The fields you should probably -end up changing are `tor_binary`, `report_dir` and `tor_datadir`. - # (For Experts Only) Tor2webmode: WARNING: provides no anonymity! Use only if you know what you are doing! @@ -122,10 +146,9 @@ Build Tor with enable-tor2web-mode ./autogen.sh ; ./configure --enable-tor2web-mode ; make Copy the tor binary from src/or/tor somewhere and set the corresponding -options in oonib/config.py +options in oonib.conf. # To launch oonib on system boot - To launch oonib on startup, you may want to use supervisord (www.supervisord.org) The following supervisord config will use the virtual environment in /home/ooni/venv_oonib and start oonib on boot:

1 0