commit de4c9f0abd91497b06397eefa3b29473e7796416 Author: Jacob Appelbaum <jacob@appelbaum.net> Date: Mon Feb 25 19:04:37 2013 -0800 Update README.md to explain easy setup steps --- README.md | 69 +++++++++++++++++++++++++++++---------- apt.key | Bin 0 -> 6624 bytes decks/before_i_commit.testdeck | 12 +++--- requirements.txt | 19 +++++++++-- setup-dependencies.sh | 66 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 2c0935c..eaf1cc2 100644 --- a/README.md +++ b/README.md @@ -9,25 +9,65 @@ using Free and Open Source Software (FL/OSS) to share observations and data about the various types, methods, and amounts of network tampering in the world. -# Let's get started with this already! +ooniprobe is the first program that users run to probe their network and to +collect data for the OONI project. Are you interested in testing your network +for signs of surveillance and censorship? Do you want to collect data to share +with others, so that you and others may better understand your network? If so, +please read this document and we hope ooniprobe will help you to gather +network data that will assist you with your endeavors! -To run OONI-probe without having to install it you must tell python that it -can import modules from the root of ooni-probe, as well as initialize the -included submodules. +## Getting started with ooniprobe is easy -## Getting started + 0) Open your favorite terminal + 1) Grab the source and install the dependencies + 2) Run ooniprobe! + 3) ... There is no step three ... -Basic requirements: +## The easy way to prep your system for running ooniprobe + + git clone https://git.torproject.org/ooni-probe.git + cd ooni-probe + ./setup-dependencies.sh + ./bin/ooniprobe --asciilulz + +## Your first network test + +We run ooniprobe with a test deck - this is a collection of tests in a single +file that tells ooniprobe how to run and what data to check or process: + + ./bin/ooniprobe -i decks/before_i_commit.testdeck + +The report output files from the above command will be located in the reports/ +directory of the source code checkout. The report output ends with the .yamloo +suffix. + +## The details + +We haven't actually installed ooniprobe - we just added the ooniprobe python +to your PYTHONPATH. We also installed all of the dependencies with your native +package manager or into a local directory managed by your user. + +## ooniprobe requirements explained + +Basic system requirements: * Git: http://git-scm.com/book/en/Getting-Started-Installing-Git * Python >= 2.6: http://www.python.org/download/releases/ * pip: http://www.pip-installer.org/en/latest/ -On debian based systems these can be installed with: - sudo apt-get install git-core python python-pip python-dev build-essential +## The more detailed way follows + +On Debian or Ubuntu GNU/Linux based systems these can be installed with: + + sudo apt-get install git-core python python-pip python-dev build-essential tor tor-geoipdb -The python dependencies required for running ooniprobe are: +Other packages that may be of interest: + + libdumbnet1 python-dumbnet python-libpcap python-pypcap python-pcapy python-dnspython + python-virtualenv virtualenvwrapper tor tor-geoipdb + +The Python dependencies required for running ooniprobe are: * Tor (>2.2.x): https://torproject.org/ * Twisted (>12.1.0): https://twistedmatrix.com/trac/ @@ -40,16 +80,9 @@ The python dependencies required for running ooniprobe are: ## Install Tor -To get the latest version of Tor you should do the following (from: https://www.torproject.org/docs/debian): - - # put in here the value of lsb_release -c (ex. oneirc for ubuntu 11.10 or squeeze for debian 6.0) - export DISTRIBUTION="squeeze" - echo "deb http://deb.torproject.org/torproject.org $DISTRIBUTION main" >> /etc/apt/sources.list - gpg --keyserver keys.gnupg.net --recv 886DDD89 - gpg --export A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89 | sudo apt-key add - - apt-get update - apt-get install tor +Install the latest version of Tor for your platform: + https://www.torproject.org/download/download.html ## Configurating a virtual environment diff --git a/apt.key b/apt.key new file mode 100644 index 0000000..2ce2110 Binary files /dev/null and b/apt.key differ diff --git a/decks/before_i_commit.testdeck b/decks/before_i_commit.testdeck index 525204d..95745ff 100644 --- a/decks/before_i_commit.testdeck +++ b/decks/before_i_commit.testdeck @@ -1,25 +1,25 @@ - options: collector: null help: 0 - logfile: before_i_commit.log + logfile: reports/before_i_commit.log pcapfile: null - reportfile: captive_portal_test.yamloo + reportfile: reports/captive_portal_test.yamloo subargs: [] test: nettests/manipulation/captiveportal.py - options: collector: null help: 0 - logfile: before_i_commit.log + logfile: reports/before_i_commit.log pcapfile: null - reportfile: dns_tamper_test.yamloo + reportfile: reports/dns_tamper_test.yamloo subargs: [-T, example_inputs/dns_tamper_test_resolvers.txt, -f, example_inputs/dns_tamper_file.txt] test: nettests/blocking/dnsconsistency.py - options: collector: null help: 0 - logfile: before_i_commit.log + logfile: reports/before_i_commit.log pcapfile: null - reportfile: http_host.yamloo + reportfile: reports/http_host.yamloo subargs: [-b, 'http://ooni.nu/test', -f, example_inputs/http_host_file.txt] test: nettests/manipulation/http_host.py # XXX this is disabled because it requires oonib to be running diff --git a/requirements.txt b/requirements.txt index 0f25b26..3a3e9bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ PyYAML>=3.10 Twisted==12.2.0 argparse>=1.2.1 distribute>=0.6.24 -dnspython>=1.10.0 docutils>=0.9.1 ipaddr>=2.1.10 pyOpenSSL>=0.13 @@ -12,12 +11,24 @@ txtorcon>=0.7 zope.component>=4.0.0 zope.event>=4.0.0 zope.interface>=4.0.1 +Pyrex # # This is a Tor Project mirror with valid SSL/TLS certs that is stable and fast # -# -https://hg.secdev.org/scapy/archive/tip.zip#egg=scapy +# Originally fetched from the hg repo on secdev.org: +# https://hg.secdev.org/scapy/archive/tip.zip#egg=scapy +# Mirrored on Tor's webserver: https://people.torproject.org/~ioerror/src/mirrors/ooniprobe/scapy-02-25-201... -# https://github.com/hellais/txsocksx/archive/master.zip +# Originally fetched from github: +# https://github.com/hellais/txsocksx/archive/master.zip +# Mirrored on Tor's webserver: https://people.torproject.org/~ioerror/src/mirrors/ooniprobe/hellais-txsocks... -# parsley>=1.1 is required +# Originally fetched from LaunchPad: +# https://launchpad.net/parsley/trunk/1.1pre1/+download/Parsley-1.1pre1.tar.gz +# Mirrored on Tor's webserver: https://people.torproject.org/~ioerror/src/mirrors/ooniprobe/Parsley-1.1pre1... +# This was 'dnspython>=1.10.0' above until it failed to download many times. +# Originally fetched from dnspython but it timed out often: +# http://www.dnspython.org/kits/1.10.0/dnspython-1.10.0.zip +# Mirrored on Tor's webserver: +https://people.torproject.org/~ioerror/src/mirrors/ooniprobe/dnspython-1.10.... diff --git a/setup-dependencies.sh b/setup-dependencies.sh new file mode 100755 index 0000000..aed5814 --- /dev/null +++ b/setup-dependencies.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Discover our Distro release +RELEASE="`lsb_release -c|cut -f 2`"; +TOR_DEB_REPO="deb.torproject.org/torproject.org"; + +# This is for Ubuntu's natty +if [ $RELEASE = "natty" ]; then + # Add Tor repo + HAVE_GPG_KEY="`apt-key finger|grep 'A3C4 F0F9 79CA A22C DBA8 F512 EE8C BC9E 886D DD89'|head -n 1`"; + if [ -z "$HAVE_GPG_KEY" ]; then + echo "It appears that you do not have the torproject.org Debian repository key installed; installing it..."; + cat apt.key | sudo apt-key add -; + else + echo "It appears that you have the torproject.org Debian repository key installed!"; + fi + + HAVE_TOR_REPO="`grep deb.torproject.org/torproject.org /etc/apt/sources.list /etc/apt/sources.list.d/* 2>&1|grep torproject|head -n 1`"; + if [ -z "$HAVE_TOR_REPO" ]; then + echo "It appears that you do not have the torproject.org Debian repository installed; installing it..."; + sudo apt-add-repository "deb $TOR_DEB_REPO $RELEASE main" + else + echo "It appears that you have the torproject.org Debian repository installed!"; + fi + + # Install the basic packages to get pip ready to roll + echo "Updating OS package list..."; + sudo apt-get update 2>&1 > /dev/null; + echo "Installing packages for your system..."; + sudo apt-get install git-core python python-pip python-dev build-essential libdumbnet1 python-dumbnet python-libpcap python-pypcap python-pcapy python-dnspython python-virtualenv virtualenvwrapper tor tor-geoipdb + + if [ ! -f ~/.virtualenvs/ooniprobe/bin/activate ]; then + # Set up the virtual environment + mkdir ~/.virtualenvs/ + virtualenv ~/.virtualenvs/ooniprobe + source ~/.virtualenvs/ooniprobe/bin/activate + else + source ~/.virtualenvs/ooniprobe/bin/activate + fi + + echo "Installing all of the Python dependency requirements with pip!"; + # Install all of the out of package manager dependencies + pip install -v --timeout 60 -r requirements.txt; + if [ $? != 0 ]; then + echo "It appears that pip is having issues installing our Python dependency requirements, we'll try again!"; + pip install -v --timeout 60 -r requirements.txt; + if [ $? != 0 ]; then + echo "It appears that pip is unable to satisfy our requirements - please run the following command:"; + echo " pip install -v --timeout 60 -r requirements.txt "; + exit 1; + fi + fi + + # Allow ooniprobe to run, if all above went well, we hope! + export PYTHONPATH=$PYTHONPATH:"`pwd`"; + export PATH=$PATH:"`pwd`"; + echo "Please add the following to your respective shell config files:"; + echo ; + echo "if [ -e ~/ooni-probe/bin ]; then"; + echo " export PATH=~/ooni-probe/bin:$PATH"; + echo "fi"; + echo "if [ -e ~/ooni-probe ]; then"; + echo 'export PYTHONPATH=$PYTHONPATH:~/ooni-probe'; + echo "fi"; + +fi