commit 4eea35a6fcff61b840c087703c9e708c7d6c0506 Author: Arturo Filastò art@fuffa.org Date: Sat Jan 4 17:55:30 2014 +0100

Update the installation and usage sections of the README

* Add a description to every core shipped test --- README.md | 244 ++++---------------- Vagrantfile | 10 +- ooni/nettests/blocking/dnsconsistency.py | 2 +- ooni/nettests/blocking/http_requests.py | 1 + ooni/nettests/blocking/tcp_connect.py | 1 + ooni/nettests/manipulation/daphne.py | 1 + ooni/nettests/manipulation/dnsspoof.py | 1 + .../manipulation/http_header_field_manipulation.py | 1 + ooni/nettests/manipulation/http_host.py | 3 +- .../manipulation/http_invalid_request_line.py | 1 + ooni/nettests/manipulation/traceroute.py | 1 + 11 files changed, 58 insertions(+), 208 deletions(-)

diff --git a/README.md b/README.md index 2d0aee1..6f09ca1 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,28 @@ with others, so that you and others may better understand your network? If so, please read this document and we hope ooniprobe will help you to gather network data that will assist you with your endeavors!

-## Disclaimer +#### Disclaimer

Note: ooni-probe takes no precautions to protect the install target machine from forensics analysis. If the fact that you have installed or used ooni probe is a liability for you, please be aware of this risk.

+## Installation

-## Getting started with ooniprobe is easy (with Vagrant) +### Linux + +We believe that ooniprobe runs reasonably well on Debian GNU/Linux wheezy as +well as versions of Ubuntu such as natty and later releases. Running ooniprobe +without installing it is supported with the following commands: + +``` +git clone https://git.torproject.org/ooni-probe.git +cd ooni-probe +./setup-dependencies.sh +python setup.py install +``` + +### Other platforms (with Vagrant)

0) [Install Vagrant](http://downloads.vagrantup.com/) and [Install Virtualbox](https://www.virtualbox.org/wiki/Downloads)

@@ -34,7 +48,7 @@ probe is a liability for you, please be aware of this risk.

On OSX:

-If youd don't have it install homebrew http://mxcl.github.io/homebrew/ +If you don't have it install homebrew http://mxcl.github.io/homebrew/

``` brew install git @@ -68,232 +82,69 @@ ooniprobe will be installed in `/ooni`. ooniprobe blocking/http_requests -f /ooni/inputs/input-pack/alexa-top-1k.txt ```

-## The easy way to prep your system for running ooniprobe - -We believe that ooniprobe runs reasonably well on Debian GNU/Linux wheezy as -well as versions of Ubuntu such as natty and later releases. Running ooniprobe -without installing it is supported with the following commands: - -``` -git clone https://git.torproject.org/ooni-probe.git -cd ooni-probe -./setup-dependencies.sh -./bin/ooniprobe --asciilulz -``` - -## Your first network test - -We run ooniprobe with a test deck - this is a collection of tests in a single -file that tells ooniprobe how to run and what data to check or process: - -``` -./bin/ooniprobe -i decks/before_i_commit.testdeck -``` - -The report output files from the above command will be located in the reports/ -directory of the source code checkout. The report output ends with the .yamloo -suffix. - -## The details - -We haven't actually installed ooniprobe - we just added the ooniprobe python -to your PYTHONPATH. We also installed all of the dependencies with your native -package manager or into a local directory managed by your user. - -## ooniprobe requirements explained - -Basic system requirements: - - * Git: http://git-scm.com/book/en/Getting-Started-Installing-Git - * Python >= 2.6: http://www.python.org/download/releases/ - * pip: http://www.pip-installer.org/en/latest/ - - -## The more detailed way follows - -On Debian or Ubuntu GNU/Linux based systems these can be installed with: - -``` -sudo apt-get install git-core python python-pip python-dev build-essential tor tor-geoipdb tcpdump python-geoip python-docutils python-ipaddr python-scapy python-pyrex -``` - -Other packages that may be of interest: +## Using ooniprobe

-``` -libdumbnet python-dumbnet python-libpcap python-pypcap pcapy dnspython -python-virtualenv virtualenvwrapper -``` +**Net test** is a set of measurements to assess what kind of internet censorship is occurring.

-The Python dependencies required for running ooniprobe are: +**Decks** are collections of ooniprobe nettests with precabled inputs.

- * Tor (>2.2.x): https://torproject.org/ - * Twisted (>12.1.0): https://twistedmatrix.com/trac/ - * PyYAML: http://pyyaml.org/ - * Scapy: http://www.secdev.org/projects/scapy/ - * pypcap: https://code.google.com/p/pypcap/ - * libdnet: https://code.google.com/p/libdnet/ - * txtorcon: https://github.com/meejah/txtorcon - * txsocksx: https://github.com/habnabit/txsocksx - * pygeoip: https://github.com/appliedsec/pygeoip - * parsley: http://launchpad.net/parsley - * docutils: http://docutils.sourceforge.net - * ipaddr: http://code.google.com/p/ipaddr-py/ - * Pyrex: http://www.cosc.canterbury.ac.nz/~greg/python/Pyrex +**Collector** is a service used to report the results of measurements.

-## Install Tor +**Test helper** is a service used by a probe for successfully performing it's measurements.

-Install the latest version of Tor for your platform: +**Bouncer** is a service used to discover the addresses of test helpers and collectors.

-[Download Tor](https://www.torproject.org/download/download.html) +### Configuring ooniprobe

-## (Optional) Install obfsproxy +You may edit the configuration for ooniprobe by editing the configuration file +found inside of `~/.ooni/ooniprobe.conf`.

-Install the latest version of obfsproxy for your platform. - -[Download Obfsproxy](https://www.torproject.org/projects/obfsproxy.html.en) - -## Configurating a virtual environment - -You are highly recommended to install python packages from inside of a virtual -environment, since pip does not download the packages via SSL and you will need -to install it system wide. - -This will require you to have installed virtualenv. - -``` -sudo apt-get install python-virtualenv virtualenvwrapper -``` - -To create a new virtual environment do - -``` -mkdir $HOME/.virtualenvs -mkvirtualenv ooni-probe -``` - -You will automatically enter the environment. To re-enter this environment in the future, type: - -``` -workon ooni-probe -``` - -For convenience, you may want to add the following to your .bashrc: - -``` -if [ -e ~/ooni-probe/bin ]; then - export PATH=~/ooni-probe/bin:$PATH -fi -if [ -e ~/ooni-probe ]; then - export PYTHONPATH=$PYTHONPATH:~/ooni-probe -fi -``` - -Add the following to $HOME/.virtualenvs/ooni-probe/bin/postactivate to automatically cd into the working directory upon activation. - -``` -if [ -e ~/ooni-probe ] ; then - cd ~/ooni-probe -fi -``` - -## Installing ooni-probe - -Clone the ooniprobe repository: - -``` -git clone https://git.torproject.org/ooni-probe.git -cd ooni-probe -``` - -Then install OONI with: - -``` -pip install -r requirements.txt -``` - -If you are not in a virtualenv you will have to run the above command as root: - -``` -sudo pip install -r requirements.txt -``` - -## Install libdnet and pypcap python bindings - -It's ideal to install these manually since the ones in debian or ubuntu are not -up to date. - -The version of pypcap and libdnet ooniprobe is current tested with are -libdnet-1.12 and pypcap 1.1, any other version should be considered untested. +By default ooniprobe will not include personal identifying information in the +test result, nor create a pcap file. This behavior can be personalized.

-For libdnet: +### Running decks

-``` -wget https://libdnet.googlecode.com/files/libdnet-1.12.tgz -tar xzf libdnet-1.12.tgz -cd libdnet-1.12 -./configure && make -cd python/ -python setup.py install -cd ../../ && rm -rf libdnet-1.12* -``` +You will find all the installed decks inside of `/usr/share/ooni/decks`.

-For pypcap: +You may then run a deck by using the command line option `-i`:

``` -git clone https://github.com/hellais/pypcap -cd pypcap/ -pip install pyrex -python setup.py install -cd ../ && rm -rf pypcap +ooniprobe -i /usr/share/decks/basic.deck ```

-## Including your geo data in the test report +### Running net tests

-Including geografical information on where your probe is located helps us -better assess the value of the test. You can personalize these setting from -inside of ooniprobe.conf - -If you wish to include geografical data in the test report, you will have to go -to the data/ directory and run: +You may list all the installed stable net tests with:

``` -make geoip +ooniprobe -s ```

-Then edit your ooniprobe.conf to point to the absolute path of where the data/ -directory is located for example: +You may then run a nettest by specifying it's name for example:

``` -geoip_data_dir: /home/your_user/ooni-probe/data/ +ooniprobe manipulation/http_header_field_manipulation ```

-## Running some tests - -To see the possible command line options run: +It is also possible to specify inputs to tests as URLs:

``` -./bin/ooniprobe --help +ooniprobe blocking/http_requests -f httpo://ihiderha53f36lsd.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1 ```

-For interesting tests to run look in the nettests/core/ directory. - -To run a test you can do so with: +You can find the result of the test in your current working directory.

-``` -./bin/ooniprobe -o report_file_name path/to/test.py -``` +By default the report result will be collected by the default ooni collector +and the addresses of test helpers will be obtained from the default bouncer.

-Normally tests take options, you can see them with: +You may also specify your own collector or bouncer with the options `-c` and +`-b`.

-``` -./bin/ooniprobe -o report_file_name path/to/test.py --help -``` +## (Optional) Install obfsproxy

-## Configuration +Install the latest version of obfsproxy for your platform.

-By default ooniprobe will not include personal identifying information in the -test result, nor create a pcap file. This behavior can be personalized by -editing your ooniprobe.conf configuration file. +[Download Obfsproxy](https://www.torproject.org/projects/obfsproxy.html.en)

## Bridges and obfsproxy bridges

@@ -314,4 +165,3 @@ If your distributation supports capabilities you can avoid needing to run OONI a ``` setcap cap_net_admin,cap_net_raw+eip /path/to/your/virtualenv's/python ``` - diff --git a/Vagrantfile b/Vagrantfile index a4e9fbe..9a9e490 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -18,10 +18,6 @@ export USE_VIRTUALENV=0 ./setup-dependencies.sh -y python setup.py install

-cd /usr/share/ooni/ -echo "[+] Building geoip stuff.." -make geoip 2>&1 > /dev/null - mkdir -p ~/.ooni cp /usr/share/ooni/ooniprobe.conf.sample ~/.ooni/ooniprobe.conf

@@ -31,12 +27,8 @@ make lists 2>&1 > /dev/null # https://code.google.com/p/pypcap/issues/detail?id=27 # pip install pydnet pypcap

-/ooni/bin/oonid -echo "You may now visit http://localhost:8042/ to start running some ooniprobe tests" -echo "" -echo "Or if you are a bit more h4x0r you can ssh into the box and use the ooniprobe CLI" echo "Login using 'vagrant ssh', and dont forget to run ooniprobe as root." -echo "First run: 'sudo su; cd /usr/share/ooni; ooniprobe -i decks/before_i_commit.testdeck'" +echo "First run: 'sudo su; cd /usr/share/ooni; ooniprobe -i decks/basic.testdeck'"

SCRIPT

diff --git a/ooni/nettests/blocking/dnsconsistency.py b/ooni/nettests/blocking/dnsconsistency.py index 8c02841..f10a48c 100644 --- a/ooni/nettests/blocking/dnsconsistency.py +++ b/ooni/nettests/blocking/dnsconsistency.py @@ -37,7 +37,7 @@ class UsageOptions(usage.Options): class DNSConsistencyTest(dnst.DNSTest):

name = "DNS Consistency" - description = "DNS censorship detection test" + description = "Checks to see if the DNS responses from a set of DNS resolvers are consistent." version = "0.6" authors = "Arturo Filastò, Isis Lovecruft" requirements = None diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py index c3c72af..790b554 100644 --- a/ooni/nettests/blocking/http_requests.py +++ b/ooni/nettests/blocking/http_requests.py @@ -28,6 +28,7 @@ class HTTPRequestsTest(httpt.HTTPTest): lengths match. """ name = "HTTP Requests Test" + description = "Performs a HTTP GET request over Tor and one over the local network and compares the two results." author = "Arturo Filastò" version = "0.2.3"

diff --git a/ooni/nettests/blocking/tcp_connect.py b/ooni/nettests/blocking/tcp_connect.py index 5b432e0..129a2d5 100644 --- a/ooni/nettests/blocking/tcp_connect.py +++ b/ooni/nettests/blocking/tcp_connect.py @@ -15,6 +15,7 @@ class TCPFactory(Factory):

class TCPConnectTest(nettest.NetTestCase): name = "TCP Connect" + description = "Performs a TCP connect scan of all the host port combinations given as input." author = "Arturo Filastò" version = "0.1" inputFile = ['file', 'f', None, diff --git a/ooni/nettests/manipulation/daphne.py b/ooni/nettests/manipulation/daphne.py index 09279fa..936111a 100644 --- a/ooni/nettests/manipulation/daphne.py +++ b/ooni/nettests/manipulation/daphne.py @@ -63,6 +63,7 @@ class daphn3Args(usage.Options): class daphn3Test(nettest.NetTestCase):

name = "Daphn3" + description = "Bisects the censors fingerprint by mutating the given input packets." usageOptions = daphn3Args inputFile = ['file', 'f', None, 'Specify the pcap or YAML file to be used as input to the test'] diff --git a/ooni/nettests/manipulation/dnsspoof.py b/ooni/nettests/manipulation/dnsspoof.py index 58465d7..82fb325 100644 --- a/ooni/nettests/manipulation/dnsspoof.py +++ b/ooni/nettests/manipulation/dnsspoof.py @@ -23,6 +23,7 @@ class UsageOptions(usage.Options):

class DNSSpoof(scapyt.ScapyTest): name = "DNS Spoof" + description = "Used to validate if the type of censorship happening is DNS spoofing or not." author = "Arturo Filastò" version = "0.0.1" timeout = 2 diff --git a/ooni/nettests/manipulation/http_header_field_manipulation.py b/ooni/nettests/manipulation/http_header_field_manipulation.py index 49bc98c..e059c42 100644 --- a/ooni/nettests/manipulation/http_header_field_manipulation.py +++ b/ooni/nettests/manipulation/http_header_field_manipulation.py @@ -42,6 +42,7 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): the ones we sent, then we have detected tampering. """ name = "HTTP Header Field Manipulation" + description = "Checks if the HTTP request the server sees is the same as the one that the client has created." author = "Arturo Filastò" version = "0.1.3"

diff --git a/ooni/nettests/manipulation/http_host.py b/ooni/nettests/manipulation/http_host.py index 41b21f1..4adb1d3 100644 --- a/ooni/nettests/manipulation/http_host.py +++ b/ooni/nettests/manipulation/http_host.py @@ -34,6 +34,7 @@ class HTTPHost(httpt.HTTPTest): got back matches the input block page. """ name = "HTTP Host" + description = "Tests a variety of different filter bypassing techniques based on the HTTP Host header field." author = "Arturo Filastò" version = "0.2.3"

@@ -111,7 +112,7 @@ class HTTPHost(httpt.HTTPTest): with a rule that will allow to detect that the body of the result is that of a censored site. """ - # If we don't see a json array we know that something is wrong for + # If we don't see a json dict we know that something is wrong for # sure if not body.startswith("{"): log.msg("This does not appear to be JSON") diff --git a/ooni/nettests/manipulation/http_invalid_request_line.py b/ooni/nettests/manipulation/http_invalid_request_line.py index 8ba124e..2c99760 100644 --- a/ooni/nettests/manipulation/http_invalid_request_line.py +++ b/ooni/nettests/manipulation/http_invalid_request_line.py @@ -20,6 +20,7 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): ascii letters or numbers ('XxXx' will be 4). """ name = "HTTP Invalid Request Line" + description = "Performs out of spec HTTP requests in the attempt to trigger a proxy error message." version = "0.2" authors = "Arturo Filastò"

diff --git a/ooni/nettests/manipulation/traceroute.py b/ooni/nettests/manipulation/traceroute.py index aaa977a..7639060 100644 --- a/ooni/nettests/manipulation/traceroute.py +++ b/ooni/nettests/manipulation/traceroute.py @@ -22,6 +22,7 @@ class UsageOptions(usage.Options):

class TracerouteTest(scapyt.BaseScapyTest): name = "Multi Protocol Traceroute Test" + description = "Performs a UDP, TCP, ICMP traceroute with destination port number set to 0, 22, 23, 53, 80, 123, 443, 8080 and 65535" author = "Arturo Filastò" version = "0.2"