commit c195aa60653ed1ebfd9623525148ed9177e80af1 Author: Arturo Filastò art@fuffa.org Date: Thu Nov 29 19:13:24 2012 +0100
Port the test page to the index of the documentation * Only include the tests that we plan supporting in the first release --- docs/source/index.rst | 110 ++++++++++++++++++++++++++++- nettests/experimental/daphn3.py | 119 -------------------------------- nettests/manipulation/captiveportal.py | 57 ++++++++-------- nettests/manipulation/daphne.py | 119 ++++++++++++++++++++++++++++++++ oonib/testhelpers/http_helpers.py | 2 +- 5 files changed, 254 insertions(+), 153 deletions(-)
diff --git a/docs/source/index.rst b/docs/source/index.rst index 389df2a..7803782 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,19 +3,121 @@ Welcome to the OONI developer documentation ===========================================
-If you are looking for how to start running the ooniprobe tool see: +ooniprobe is tool for performing internet censorship measurements. Our goal is +to achieve a command data format and set of methodologies for conducting +censorship related research. + +If you are a user interesting in running the ooniprobe command line tool see:
https://github.com/hellais/ooni-probe#getting-started
+The two main software components of ooniprobe are ooniprobe and oonib. + +ooniprobe +********* + +Is the tool that volunteers and researches interested in contributing data to +the project should be running. + +ooniprobe allows the user to select what test should be run and what backend +should be used for storing the test report and/or assisting them in the running +of the test. + +ooniprobe tests are divided into two categories: **Traffic Manipulation** and +**Content Blocking**. + +**Traffic Manipulation** tests aim to detect the presence of some sort of +tampering with the internet traffic between the probe and a remote test helper +backend. As such they usually require the selection of a oonib backend +component for running the test. + +**Content Blocking** are aimed at enumerating the kind of content that is +blocked from the probes network point of view. As such they usually require to +have specified an input list for running the test. + +Core ooniprobe Tests +-------------------- + +The source for `Content blocking tests +https://gitweb.torproject.org/ooni-probe.git/tree/HEAD:/nettests/blocking`_ +and `Traffic Manipulation tests +https://gitweb.torproject.org/ooni-probe.git/tree/HEAD:/nettests/blocking`_ +can be found in the nettests/blocking and nettests/manipulation directories +respectively. + +Content Blocking Tests +...................... + + * ::`DNSTamper tests/dnstamper` + + * ::`HTTP Requests tests/http_requests` + + * ::`TCP Connect tests/tcpconnect` + + +Traffic Manipulation Tests +.......................... + + * ::`HTTP Invalid Request Line: tests/http_invalid_request_line` + + * ::`DNS Spoof: tests/dnsspoof` + + * ::`HTTP Header Field Manipulation: tests/http_header_field_manipulation` + + * ::`Traceroute: tests/traceroute` + + * ::`HTTP Host: tests/http_host` + + * ::`Daphne: tests/daphne` + +Other tests +........... + +We also have some other tests that are currently not fully supported or still +being experimented with. + +You can find these in ::`nettests/experimental: +https://gitweb.torproject.org/ooni-probe.git/tree/HEAD:/nettests/experimenta.... + +Tests that don't do a measurement but are useful for scanning can be found in +::`nettests/scanning: +https://gitweb.torproject.org/ooni-probe.git/tree/HEAD:/nettests/scanning%60. + +Tests that involve running third party tools may be found in +::`nettests/third_party: +https://gitweb.torproject.org/ooni-probe.git/tree/HEAD:/nettests/third_party.... + +oonib +***** + +This is the server side component of ooniprobe. It will store that data +collected from ooniprobes and it will run a series of Test Helpers that assist +`Traffic Manipulation`_ in performing their measurements. + +Test Helpers +............ + +The currently implemented test helpers are the following: + + * ::`SSL Test Helpers: + https://gitweb.torproject.org/ooni-probe.git/tree/HEAD:/oonib/testhelpers/ss.... + + * ::`HTTP Test Helpers: + https://gitweb.torproject.org/ooni-probe.git/tree/HEAD:/oonib/testhelpers/ht.... + + * ::`TCP Test Helpers: + https://gitweb.torproject.org/ooni-probe.git/tree/HEAD:/oonib/testhelpers/tc.... + + * ::`DNS Test Helpers: + https://gitweb.torproject.org/ooni-probe.git/tree/HEAD:/oonib/testhelpers/dn....
-Contents -******** +More developer documentation +*****************************
.. toctree:: :maxdepth: 2 :glob:
- tests/* oonib install writing_tests diff --git a/nettests/experimental/daphn3.py b/nettests/experimental/daphn3.py deleted file mode 100644 index 09279fa..0000000 --- a/nettests/experimental/daphn3.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- encoding: utf-8 -*- -from twisted.python import usage -from twisted.internet import protocol, endpoints, reactor - -from ooni import nettest -from ooni.kit import daphn3 -from ooni.utils import log - -class Daphn3ClientProtocol(daphn3.Daphn3Protocol): - def nextStep(self): - log.debug("Moving on to next step in the state walk") - self.current_data_received = 0 - if self.current_step >= (len(self.steps) - 1): - log.msg("Reached the end of the state machine") - log.msg("Censorship fingerpint bisected!") - step_idx, mutation_idx = self.factory.mutation - log.msg("step_idx: %s | mutation_id: %s" % (step_idx, mutation_idx)) - #self.transport.loseConnection() - if self.report: - self.report['mutation_idx'] = mutation_idx - self.report['step_idx'] = step_idx - self.d.callback(None) - return - else: - self.current_step += 1 - if self._current_step_role() == self.role: - # We need to send more data because we are again responsible for - # doing so. - self.sendPayload() - - -class Daphn3ClientFactory(protocol.ClientFactory): - protocol = daphn3.Daphn3Protocol - mutation = [0,0] - steps = None - - def buildProtocol(self, addr): - p = self.protocol() - p.steps = self.steps - p.factory = self - return p - - def startedConnecting(self, connector): - log.msg("Started connecting %s" % connector) - - def clientConnectionFailed(self, reason, connector): - log.err("We failed connecting the the OONIB") - log.err("Cannot perform test. Perhaps it got blocked?") - log.err("Please report this to tor-assistants@torproject.org") - - def clientConnectionLost(self, reason, connector): - log.err("Daphn3 client connection lost") - print reason - -class daphn3Args(usage.Options): - optParameters = [ - ['host', 'h', '127.0.0.1', 'Target Hostname'], - ['port', 'p', 57003, 'Target port number']] - - optFlags = [['pcap', 'c', 'Specify that the input file is a pcap file'], - ['yaml', 'y', 'Specify that the input file is a YAML file (default)']] - -class daphn3Test(nettest.NetTestCase): - - name = "Daphn3" - usageOptions = daphn3Args - inputFile = ['file', 'f', None, - 'Specify the pcap or YAML file to be used as input to the test'] - - #requiredOptions = ['file'] - - steps = None - - def inputProcessor(self, filename): - """ - step_idx is the step in the packet exchange - ex. - [.X.] are packets sent by a client or a server - - client: [.1.] [.3.] [.4.] - server: [.2.] [.5.] - - mutation_idx: is the sub index of the packet as in the byte of the - packet at the step_idx that is to be mutated - - """ - if self.localOptions['pcap']: - daphn3Steps = daphn3.read_pcap(filename) - else: - daphn3Steps = daphn3.read_yaml(filename) - log.debug("Loaded these steps %s" % daphn3Steps) - yield daphn3Steps - - def test_daphn3(self): - host = self.localOptions['host'] - port = int(self.localOptions['port']) - - def failure(failure): - log.msg("Failed to connect") - self.report['censored'] = True - self.report['mutation'] = 0 - raise Exception("Error in connection, perhaps the backend is censored") - return - - def success(protocol): - log.msg("Successfully connected") - protocol.sendPayload() - return protocol.d - - log.msg("Connecting to %s:%s" % (host, port)) - endpoint = endpoints.TCP4ClientEndpoint(reactor, host, port) - daphn3_factory = Daphn3ClientFactory() - daphn3_factory.steps = self.input - daphn3_factory.report = self.report - d = endpoint.connect(daphn3_factory) - d.addErrback(failure) - d.addCallback(success) - return d - diff --git a/nettests/manipulation/captiveportal.py b/nettests/manipulation/captiveportal.py index be8da27..082960e 100644 --- a/nettests/manipulation/captiveportal.py +++ b/nettests/manipulation/captiveportal.py @@ -1,34 +1,33 @@ # -*- coding: utf-8 -*- -""" - captiveportal - ************* - - This test is a collection of tests to detect the presence of a - captive portal. Code is taken, in part, from the old ooni-probe, - which was written by Jacob Appelbaum and Arturo Filastò. - - This module performs multiple tests that match specific vendor captive - portal tests. This is a basic internet captive portal filter tester written - for RECon 2011. - - Read the following URLs to understand the captive portal detection process - for various vendors: - - http://technet.microsoft.com/en-us/library/cc766017%28WS.10%29.aspx - http://blog.superuser.com/2011/05/16/windows-7-network-awareness/ - http://isc.sans.org/diary.html?storyid=10312& - http://src.chromium.org/viewvc/chrome?view=rev&revision=74608 - http://code.google.com/p/chromium-os/issues/detail?3281ttp, - http://crbug.com/52489 - http://crbug.com/71736 - https://bugzilla.mozilla.org/show_bug.cgi?id=562917 - https://bugzilla.mozilla.org/show_bug.cgi?id=603505 - http://lists.w3.org/Archives/Public/ietf-http-wg/2011JanMar/0086.html - http://tools.ietf.org/html/draft-nottingham-http-portal-02 - - :copyright: (c) 2012 Isis Lovecruft - :license: see LICENSE for more details -""" +# captiveportal +# ************* +# +# This test is a collection of tests to detect the presence of a +# captive portal. Code is taken, in part, from the old ooni-probe, +# which was written by Jacob Appelbaum and Arturo Filastò. +# +# This module performs multiple tests that match specific vendor captive +# portal tests. This is a basic internet captive portal filter tester written +# for RECon 2011. +# +# Read the following URLs to understand the captive portal detection process +# for various vendors: +# +# http://technet.microsoft.com/en-us/library/cc766017%28WS.10%29.aspx +# http://blog.superuser.com/2011/05/16/windows-7-network-awareness/ +# http://isc.sans.org/diary.html?storyid=10312& +# http://src.chromium.org/viewvc/chrome?view=rev&revision=74608 +# http://code.google.com/p/chromium-os/issues/detail?3281ttp, +# http://crbug.com/52489 +# http://crbug.com/71736 +# https://bugzilla.mozilla.org/show_bug.cgi?id=562917 +# https://bugzilla.mozilla.org/show_bug.cgi?id=603505 +# http://lists.w3.org/Archives/Public/ietf-http-wg/2011JanMar/0086.html +# http://tools.ietf.org/html/draft-nottingham-http-portal-02 +# +# :authors: Jacob Appelbaum, Arturo Filastò, Isis Lovecruft +# :license: see LICENSE for more details + import base64 import os import random diff --git a/nettests/manipulation/daphne.py b/nettests/manipulation/daphne.py new file mode 100644 index 0000000..09279fa --- /dev/null +++ b/nettests/manipulation/daphne.py @@ -0,0 +1,119 @@ +# -*- encoding: utf-8 -*- +from twisted.python import usage +from twisted.internet import protocol, endpoints, reactor + +from ooni import nettest +from ooni.kit import daphn3 +from ooni.utils import log + +class Daphn3ClientProtocol(daphn3.Daphn3Protocol): + def nextStep(self): + log.debug("Moving on to next step in the state walk") + self.current_data_received = 0 + if self.current_step >= (len(self.steps) - 1): + log.msg("Reached the end of the state machine") + log.msg("Censorship fingerpint bisected!") + step_idx, mutation_idx = self.factory.mutation + log.msg("step_idx: %s | mutation_id: %s" % (step_idx, mutation_idx)) + #self.transport.loseConnection() + if self.report: + self.report['mutation_idx'] = mutation_idx + self.report['step_idx'] = step_idx + self.d.callback(None) + return + else: + self.current_step += 1 + if self._current_step_role() == self.role: + # We need to send more data because we are again responsible for + # doing so. + self.sendPayload() + + +class Daphn3ClientFactory(protocol.ClientFactory): + protocol = daphn3.Daphn3Protocol + mutation = [0,0] + steps = None + + def buildProtocol(self, addr): + p = self.protocol() + p.steps = self.steps + p.factory = self + return p + + def startedConnecting(self, connector): + log.msg("Started connecting %s" % connector) + + def clientConnectionFailed(self, reason, connector): + log.err("We failed connecting the the OONIB") + log.err("Cannot perform test. Perhaps it got blocked?") + log.err("Please report this to tor-assistants@torproject.org") + + def clientConnectionLost(self, reason, connector): + log.err("Daphn3 client connection lost") + print reason + +class daphn3Args(usage.Options): + optParameters = [ + ['host', 'h', '127.0.0.1', 'Target Hostname'], + ['port', 'p', 57003, 'Target port number']] + + optFlags = [['pcap', 'c', 'Specify that the input file is a pcap file'], + ['yaml', 'y', 'Specify that the input file is a YAML file (default)']] + +class daphn3Test(nettest.NetTestCase): + + name = "Daphn3" + usageOptions = daphn3Args + inputFile = ['file', 'f', None, + 'Specify the pcap or YAML file to be used as input to the test'] + + #requiredOptions = ['file'] + + steps = None + + def inputProcessor(self, filename): + """ + step_idx is the step in the packet exchange + ex. + [.X.] are packets sent by a client or a server + + client: [.1.] [.3.] [.4.] + server: [.2.] [.5.] + + mutation_idx: is the sub index of the packet as in the byte of the + packet at the step_idx that is to be mutated + + """ + if self.localOptions['pcap']: + daphn3Steps = daphn3.read_pcap(filename) + else: + daphn3Steps = daphn3.read_yaml(filename) + log.debug("Loaded these steps %s" % daphn3Steps) + yield daphn3Steps + + def test_daphn3(self): + host = self.localOptions['host'] + port = int(self.localOptions['port']) + + def failure(failure): + log.msg("Failed to connect") + self.report['censored'] = True + self.report['mutation'] = 0 + raise Exception("Error in connection, perhaps the backend is censored") + return + + def success(protocol): + log.msg("Successfully connected") + protocol.sendPayload() + return protocol.d + + log.msg("Connecting to %s:%s" % (host, port)) + endpoint = endpoints.TCP4ClientEndpoint(reactor, host, port) + daphn3_factory = Daphn3ClientFactory() + daphn3_factory.steps = self.input + daphn3_factory.report = self.report + d = endpoint.connect(daphn3_factory) + d.addErrback(failure) + d.addCallback(success) + return d + diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py index 6415291..3a76b9a 100644 --- a/oonib/testhelpers/http_helpers.py +++ b/oonib/testhelpers/http_helpers.py @@ -26,7 +26,7 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin): The returned JSON dict looks like so:
{ - 'request_headers': + 'request_headers': [['User-Agent', 'IE6'], ['Content-Length', 200]] 'request_line': 'GET / HTTP/1.1'