commit 49dee949de6faf031db5ae2510f9ff67e9d6768a Author: Arturo Filastò <art@fuffa.org> Date: Sat May 2 16:51:35 2015 +0200 Documentation related fixes * Indentation fixes * Move examples to examples directory. --- data/nettests/examples/example_dns_http.py | 11 ------ data/nettests/examples/example_dnst.py | 13 ------- data/nettests/examples/example_http_checksum.py | 27 --------------- data/nettests/examples/example_httpt.py | 36 -------------------- data/nettests/examples/example_myip.py | 21 ------------ data/nettests/examples/example_process.py | 10 ------ data/nettests/examples/example_scapyt.py | 29 ---------------- data/nettests/examples/example_scapyt_yield.py | 25 -------------- data/nettests/examples/example_simple.py | 8 ----- data/nettests/examples/example_tcpt.py | 21 ------------ docs/source/api/ooni.rst | 23 ++----------- docs/source/index.rst | 16 ++++----- ooni/nettests/examples/example_dns_http.py | 11 ++++++ ooni/nettests/examples/example_dnst.py | 13 +++++++ ooni/nettests/examples/example_http_checksum.py | 27 +++++++++++++++ ooni/nettests/examples/example_httpt.py | 36 ++++++++++++++++++++ ooni/nettests/examples/example_myip.py | 21 ++++++++++++ ooni/nettests/examples/example_process.py | 10 ++++++ ooni/nettests/examples/example_scapyt.py | 29 ++++++++++++++++ ooni/nettests/examples/example_scapyt_yield.py | 25 ++++++++++++++ ooni/nettests/examples/example_simple.py | 8 +++++ ooni/nettests/examples/example_simple_post.py | 11 ++++++ ooni/nettests/examples/example_tcpt.py | 21 ++++++++++++ ooni/nettests/manipulation/captiveportal.py | 1 - .../manipulation/http_header_field_manipulation.py | 1 + 25 files changed, 224 insertions(+), 230 deletions(-) diff --git a/data/nettests/examples/__init__.py b/data/nettests/examples/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/data/nettests/examples/example_dns_http.py b/data/nettests/examples/example_dns_http.py deleted file mode 100644 index 9b76775..0000000 --- a/data/nettests/examples/example_dns_http.py +++ /dev/null @@ -1,11 +0,0 @@ -from twisted.internet import defer -from ooni.templates import httpt, dnst - -class TestDNSandHTTP(httpt.HTTPTest, dnst.DNSTest): - - @defer.inlineCallbacks - def test_http_and_dns(self): - yield self.doRequest('http://torproject.org') - yield self.performALookup('torproject.org', ('8.8.8.8', 53)) - - diff --git a/data/nettests/examples/example_dnst.py b/data/nettests/examples/example_dnst.py deleted file mode 100644 index 6905637..0000000 --- a/data/nettests/examples/example_dnst.py +++ /dev/null @@ -1,13 +0,0 @@ -from ooni.templates import dnst - -class ExampleDNSTest(dnst.DNSTest): - inputFile = ['file', 'f', None, 'foobar'] - - def test_a_lookup(self): - def gotResult(result): - # Result is an array containing all the A record lookup results - print result - - d = self.performALookup('torproject.org', ('8.8.8.8', 53)) - d.addCallback(gotResult) - return d diff --git a/data/nettests/examples/example_http_checksum.py b/data/nettests/examples/example_http_checksum.py deleted file mode 100644 index 9226b52..0000000 --- a/data/nettests/examples/example_http_checksum.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Aaron Gibson -# :licence: see LICENSE - -from ooni.utils import log -from ooni.templates import httpt -from hashlib import sha256 - -class SHA256HTTPBodyTest(httpt.HTTPTest): - name = "ChecksumHTTPBodyTest" - author = "Aaron Gibson" - version = 0.1 - - inputFile = ['file', 'f', None, - 'List of URLS to perform GET requests to'] - - def test_http(self): - if self.input: - url = self.input - return self.doRequest(url) - else: - raise Exception("No input specified") - - def processResponseBody(self, body): - body_sha256sum = sha256(body).digest() - self.report['checksum'] = body_sha256sum diff --git a/data/nettests/examples/example_httpt.py b/data/nettests/examples/example_httpt.py deleted file mode 100644 index e76aed4..0000000 --- a/data/nettests/examples/example_httpt.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -from ooni.utils import log -from ooni.templates import httpt - -class ExampleHTTP(httpt.HTTPTest): - name = "Example HTTP Test" - author = "Arturo Filastò" - version = 0.1 - - inputs = ['http://google.com/', 'http://wikileaks.org/', - 'http://torproject.org/'] - - def test_http(self): - if self.input: - url = self.input - return self.doRequest(url) - else: - raise Exception("No input specified") - - def processResponseBody(self, body): - # XXX here shall go your logic - # for processing the body - if 'blocked' in body: - self.report['censored'] = True - else: - self.report['censored'] = False - - def processResponseHeaders(self, headers): - # XXX place in here all the logic for handling the processing of HTTP - # Headers. - pass - diff --git a/data/nettests/examples/example_myip.py b/data/nettests/examples/example_myip.py deleted file mode 100644 index 70cf773..0000000 --- a/data/nettests/examples/example_myip.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -from ooni.templates import httpt -class MyIP(httpt.HTTPTest): - inputs = ['https://check.torproject.org'] - - def test_lookup(self): - return self.doRequest(self.input) - - def processResponseBody(self, body): - import re - regexp = "Your IP address appears to be: <b>(.+?)<\/b>" - match = re.search(regexp, body) - try: - self.report['myip'] = match.group(1) - except: - self.report['myip'] = None - diff --git a/data/nettests/examples/example_process.py b/data/nettests/examples/example_process.py deleted file mode 100644 index a07f5f1..0000000 --- a/data/nettests/examples/example_process.py +++ /dev/null @@ -1,10 +0,0 @@ -from twisted.internet import defer - -from ooni.templates import process - - -class TestProcessExample(process.ProcessTest): - @defer.inlineCallbacks - def test_http_and_dns(self): - yield self.run(["echo", "Hello world!"]) - yield self.run(["sleep", "10"]) diff --git a/data/nettests/examples/example_scapyt.py b/data/nettests/examples/example_scapyt.py deleted file mode 100644 index ba04072..0000000 --- a/data/nettests/examples/example_scapyt.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :licence: see LICENSE - -from twisted.python import usage - -from scapy.all import IP, ICMP - -from ooni.templates import scapyt - -class UsageOptions(usage.Options): - optParameters = [['target', 't', '8.8.8.8', "Specify the target to ping"]] - -class ExampleICMPPingScapy(scapyt.BaseScapyTest): - name = "Example ICMP Ping Test" - - usageOptions = UsageOptions - - def test_icmp_ping(self): - def finished(packets): - print packets - answered, unanswered = packets - for snd, rcv in answered: - rcv.show() - - packets = IP(dst=self.localOptions['target'])/ICMP() - d = self.sr(packets) - d.addCallback(finished) - return d diff --git a/data/nettests/examples/example_scapyt_yield.py b/data/nettests/examples/example_scapyt_yield.py deleted file mode 100644 index 311b5aa..0000000 --- a/data/nettests/examples/example_scapyt_yield.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :licence: see LICENSE - -from twisted.python import usage -from twisted.internet import defer - -from scapy.all import IP, ICMP - -from ooni.templates import scapyt - -class UsageOptions(usage.Options): - optParameters = [['target', 't', self.localOptions['target'], "Specify the target to ping"]] - -class ExampleICMPPingScapyYield(scapyt.BaseScapyTest): - name = "Example ICMP Ping Test" - - usageOptions = UsageOptions - - @defer.inlineCallbacks - def test_icmp_ping(self): - packets = IP(dst=self.localOptions['target'])/ICMP() - answered, unanswered = yield self.sr(packets) - for snd, rcv in answered: - rcv.show() diff --git a/data/nettests/examples/example_simple.py b/data/nettests/examples/example_simple.py deleted file mode 100644 index 24de5a6..0000000 --- a/data/nettests/examples/example_simple.py +++ /dev/null @@ -1,8 +0,0 @@ -from twisted.internet import defer -from ooni import nettest - -class MyIP(nettest.NetTestCase): - def test_simple(self): - self.report['foobar'] = 'antani' - return defer.succeed(42) - diff --git a/data/nettests/examples/example_tcpt.py b/data/nettests/examples/example_tcpt.py deleted file mode 100644 index 613160b..0000000 --- a/data/nettests/examples/example_tcpt.py +++ /dev/null @@ -1,21 +0,0 @@ - -from twisted.internet.error import ConnectionRefusedError -from ooni.utils import log -from ooni.templates import tcpt - -class ExampleTCPT(tcpt.TCPTest): - def test_hello_world(self): - def got_response(response): - print "Got this data %s" % response - - def connection_failed(failure): - failure.trap(ConnectionRefusedError) - print "Connection Refused" - - self.address = "127.0.0.1" - self.port = 57002 - payload = "Hello World!\n\r" - d = self.sendPayload(payload) - d.addErrback(connection_failed) - d.addCallback(got_response) - return d diff --git a/docs/source/api/ooni.rst b/docs/source/api/ooni.rst index 3ff1ddb..10aea40 100644 --- a/docs/source/api/ooni.rst +++ b/docs/source/api/ooni.rst @@ -1,14 +1,6 @@ Measurement Developer API ========================= -:mod:`inputunit` Module ------------------------ - -.. automodule:: ooni.inputunit - :members: - :undoc-members: - :show-inheritance: - :mod:`nettest` Module --------------------- @@ -17,10 +9,10 @@ Measurement Developer API :undoc-members: :show-inheritance: -:mod:`config` Module ---------------------- +:mod:`settings` Module +---------------------- -.. automodule:: ooni.config +.. automodule:: ooni.settings :members: :undoc-members: :show-inheritance: @@ -41,20 +33,11 @@ Measurement Developer API :undoc-members: :show-inheritance: -:mod:`runner` Module --------------------- - -.. automodule:: ooni.runner - :members: - :undoc-members: - :show-inheritance: Subpackages ----------- .. toctree:: - ooni.lib ooni.templates ooni.utils - diff --git a/docs/source/index.rst b/docs/source/index.rst index ae0db6b..b042d48 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -14,7 +14,7 @@ world. - John Gilmore; TIME magazine (6 December 1993) ooniprobe -********* +--------- ooniprobe is the first program that users run to probe their network and to collect data for the OONI project. Are you interested in testing your network @@ -403,7 +403,7 @@ can be found in the nettests/blocking and nettests/manipulation directories respectively. Content Blocking Tests -...................... +^^^^^^^^^^^^^^^^^^^^^^ * `DNSConsistency <tests/dnsconsistency.html>`_ @@ -413,7 +413,7 @@ Content Blocking Tests Traffic Manipulation Tests -.......................... +^^^^^^^^^^^^^^^^^^^^^^^^^^^ * `HTTP Invalid Request Line: <tests/http_invalid_request_line.html>`_ @@ -426,7 +426,7 @@ Traffic Manipulation Tests * `HTTP Host <tests/http_host.html>`_ Other tests -........... +^^^^^^^^^^^ We also have some other tests that are currently not fully supported or still being experimented with. @@ -447,14 +447,14 @@ Tests that involve running third party tools may be found in: <https://gitweb.torproject.org/ooni-probe.git/tree/HEAD:/ooni/nettests/third_party>`_ oonib -***** +----- This is the server side component of ooniprobe. It will store that data collected from ooniprobes and it will run a series of Test Helpers that assist `Traffic Manipulation Tests`_ in performing their measurements. Test Helpers ------------- +^^^^^^^^^^^^ The currently implemented test helpers are the following: @@ -471,7 +471,7 @@ The currently implemented test helpers are the following: <https://gitweb.torproject.org/oonib.git/blob/HEAD:/oonib/testhelpers/dns_helpers.py>`_ Threat Model -************ +------------ Our adversary is capable of doing country wide network surveillance and manipulation of network traffic. @@ -500,7 +500,7 @@ allowing the user to freely choose what threat model they wish to adere to. More developer documentation -**************************** +---------------------------- .. toctree:: :maxdepth: 2 diff --git a/ooni/nettests/examples/__init__.py b/ooni/nettests/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ooni/nettests/examples/example_dns_http.py b/ooni/nettests/examples/example_dns_http.py new file mode 100644 index 0000000..9b76775 --- /dev/null +++ b/ooni/nettests/examples/example_dns_http.py @@ -0,0 +1,11 @@ +from twisted.internet import defer +from ooni.templates import httpt, dnst + +class TestDNSandHTTP(httpt.HTTPTest, dnst.DNSTest): + + @defer.inlineCallbacks + def test_http_and_dns(self): + yield self.doRequest('http://torproject.org') + yield self.performALookup('torproject.org', ('8.8.8.8', 53)) + + diff --git a/ooni/nettests/examples/example_dnst.py b/ooni/nettests/examples/example_dnst.py new file mode 100644 index 0000000..6905637 --- /dev/null +++ b/ooni/nettests/examples/example_dnst.py @@ -0,0 +1,13 @@ +from ooni.templates import dnst + +class ExampleDNSTest(dnst.DNSTest): + inputFile = ['file', 'f', None, 'foobar'] + + def test_a_lookup(self): + def gotResult(result): + # Result is an array containing all the A record lookup results + print result + + d = self.performALookup('torproject.org', ('8.8.8.8', 53)) + d.addCallback(gotResult) + return d diff --git a/ooni/nettests/examples/example_http_checksum.py b/ooni/nettests/examples/example_http_checksum.py new file mode 100644 index 0000000..9226b52 --- /dev/null +++ b/ooni/nettests/examples/example_http_checksum.py @@ -0,0 +1,27 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Aaron Gibson +# :licence: see LICENSE + +from ooni.utils import log +from ooni.templates import httpt +from hashlib import sha256 + +class SHA256HTTPBodyTest(httpt.HTTPTest): + name = "ChecksumHTTPBodyTest" + author = "Aaron Gibson" + version = 0.1 + + inputFile = ['file', 'f', None, + 'List of URLS to perform GET requests to'] + + def test_http(self): + if self.input: + url = self.input + return self.doRequest(url) + else: + raise Exception("No input specified") + + def processResponseBody(self, body): + body_sha256sum = sha256(body).digest() + self.report['checksum'] = body_sha256sum diff --git a/ooni/nettests/examples/example_httpt.py b/ooni/nettests/examples/example_httpt.py new file mode 100644 index 0000000..e76aed4 --- /dev/null +++ b/ooni/nettests/examples/example_httpt.py @@ -0,0 +1,36 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni.utils import log +from ooni.templates import httpt + +class ExampleHTTP(httpt.HTTPTest): + name = "Example HTTP Test" + author = "Arturo Filastò" + version = 0.1 + + inputs = ['http://google.com/', 'http://wikileaks.org/', + 'http://torproject.org/'] + + def test_http(self): + if self.input: + url = self.input + return self.doRequest(url) + else: + raise Exception("No input specified") + + def processResponseBody(self, body): + # XXX here shall go your logic + # for processing the body + if 'blocked' in body: + self.report['censored'] = True + else: + self.report['censored'] = False + + def processResponseHeaders(self, headers): + # XXX place in here all the logic for handling the processing of HTTP + # Headers. + pass + diff --git a/ooni/nettests/examples/example_myip.py b/ooni/nettests/examples/example_myip.py new file mode 100644 index 0000000..70cf773 --- /dev/null +++ b/ooni/nettests/examples/example_myip.py @@ -0,0 +1,21 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni.templates import httpt +class MyIP(httpt.HTTPTest): + inputs = ['https://check.torproject.org'] + + def test_lookup(self): + return self.doRequest(self.input) + + def processResponseBody(self, body): + import re + regexp = "Your IP address appears to be: <b>(.+?)<\/b>" + match = re.search(regexp, body) + try: + self.report['myip'] = match.group(1) + except: + self.report['myip'] = None + diff --git a/ooni/nettests/examples/example_postprocessor.py b/ooni/nettests/examples/example_postprocessor.py new file mode 100644 index 0000000..e69de29 diff --git a/ooni/nettests/examples/example_process.py b/ooni/nettests/examples/example_process.py new file mode 100644 index 0000000..a07f5f1 --- /dev/null +++ b/ooni/nettests/examples/example_process.py @@ -0,0 +1,10 @@ +from twisted.internet import defer + +from ooni.templates import process + + +class TestProcessExample(process.ProcessTest): + @defer.inlineCallbacks + def test_http_and_dns(self): + yield self.run(["echo", "Hello world!"]) + yield self.run(["sleep", "10"]) diff --git a/ooni/nettests/examples/example_scapyt.py b/ooni/nettests/examples/example_scapyt.py new file mode 100644 index 0000000..ba04072 --- /dev/null +++ b/ooni/nettests/examples/example_scapyt.py @@ -0,0 +1,29 @@ +# -*- encoding: utf-8 -*- +# +# :licence: see LICENSE + +from twisted.python import usage + +from scapy.all import IP, ICMP + +from ooni.templates import scapyt + +class UsageOptions(usage.Options): + optParameters = [['target', 't', '8.8.8.8', "Specify the target to ping"]] + +class ExampleICMPPingScapy(scapyt.BaseScapyTest): + name = "Example ICMP Ping Test" + + usageOptions = UsageOptions + + def test_icmp_ping(self): + def finished(packets): + print packets + answered, unanswered = packets + for snd, rcv in answered: + rcv.show() + + packets = IP(dst=self.localOptions['target'])/ICMP() + d = self.sr(packets) + d.addCallback(finished) + return d diff --git a/ooni/nettests/examples/example_scapyt_yield.py b/ooni/nettests/examples/example_scapyt_yield.py new file mode 100644 index 0000000..311b5aa --- /dev/null +++ b/ooni/nettests/examples/example_scapyt_yield.py @@ -0,0 +1,25 @@ +# -*- encoding: utf-8 -*- +# +# :licence: see LICENSE + +from twisted.python import usage +from twisted.internet import defer + +from scapy.all import IP, ICMP + +from ooni.templates import scapyt + +class UsageOptions(usage.Options): + optParameters = [['target', 't', self.localOptions['target'], "Specify the target to ping"]] + +class ExampleICMPPingScapyYield(scapyt.BaseScapyTest): + name = "Example ICMP Ping Test" + + usageOptions = UsageOptions + + @defer.inlineCallbacks + def test_icmp_ping(self): + packets = IP(dst=self.localOptions['target'])/ICMP() + answered, unanswered = yield self.sr(packets) + for snd, rcv in answered: + rcv.show() diff --git a/ooni/nettests/examples/example_simple.py b/ooni/nettests/examples/example_simple.py new file mode 100644 index 0000000..24de5a6 --- /dev/null +++ b/ooni/nettests/examples/example_simple.py @@ -0,0 +1,8 @@ +from twisted.internet import defer +from ooni import nettest + +class MyIP(nettest.NetTestCase): + def test_simple(self): + self.report['foobar'] = 'antani' + return defer.succeed(42) + diff --git a/ooni/nettests/examples/example_simple_post.py b/ooni/nettests/examples/example_simple_post.py new file mode 100644 index 0000000..38229d7 --- /dev/null +++ b/ooni/nettests/examples/example_simple_post.py @@ -0,0 +1,11 @@ +from twisted.internet import defer +from ooni import nettest + +class MyIP(nettest.NetTestCase): + def test_simple(self): + self.report['foobar'] = 'antani' + return defer.succeed(42) + + def postProcessor(self, measurements): + print measurements + self.report['antani'] = 'testing' diff --git a/ooni/nettests/examples/example_tcpt.py b/ooni/nettests/examples/example_tcpt.py new file mode 100644 index 0000000..613160b --- /dev/null +++ b/ooni/nettests/examples/example_tcpt.py @@ -0,0 +1,21 @@ + +from twisted.internet.error import ConnectionRefusedError +from ooni.utils import log +from ooni.templates import tcpt + +class ExampleTCPT(tcpt.TCPTest): + def test_hello_world(self): + def got_response(response): + print "Got this data %s" % response + + def connection_failed(failure): + failure.trap(ConnectionRefusedError) + print "Connection Refused" + + self.address = "127.0.0.1" + self.port = 57002 + payload = "Hello World!\n\r" + d = self.sendPayload(payload) + d.addErrback(connection_failed) + d.addCallback(got_response) + return d diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index 344ae1c..f700973 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -617,7 +617,6 @@ class CaptivePortal(httpt.HTTPTest, dnst.DNSTest): Runs the CaptivePortal(Test). CONFIG OPTIONS - -------------- If "do_captive_portal_vendor_tests" is set to "true", then vendor specific captive portal HTTP-based tests will be run. diff --git a/ooni/nettests/manipulation/http_header_field_manipulation.py b/ooni/nettests/manipulation/http_header_field_manipulation.py index 4e47040..d063eed 100644 --- a/ooni/nettests/manipulation/http_header_field_manipulation.py +++ b/ooni/nettests/manipulation/http_header_field_manipulation.py @@ -122,6 +122,7 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): * **header_field_value** when the header field value does not match with the one we transmitted. + """ log.msg("Checking for tampering on %s" % self.url)