- tor-commits - lists.torproject.org

r25922: {website} fix mistaken \ breaking a comment in the wml. (website/trunk/projects/en)
by Andrew Lewman 29 Nov '12

29 Nov '12

Author: phobos Date: 2012-11-29 16:21:07 +0000 (Thu, 29 Nov 2012) New Revision: 25922 Modified: website/trunk/projects/en/projects.wml Log: fix mistaken \ breaking a comment in the wml. Modified: website/trunk/projects/en/projects.wml =================================================================== --- website/trunk/projects/en/projects.wml 2012-11-28 15:45:11 UTC (rev 25921) +++ website/trunk/projects/en/projects.wml 2012-11-29 16:21:07 UTC (rev 25922) @@ -1,4 +1,4 @@ -/## translation metadata +## translation metadata # Revision: $Revision$ # Translation-Priority: 3-low

1 0

[translation/liveusb-creator] Update translations for liveusb-creator
by translation＠torproject.org 29 Nov '12

29 Nov '12

commit b928b112cdef9501d52a7407f9b8d84ed1421e8d Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 29 16:15:55 2012 +0000 Update translations for liveusb-creator --- eu/eu.po | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eu/eu.po b/eu/eu.po index fa101de..a24e6c5 100644 --- a/eu/eu.po +++ b/eu/eu.po @@ -10,8 +10,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-04-13 14:16+0200\n" -"PO-Revision-Date: 2012-11-23 18:31+0000\n" -"Last-Translator: totorika93 <totorika93(a)gmail.com>\n" +"PO-Revision-Date: 2012-11-29 16:07+0000\n" +"Last-Translator: Antxon Baldarra <baldarra(a)euskalerria.org>\n" "Language-Team: LANGUAGE <LL(a)li.org>\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -38,7 +38,7 @@ msgstr "Gailua txikiegia da: gutxienez %s MiBekoa izan behar da." msgid "" "Error: Cannot set the label or obtain the UUID of your device. Unable to " "continue." -msgstr "" +msgstr "Akatsa: Ezin da etiketa ezarri edo zure gailuaren UUIDa eskuratu. Ezin da jarraitu." #: ../liveusb/gui.py:276 #, python-format @@ -54,11 +54,11 @@ msgid "" "Warning: This tool needs to be run as an Administrator. To do this, right " "click on the icon and open the Properties. Under the Compatibility tab, " "check the \"Run this program as an administrator\" box." -msgstr "" +msgstr "Abisua: Tresna hau Administratzaile bezala exekutatu behar da. Hau egiteko, sakatu eskuinarekin ikonoan eta ireki Propietateak. Bateragarritasuna fitxan, markatu \"Programa hau administratzale bezala exekutatu\" kutxa." #: ../liveusb/gui.py:413 msgid "Unable to find any USB drive" -msgstr "" +msgstr "Ezin da USB gailurik bilatu." #: ../liveusb/gui.py:521 msgid "" @@ -73,7 +73,7 @@ msgstr "%(device)s gailuan ez dago leku askerik." #: ../liveusb/gui.py:533 msgid "Partition is FAT16; Restricting overlay size to 2G" -msgstr "" +msgstr "Partizioa FAT16 da; Gainjarri neurria 2Gtara mugatzen" #: ../liveusb/gui.py:577 msgid "Persistent Storage" @@ -235,7 +235,7 @@ msgstr "" #: ../liveusb/creator.py:497 msgid "Removing existing Live OS" -msgstr "" +msgstr "Dagoen Live OS kentzen" #: ../liveusb/creator.py:506 ../liveusb/creator.py:517 #, python-format

1 0

[ooni-probe/master] Write documentation for HTTP Invalid Requests test
by art＠torproject.org 29 Nov '12

29 Nov '12

commit d67eecfd05f03002e3a9334db82ab2d8c2658368 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 29 17:09:34 2012 +0100 Write documentation for HTTP Invalid Requests test --- docs/source/tests/http_invalid_requests.rst | 164 ++++++++++++++++++++ nettests/manipulation/http_invalid_request_line.py | 19 ++- ooni/templates/tcpt.py | 2 +- requirements.txt | 2 +- 4 files changed, 177 insertions(+), 10 deletions(-) diff --git a/docs/source/tests/http_invalid_requests.rst b/docs/source/tests/http_invalid_requests.rst new file mode 100644 index 0000000..f14a18d --- /dev/null +++ b/docs/source/tests/http_invalid_requests.rst @@ -0,0 +1,164 @@ +Details +======= + +*Test Name*: HTTP Invalid Request Line + +*Current version*: 0.1.3 + +*NetTest*: HTTP Invalid Request Line (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/nettests/manipulati…) + +*Test Helper*: TCPEchoHelper (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/oonib/testhelpers/t…) + +*Test Type*: Traffic Manipulation + +*Requires Root*: No + +*WARNING*: This test is more dangerous to run than any other one and you +should do it only if you know what you are doing. + +Description +=========== + +The goal of this test is to do some very basic and not very noisy fuzzing +on the HTTP request line. We generate a series of requests that are not +valid HTTP requests. + +The remote backend runs a TCP echo server. If the response from the backend +does not match with what we have sent then we say that tampering is occuring. + +The idea behind this is that certain transparent HTTP proxies may not be +properly parsing the HTTP request line. + +Unless elsewhere stated 'Xx'\*N refers to N\*2 random upper or lowercase ascii +letters or numbers ('XxXx' will be 4). + +What we fuzz is the following: + +test random invalid method +************************** + +This sends random 4 letter HTTP request method. + +The request on the wire will look like this: + +:: + XxXxX / HTTP/1.1\n\r + + +test random invalid field count +****************************** + +This generates a request that looks like this: + +:: + XxXxX XxXxX XxXxX XxXxX + + This may trigger some bugs in the HTTP parsers of transparent HTTP + proxies. + + +test random big request method +****************************** + +This generates a request that looks like this: + +:: + + Xx*512 / HTTP/1.1 + + +test random invalid version number +********************************** + +This generates a request that looks like this: + +:: + GET / HTTP/XxX + +This attemps to trigger bugs in the parsing of the HTTP version number, that +is usually being split on the `.`. + +How to run the test +=================== + +`./bin/ooniprobe nettests/manipulation/http_invalid_request_line.py -b <address of backend>` + +*address of the backend* is the hostname or IP address of a backend that runs +a TCP echo server on port 80. + +Sample report +============= + +From running: + +`./bin/ooniprobe nettests/manipulation/http_invalid_request_line.py -b 127.0.0.1 -p 57002` + +:: + + ########################################### + # OONI Probe Report for HTTP Invalid Request Line test + # Thu Nov 29 16:41:15 2012 + ########################################### + --- + options: + collector: null + help: 0 + logfile: null + pcapfile: null + reportfile: null + resume: 0 + subargs: [-b, 127.0.0.1, -p, '57002'] + test: nettests/manipulation/http_invalid_request_line.py + probe_asn: null + probe_cc: null + probe_ip: 127.0.0.1 + software_name: ooniprobe + software_version: 0.0.7.1-alpha + start_time: 1354200075.0 + test_name: HTTP Invalid Request Line + test_version: 0.1.3 + ... + --- + input: null + report: + received: ["L3F6 / HTTP/1.1\n\r"] + sent: ["L3F6 / HTTP/1.1\n\r"] + tampering: false + test_name: test_random_invalid_method + test_runtime: 5.011919021606445 + test_started: 1354203675.481258 + ... + --- + input: null + report: + received: ["GET / HTTP/Ayo\n\r"] + sent: ["GET / HTTP/Ayo\n\r"] + tampering: false + test_name: test_random_invalid_version_number + test_runtime: 5.019288063049316 + test_started: 1354203675.48221 + ... + --- + input: null + report: + received: ["RZRcu OyLtu Jtu2T cs0ER\n\r"] + sent: ["RZRcu OyLtu Jtu2T cs0ER\n\r"] + tampering: false + test_name: test_random_invalid_field_count + test_runtime: 5.022854804992676 + test_started: 1354203675.483127 + ... + --- + input: null + report: + received: ["iB2HrwJeB512y5CrAAaIpETZ5RMprmGO4RCex24Kmqkjguc2XsOrGXR3qJIERw1IX13uGWVs1kOd96Y0zsR4ufcktGFnP0gYakTq7GA63rNNOmlG9tNXABnBEHfkHYhrwdrewAKdqZ2lGls66NBY7fbL9xkOsHBjXq7TkvS8MOeTb74wcxKymp1NLOa8u7C8XP9qpafQtWBrC4dkEVjppjlFyetg1tu8zomDUCBx9y2tB411d8lw7WOSMfDiQWG327aVanxaOj9WgZ2u2eu0595UiJxZxMk1LYa96vHvmB1DrX0DoJUkUg2fcEOlia3pVoFfcpbd3TQ0GwEoewU3F48Qvpf2AuOIcPgrYa1XszLCyUoToygc5J6WgIH7f1phsEpmmZ6my9KChZdnazc7mNbCwLB3Z1wMwcoFW1XuDvAhTr8OYoY17360SYkAqEBhYh8uiD4xdIq0T0KJzsJW5wghjCMjRjFyfk3wyDXaPLp7vkeeqbA3FNFatFQlUCTIkqqqM3EjAojynvCDVyrThGlmsauS9Ejhc9TaDojeT3s8HZY3KIDHwCRvpGgpwFie2NyhzmwY2y4YFMcnGXT2jlwsVE1K2fZZ7yGbKqbciuh5Q32JNvQBdGy3N63PWyZIxWhfythHOVBE9GXAr99DqEXUeiRkfNer5aVRt87rDBuLx13IrbCE6runHdoZEq0iXs7IySfw5qqt00fExbaN9UlSDyxyYIrIymrSTSUWWprTqiezn1toQZpxEl53pv4ZnLx00CSMYxWTzMDNSHLpovdAI9A27ugY2uHmPETKHbaLsTzNRtNiZAS9WidDi0oAxdICB47dnJA2CXvBaXeq2nsRasgQ9qqNl0RbTC8vhXgIq8nZZQn0iehXsP5mPXshpALyfAhoLwiChRif8FODkKdAgciLviTvyadpecb6HqRVfM8rLzMPkuXT2I21yB7NVQQLraJOeDO TvmtsV2gw2I2ON8xKtHO78VQfo357yBryndTQTRTk9FPnijwcwusn\ + \ / HTTP/1.1\n\r"] + sent: ["iB2HrwJeB512y5CrAAaIpETZ5RMprmGO4RCex24Kmqkjguc2XsOrGXR3qJIERw1IX13uGWVs1kOd96Y0zsR4ufcktGFnP0gYakTq7GA63rNNOmlG9tNXABnBEHfkHYhrwdrewAKdqZ2lGls66NBY7fbL9xkOsHBjXq7TkvS8MOeTb74wcxKymp1NLOa8u7C8XP9qpafQtWBrC4dkEVjppjlFyetg1tu8zomDUCBx9y2tB411d8lw7WOSMfDiQWG327aVanxaOj9WgZ2u2eu0595UiJxZxMk1LYa96vHvmB1DrX0DoJUkUg2fcEOlia3pVoFfcpbd3TQ0GwEoewU3F48Qvpf2AuOIcPgrYa1XszLCyUoToygc5J6WgIH7f1phsEpmmZ6my9KChZdnazc7mNbCwLB3Z1wMwcoFW1XuDvAhTr8OYoY17360SYkAqEBhYh8uiD4xdIq0T0KJzsJW5wghjCMjRjFyfk3wyDXaPLp7vkeeqbA3FNFatFQlUCTIkqqqM3EjAojynvCDVyrThGlmsauS9Ejhc9TaDojeT3s8HZY3KIDHwCRvpGgpwFie2NyhzmwY2y4YFMcnGXT2jlwsVE1K2fZZ7yGbKqbciuh5Q32JNvQBdGy3N63PWyZIxWhfythHOVBE9GXAr99DqEXUeiRkfNer5aVRt87rDBuLx13IrbCE6runHdoZEq0iXs7IySfw5qqt00fExbaN9UlSDyxyYIrIymrSTSUWWprTqiezn1toQZpxEl53pv4ZnLx00CSMYxWTzMDNSHLpovdAI9A27ugY2uHmPETKHbaLsTzNRtNiZAS9WidDi0oAxdICB47dnJA2CXvBaXeq2nsRasgQ9qqNl0RbTC8vhXgIq8nZZQn0iehXsP5mPXshpALyfAhoLwiChRif8FODkKdAgciLviTvyadpecb6HqRVfM8rLzMPkuXT2I21yB7NVQQLraJOeDOTvmt sV2gw2I2ON8xKtHO78VQfo357yBryndTQTRTk9FPnijwcwusn\ + \ / HTTP/1.1\n\r"] + tampering: false + test_name: test_random_big_request_method + test_runtime: 5.026142120361328 + test_started: 1354203675.484211 + ... + + diff --git a/nettests/manipulation/http_invalid_request_line.py b/nettests/manipulation/http_invalid_request_line.py index 00ab24d..01626ab 100644 --- a/nettests/manipulation/http_invalid_request_line.py +++ b/nettests/manipulation/http_invalid_request_line.py @@ -1,12 +1,14 @@ # -*- encoding: utf-8 -*- from twisted.python import usage -from ooni.utils import randomStr +from ooni.utils import log +from ooni.utils import randomStr, randomSTR from ooni.templates import tcpt class UsageOptions(usage.Options): optParameters = [['backend', 'b', '127.0.0.1', - 'The OONI backend that runs a TCP echo server']] + 'The OONI backend that runs a TCP echo server'], + ['backendport', 'p', 80, 'Specify the port that the TCP echo server is running (should only be set for debugging)']] class HTTPInvalidRequestLine(tcpt.TCPTest): """ @@ -14,10 +16,10 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): on the HTTP request line. We generate a series of requests that are not valid HTTP requests. - Unless elsewhere stated 'Xx'*N refers to N*2 random upper or lowercase ascii - letters or numbers ('XxXx' will be 4). + Unless elsewhere stated 'Xx'*N refers to N*2 random upper or lowercase + ascii letters or numbers ('XxXx' will be 4). """ - name = "HTTP Invalid Requests" + name = "HTTP Invalid Request Line" version = "0.1.3" authors = "Arturo Filastò" @@ -28,14 +30,15 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): requiredOptions = ['backend'] def setUp(self): - self.port = 80 + self.port = int(self.localOptions['backendport']) self.address = self.localOptions['backend'] def check_for_manipulation(self, response, payload): + log.debug("Checking if %s == %s" % (response, payload)) if response != payload: self.report['tampering'] = True else: - self.report['tampering'] = 'unknown' + self.report['tampering'] = False def test_random_invalid_method(self): """ @@ -57,7 +60,7 @@ class HTTPInvalidRequestLine(tcpt.TCPTest): Proxy-Connection: close """ - payload = randomStr(10) + " / HTTP/1.1\n\r" + payload = randomSTR(4) + " / HTTP/1.1\n\r" d = self.sendPayload(payload) d.addCallback(self.check_for_manipulation, payload) diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py index 26f38ed..323ead8 100644 --- a/ooni/templates/tcpt.py +++ b/ooni/templates/tcpt.py @@ -68,7 +68,7 @@ class TCPTest(NetTestCase): self.report['received'].append(proto.received_data) proto.transport.loseConnection() log.debug("Closing connection") - d1.callback(self.report['received']) + d1.callback(proto.received_data) def timedOut(proto): self.report['failure'] = 'tcp_timed_out_error' diff --git a/requirements.txt b/requirements.txt index 4ddb915..ce2e721 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ repoze.sphinx.autointerface>=0.7.1 https://hg.secdev.org/scapy/archive/tip.zip#egg=scapy storm>=0.19 transaction>=1.3.0 -txtorcon +txtorcon>=0.7 wsgiref>=0.1.2 zope.component>=4.0.0 zope.event>=4.0.0

1 0

[ooni-probe/master] Make getDiff take as required argument a TrueHeaders object instead of a dict
by art＠torproject.org 29 Nov '12

29 Nov '12

commit a3025bf24be234c6fbb6b9fad251ad5fa0ef4e6f Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 29 13:13:46 2012 +0100 Make getDiff take as required argument a TrueHeaders object instead of a dict --- ooni/templates/httpt.py | 3 ++- ooni/utils/txagentwithsocks.py | 20 +++++++++++++------- tests/test_trueheaders.py | 8 ++++---- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index dcd8ff9..804a3e4 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -115,9 +115,10 @@ class HTTPTest(NetTestCase): failure (instance): An instance of :class:twisted.internet.failure.Failure """ log.debug("Adding %s to report" % request) + request_headers = TrueHeaders(request['headers']) request_response = { 'request': { - 'headers': request['headers'], + 'headers': list(request_headers.getAllRawHeaders()), 'body': request['body'], 'url': request['url'], 'method': request['method'] diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py index ffaa733..7c34288 100644 --- a/ooni/utils/txagentwithsocks.py +++ b/ooni/utils/txagentwithsocks.py @@ -149,18 +149,25 @@ class TrueHeaders(http_headers.Headers): self._rawHeaders[name.lower()]['name'] = name self._rawHeaders[name.lower()]['values'] = values - def getDiff(self, header_dict, ignore=[]): + def getDiff(self, headers, ignore=[]): """ - ignore: specify a list of header fields to ignore - Returns a set containing the header names that are not present in - header_dict or not present in self. + Args: + + headers: a TrueHeaders object + + ignore: specify a list of header fields to ignore + + Returns: + + a set containing the header names that are not present in + header_dict or not present in self. """ diff = set() field_names = [] headers_a = copy(self) - headers_b = TrueHeaders(header_dict) + headers_b = copy(headers) for name in ignore: try: del headers_a._rawHeaders[name.lower()] @@ -176,8 +183,7 @@ class TrueHeaders(http_headers.Headers): field_names.append(k) for name in field_names: - if self.getRawHeaders(name) and \ - name in header_dict: + if self.getRawHeaders(name) and headers.getRawHeaders(name): pass else: diff.add(name) diff --git a/tests/test_trueheaders.py b/tests/test_trueheaders.py index 33521b8..9ac0a27 100644 --- a/tests/test_trueheaders.py +++ b/tests/test_trueheaders.py @@ -23,18 +23,18 @@ dummy_headers_dict3 = { class TestTrueHeaders(unittest.TestCase): def test_names_match(self): th = TrueHeaders(dummy_headers_dict) - self.assertEqual(th.getDiff(dummy_headers_dict), set()) + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), set()) def test_names_not_match(self): th = TrueHeaders(dummy_headers_dict) - self.assertEqual(th.getDiff(dummy_headers_dict2), set(['Header3'])) + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3'])) th = TrueHeaders(dummy_headers_dict3) - self.assertEqual(th.getDiff(dummy_headers_dict2), set(['Header3', 'Header4'])) + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3', 'Header4'])) def test_names_match_expect_ignore(self): th = TrueHeaders(dummy_headers_dict) - self.assertEqual(th.getDiff(dummy_headers_dict2, ignore=['Header3']), set()) + self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), set())

1 0

[ooni-probe/master] Add documentation for DNS Tamper test
by art＠torproject.org 29 Nov '12

29 Nov '12

commit 340e4b932f0817f552f12c2900039020fe2eba63 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 29 12:26:50 2012 +0100 Add documentation for DNS Tamper test --- docs/source/tests/dnstamper.rst | 229 +++++++++++++++++++++++---------------- 1 files changed, 137 insertions(+), 92 deletions(-) diff --git a/docs/source/tests/dnstamper.rst b/docs/source/tests/dnstamper.rst index d37afa8..b97131b 100644 --- a/docs/source/tests/dnstamper.rst +++ b/docs/source/tests/dnstamper.rst @@ -3,11 +3,11 @@ Details *Test Name*: DNS Tamper -*Current version*: 0.3 +*Current version*: 0.4 -*NetTest*: DNSTamperTest (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/nettests/core/dnsta…) +*NetTest*: DNS Tamper Test (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/nettests/blocking/d…) -*Test Helper*: DNSTestHelper (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/oonib/testhelpers/d…) +*Test Helper*: DNS Test Helper (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/oonib/testhelpers/d…) *Test Type*: Content Blocking @@ -28,7 +28,9 @@ youtube, etc. How to run the test =================== -`./bin/ooniprobe nettests/core/dnstamper.py -t <test resolvers file> -f <input file> -b IP:PORT` +`./bin/ooniprobe nettests/core/dnstamper.py [-t <test resolvers>|-T <test resolver file>-f <input file> -b IP:PORT` + +*test resolvers* is a single test resolver (IP address) *test resolvers file* is a file containing the IP addresses of the resolvers to test for censorship, one per line. @@ -44,92 +46,135 @@ From running: :: - ########################################### - # OONI Probe Report for DNS tamper test - # Tue Nov 20 20:38:54 2012 - ########################################### - --- - {probe_asn: null, probe_cc: null, probe_ip: 127.0.0.1, software_name: ooniprobe, software_version: 0.0.7.1-alpha, - start_time: 1353436734.0, test_name: DNS tamper, test_version: '0.3'} - ... - --- - input: torproject.org - report: - control_resolver: &id001 [8.8.8.8, 53] - queries: - - addrs: [86.59.30.40, 38.229.72.14, 38.229.72.16, 82.195.75.101] - answers: - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=86.59.30.40 - ttl=142>] - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=38.229.72.14 - ttl=142>] - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=38.229.72.16 - ttl=142>] - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=82.195.75.101 - ttl=142>] - query: '[Query(''torproject.org'', 1, 1)]' - query_type: A - resolver: *id001 - - addrs: [86.59.30.40, 38.229.72.14, 38.229.72.16, 82.195.75.101] - answers: - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=86.59.30.40 - ttl=142>] - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=38.229.72.14 - ttl=142>] - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=38.229.72.16 - ttl=142>] - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=82.195.75.101 - ttl=142>] - query: '[Query(''torproject.org'', 1, 1)]' - query_type: A - resolver: [8.8.8.8, 53] - - addrs: [86.59.30.40, 38.229.72.14, 38.229.72.16, 82.195.75.101] - answers: - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=86.59.30.40 - ttl=142>] - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=38.229.72.14 - ttl=142>] - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=38.229.72.16 - ttl=142>] - - [<RR name=torproject.org type=A class=IN ttl=142s auth=False>, <A address=82.195.75.101 - ttl=142>] - query: '[Query(''torproject.org'', 1, 1)]' - query_type: A - resolver: [8.8.4.4, 53] - tampering: {8.8.4.4: false, 8.8.8.8: false} - test_resolvers: [8.8.8.8, 8.8.4.4] - test_name: test_a_queries - test_started: 1353440334.075345 - ... - --- - input: ooni.nu - report: - control_resolver: &id001 [8.8.8.8, 53] - queries: - - addrs: [178.79.139.176] - answers: - - [<RR name=ooni.nu type=A class=IN ttl=1478s auth=False>, <A address=178.79.139.176 - ttl=1478>] - query: '[Query(''ooni.nu'', 1, 1)]' - query_type: A - resolver: *id001 - - addrs: [178.79.139.176] - answers: - - [<RR name=ooni.nu type=A class=IN ttl=1478s auth=False>, <A address=178.79.139.176 - ttl=1478>] - query: '[Query(''ooni.nu'', 1, 1)]' - query_type: A - resolver: [8.8.8.8, 53] - - addrs: [178.79.139.176] - answers: - - [<RR name=ooni.nu type=A class=IN ttl=1478s auth=False>, <A address=178.79.139.176 - ttl=1478>] - query: '[Query(''ooni.nu'', 1, 1)]' - query_type: A - resolver: [8.8.4.4, 53] - tampering: {8.8.4.4: false, 8.8.8.8: false} - test_resolvers: [8.8.8.8, 8.8.4.4] - test_name: test_a_queries - test_started: 1353440334.077116 - ... + ########################################### + # OONI Probe Report for DNS tamper test + # Thu Nov 29 12:17:19 2012 + ########################################### + --- + options: + collector: null + help: 0 + logfile: null + pcapfile: null + reportfile: null + resume: 0 + subargs: [-t, 8.8.8.8, -f, test_inputs/dns_tamper_file.txt] + test: nettests/blocking/dnstamper.py + probe_asn: null + probe_cc: null + probe_ip: 127.0.0.1 + software_name: ooniprobe + software_version: 0.0.7.1-alpha + start_time: 1354184239.0 + test_name: DNS tamper + test_version: '0.4' + ... + --- + input: torproject.org + report: + control_resolver: &id001 [8.8.8.8, 53] + queries: + - addrs: [86.59.30.40, 38.229.72.14, 38.229.72.16, 82.195.75.101] + answers: + - [<RR name=torproject.org type=A class=IN ttl=91s auth=False>, <A address=86.59.30.40 + ttl=91>] + - [<RR name=torproject.org type=A class=IN ttl=91s auth=False>, <A address=38.229.72.14 + ttl=91>] + - [<RR name=torproject.org type=A class=IN ttl=91s auth=False>, <A address=38.229.72.16 + ttl=91>] + - [<RR name=torproject.org type=A class=IN ttl=91s auth=False>, <A address=82.195.75.101 + ttl=91>] + query: '[Query(''torproject.org'', 1, 1)]' + query_type: A + resolver: *id001 + - addrs: [86.59.30.40, 38.229.72.14, 38.229.72.16, 82.195.75.101] + answers: + - [<RR name=torproject.org type=A class=IN ttl=91s auth=False>, <A address=86.59.30.40 + ttl=91>] + - [<RR name=torproject.org type=A class=IN ttl=91s auth=False>, <A address=38.229.72.14 + ttl=91>] + - [<RR name=torproject.org type=A class=IN ttl=91s auth=False>, <A address=38.229.72.16 + ttl=91>] + - [<RR name=torproject.org type=A class=IN ttl=91s auth=False>, <A address=82.195.75.101 + ttl=91>] + query: '[Query(''torproject.org'', 1, 1)]' + query_type: A + resolver: [8.8.8.8, 53] + tampering: {8.8.8.8: false} + test_name: test_a_lookup + test_runtime: 0.0733950138092041 + test_started: 1354187839.508863 + ... + --- + input: google.com + report: + control_resolver: &id001 [8.8.8.8, 53] + queries: + - addrs: [173.194.69.100, 173.194.69.139, 173.194.69.113, 173.194.69.101, 173.194.69.138, + 173.194.69.102] + answers: + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.100 + ttl=54>] + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.139 + ttl=54>] + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.113 + ttl=54>] + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.101 + ttl=54>] + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.138 + ttl=54>] + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.102 + ttl=54>] + query: '[Query(''google.com'', 1, 1)]' + query_type: A + resolver: *id001 + - addrs: [173.194.69.100, 173.194.69.139, 173.194.69.113, 173.194.69.101, 173.194.69.138, + 173.194.69.102] + answers: + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.100 + ttl=54>] + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.139 + ttl=54>] + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.113 + ttl=54>] + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.101 + ttl=54>] + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.138 + ttl=54>] + - [<RR name=google.com type=A class=IN ttl=54s auth=False>, <A address=173.194.69.102 + ttl=54>] + query: '[Query(''google.com'', 1, 1)]' + query_type: A + resolver: [8.8.8.8, 53] + tampering: {8.8.8.8: false} + test_name: test_a_lookup + test_runtime: 0.08325004577636719 + test_started: 1354187839.51091 + ... + --- + input: measurementlab.net + report: + control_resolver: &id001 [8.8.8.8, 53] + queries: + - addrs: [72.249.86.184] + answers: + - [<RR name=measurementlab.net type=A class=IN ttl=600s auth=False>, <A address=72.249.86.184 + ttl=600>] + query: '[Query(''measurementlab.net'', 1, 1)]' + query_type: A + resolver: *id001 + - addrs: [72.249.86.184] + answers: + - [<RR name=measurementlab.net type=A class=IN ttl=600s auth=False>, <A address=72.249.86.184 + ttl=600>] + query: '[Query(''measurementlab.net'', 1, 1)]' + query_type: A + resolver: [8.8.8.8, 53] + tampering: {8.8.8.8: false} + test_name: test_a_lookup + test_runtime: 0.2565779685974121 + test_started: 1354187839.512434 + ... + +Notes: Query is the string repsentation of :class:twisted.names.dns.Query

1 0

[ooni-probe/master] Rename test_inputs to example_inputs
by art＠torproject.org 29 Nov '12

29 Nov '12

commit 3a6ca3d13551c04f10a1f4e3feed0921c56fcaee Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 29 13:27:00 2012 +0100 Rename test_inputs to example_inputs --- example_inputs/README | 2 ++ example_inputs/daph3_yaml_file.txt | 2 ++ example_inputs/dns_tamper_file.txt | 3 +++ example_inputs/dns_tamper_test_resolvers.txt | 2 ++ example_inputs/http_host_file.txt | 2 ++ example_inputs/keyword_filtering_file.txt | 2 ++ example_inputs/url_lists_file.txt | 2 ++ test_inputs/README | 2 -- test_inputs/daph3_yaml_file.txt | 2 -- test_inputs/dns_tamper_file.txt | 3 --- test_inputs/dns_tamper_test_resolvers.txt | 2 -- test_inputs/http_host_file.txt | 2 -- test_inputs/keyword_filtering_file.txt | 2 -- test_inputs/url_lists_file.txt | 2 -- 14 files changed, 15 insertions(+), 15 deletions(-) diff --git a/example_inputs/README b/example_inputs/README new file mode 100644 index 0000000..cbf0782 --- /dev/null +++ b/example_inputs/README @@ -0,0 +1,2 @@ +These inputs are here to assist the running of before_i_commit.sh that is used to verify proper functioning of ooniprobe. + diff --git a/example_inputs/daph3_yaml_file.txt b/example_inputs/daph3_yaml_file.txt new file mode 100644 index 0000000..2412a08 --- /dev/null +++ b/example_inputs/daph3_yaml_file.txt @@ -0,0 +1,2 @@ +{client: antani, server: sblinda} + diff --git a/example_inputs/dns_tamper_file.txt b/example_inputs/dns_tamper_file.txt new file mode 100644 index 0000000..25f365c --- /dev/null +++ b/example_inputs/dns_tamper_file.txt @@ -0,0 +1,3 @@ +torproject.org +google.com +measurementlab.net diff --git a/example_inputs/dns_tamper_test_resolvers.txt b/example_inputs/dns_tamper_test_resolvers.txt new file mode 100644 index 0000000..14c77e0 --- /dev/null +++ b/example_inputs/dns_tamper_test_resolvers.txt @@ -0,0 +1,2 @@ +8.8.8.8 +8.8.4.4 diff --git a/example_inputs/http_host_file.txt b/example_inputs/http_host_file.txt new file mode 100644 index 0000000..12afb18 --- /dev/null +++ b/example_inputs/http_host_file.txt @@ -0,0 +1,2 @@ +torproject.org +ooni.nu diff --git a/example_inputs/keyword_filtering_file.txt b/example_inputs/keyword_filtering_file.txt new file mode 100644 index 0000000..4583bae --- /dev/null +++ b/example_inputs/keyword_filtering_file.txt @@ -0,0 +1,2 @@ +antani +sblinda diff --git a/example_inputs/url_lists_file.txt b/example_inputs/url_lists_file.txt new file mode 100644 index 0000000..16a4f58 --- /dev/null +++ b/example_inputs/url_lists_file.txt @@ -0,0 +1,2 @@ +http://ooni.nu/test +http://torproject.org/ diff --git a/test_inputs/README b/test_inputs/README deleted file mode 100644 index cbf0782..0000000 --- a/test_inputs/README +++ /dev/null @@ -1,2 +0,0 @@ -These inputs are here to assist the running of before_i_commit.sh that is used to verify proper functioning of ooniprobe. - diff --git a/test_inputs/daph3_yaml_file.txt b/test_inputs/daph3_yaml_file.txt deleted file mode 100644 index 2412a08..0000000 --- a/test_inputs/daph3_yaml_file.txt +++ /dev/null @@ -1,2 +0,0 @@ -{client: antani, server: sblinda} - diff --git a/test_inputs/dns_tamper_file.txt b/test_inputs/dns_tamper_file.txt deleted file mode 100644 index 25f365c..0000000 --- a/test_inputs/dns_tamper_file.txt +++ /dev/null @@ -1,3 +0,0 @@ -torproject.org -google.com -measurementlab.net diff --git a/test_inputs/dns_tamper_test_resolvers.txt b/test_inputs/dns_tamper_test_resolvers.txt deleted file mode 100644 index 14c77e0..0000000 --- a/test_inputs/dns_tamper_test_resolvers.txt +++ /dev/null @@ -1,2 +0,0 @@ -8.8.8.8 -8.8.4.4 diff --git a/test_inputs/http_host_file.txt b/test_inputs/http_host_file.txt deleted file mode 100644 index 12afb18..0000000 --- a/test_inputs/http_host_file.txt +++ /dev/null @@ -1,2 +0,0 @@ -torproject.org -ooni.nu diff --git a/test_inputs/keyword_filtering_file.txt b/test_inputs/keyword_filtering_file.txt deleted file mode 100644 index 4583bae..0000000 --- a/test_inputs/keyword_filtering_file.txt +++ /dev/null @@ -1,2 +0,0 @@ -antani -sblinda diff --git a/test_inputs/url_lists_file.txt b/test_inputs/url_lists_file.txt deleted file mode 100644 index 16a4f58..0000000 --- a/test_inputs/url_lists_file.txt +++ /dev/null @@ -1,2 +0,0 @@ -http://ooni.nu/test -http://torproject.org/

1 0

[ooni-probe/master] Update before_i_commit testdeck
by art＠torproject.org 29 Nov '12

29 Nov '12

commit 25aa38d851dae046bbeff96bc8cb9104bdad127f Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 29 13:38:49 2012 +0100 Update before_i_commit testdeck * Other misc changes --- before_i_commit.testdeck | 6 +++--- .../manipulation/http_header_field_manipulation.py | 3 ++- ooni/runner.py | 1 - ooni/utils/geodata.py | 1 - ooni/utils/hacks.py | 3 +-- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/before_i_commit.testdeck b/before_i_commit.testdeck index af420be..7877d74 100644 --- a/before_i_commit.testdeck +++ b/before_i_commit.testdeck @@ -12,7 +12,7 @@ logfile: before_i_commit.log pcapfile: null reportfile: dns_tamper_test.yamloo - subargs: [-T, test_inputs/dns_tamper_test_resolvers.txt, -f, test_inputs/dns_tamper_file.txt] + subargs: [-T, example_inputs/dns_tamper_test_resolvers.txt, -f, example_inputs/dns_tamper_file.txt] test: nettests/blocking/dnstamper.py - options: collector: null @@ -20,7 +20,7 @@ logfile: before_i_commit.log pcapfile: null reportfile: http_host.yamloo - subargs: [-b, 'http://ooni.nu/test', -f, test_inputs/http_host_file.txt] + subargs: [-b, 'http://ooni.nu/test', -f, example_inputs/http_host_file.txt] test: nettests/manipulation/http_host.py # XXX this is disabled because it requires oonib to be running #- options: @@ -29,5 +29,5 @@ # logfile: null # pcapfile: null # reportfile: null -# subargs: [-h, test_inputs/test_header_field_manipulation.txt] +# subargs: [-h, example_inputs/test_header_field_manipulation.txt] # test: nettests/core/http_header_field_manipulation.py diff --git a/nettests/manipulation/http_header_field_manipulation.py b/nettests/manipulation/http_header_field_manipulation.py index 08ee8c7..0140638 100644 --- a/nettests/manipulation/http_header_field_manipulation.py +++ b/nettests/manipulation/http_header_field_manipulation.py @@ -136,7 +136,8 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest): self.report['tampering']['request_line_capitalization'] = True request_headers = TrueHeaders(self.request_headers) - diff = request_headers.getDiff(response_headers_dict, ignore=['Connection']) + diff = request_headers.getDiff(TrueHeaders(response_headers_dict), + ignore=['Connection']) if diff: self.report['tampering']['header_field_name'] = True else: diff --git a/ooni/runner.py b/ooni/runner.py index 4f59d17..bc9b874 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -5,7 +5,6 @@ # Handles running ooni.nettests as well as # ooni.plugoo.tests.OONITests. # -# :authors: Arturo Filastò, Isis Lovecruft # :license: see included LICENSE file import os diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index 447d2eb..6acefd8 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -5,7 +5,6 @@ # In here go functions related to the understanding of # geographical information of the probe # -# :authors: Arturo Filastò # :licence: see LICENSE import re diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py index d4e832c..18e9102 100644 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@ -4,8 +4,7 @@ # ******** # When some software has issues and we need to fix it in a # hackish way, we put it in here. This one day will be empty. -# -# :authors: Arturo Filastò +# # :licence: see LICENSE

1 0

[ooni-probe/master] Test and document HTTP Requests test
by art＠torproject.org 29 Nov '12

29 Nov '12

commit 4922b1ce9795b19a8e726217f45fc65ad17ea162 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 29 13:35:27 2012 +0100 Test and document HTTP Requests test * Add support reporting the modification of response headers Note: This test used to be called HTTP Body Length, but got extended to support response headers. --- docs/source/tests/http_requests.rst | 238 +++++++++++++++++++++++++++++++++ nettests/blocking/http_body_length.py | 90 ------------- nettests/blocking/http_requests.py | 120 +++++++++++++++++ 3 files changed, 358 insertions(+), 90 deletions(-) diff --git a/docs/source/tests/http_requests.rst b/docs/source/tests/http_requests.rst new file mode 100644 index 0000000..cc98882 --- /dev/null +++ b/docs/source/tests/http_requests.rst @@ -0,0 +1,238 @@ +Details +======= + +*Test Name*: HTTP Requests + +*Current version*: 0.1 + +*NetTest*: HTTP Requests (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/nettests/blocking/h…) + +*Test Helper*: None + +*Test Type*: Content Blocking + +*Requires Root*: No + +Description +=========== + +This test perform a HTTP GET request for the / resource over the test network +and over Tor. It then compares the two responses to see if the response bodies of the two requests match and if the +proportion between the expected body length (the one over Tor) and the one over +the control network match. + +If the proportion between the two body lengths is <= a certain tollerance +factor (by default set to 0.8), then we say that they do not match. + +The reason for doing so is that a lot of sites serve geolocalized content based +on the location from which the request originated from. + +How to run the test +=================== + +To test a single site run: + +`./bin/ooniprobe nettests/blocking/http_requests.py -u http://<test_site>/` + +To test a set of sites from a list containing sites to test run: + +`./bin/ooniprobe nettests/blocking/http_requests.py -f <input_file>` + + +Sample report +============= + +`./bin/ooniprobe nettests/blocking/http_requests.py -f example_inputs/url_lists_file.txt` + +:: + + ########################################### + # OONI Probe Report for HTTP Requests Test test + # Thu Nov 29 13:20:06 2012 + ########################################### + --- + options: + collector: null + help: 0 + logfile: null + pcapfile: null + reportfile: null + resume: 0 + subargs: [-f, example_inputs/url_lists_file.txt] + test: nettests/blocking/http_requests.py + probe_asn: null + probe_cc: null + probe_ip: 127.0.0.1 + software_name: ooniprobe + software_version: 0.0.7.1-alpha + start_time: 1354188006.0 + test_name: HTTP Requests Test + test_version: '0.1' + ... + --- + input: http://ooni.nu/test + report: + agent: agent + body_length_match: false + body_proportion: 0.9732142857142857 + factor: 0.8 + requests: + - request: + body: null + headers: + - - User-Agent + - - &id001 [Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1), 'Internet + Explorer 7, Windows Vista'] + method: GET + url: http://ooni.nu/test + response: + body: "\n<html>\n <head>\n <meta http-equiv=\"refresh\" content=\"\ + 0;URL=http://ooni.nu/test/\">\n </head>\n <body bgcolor=\"#FFFFFF\"\ + \ text=\"#000000\">\n <a href=\"http://ooni.nu/test/\">click here</a>\n\ + \ </body>\n</html>\n" + code: 302 + headers: + - - Content-Length + - ['218'] + - - Server + - [TwistedWeb/10.1.0] + - - Connection + - [close] + - - Location + - ['http://ooni.nu/test/'] + - - Date + - ['Thu, 29 Nov 2012 12:20:25 GMT'] + - - Content-Type + - [text/html] + - request: + body: null + headers: + - - User-Agent + - - *id001 + method: GET + url: shttp://ooni.nu/test + response: + body: "\n<html>\n <head>\n <meta http-equiv=\"refresh\" content=\"\ + 0;URL=http://ooni.nu:80/test/\">\n </head>\n <body bgcolor=\"#FFFFFF\"\ + \ text=\"#000000\">\n <a href=\"http://ooni.nu:80/test/\">click here</a>\n\ + \ </body>\n</html>\n" + code: 302 + headers: + - - Content-Length + - ['224'] + - - Server + - [TwistedWeb/10.1.0] + - - Connection + - [close] + - - Location + - ['http://ooni.nu:80/test/'] + - - Date + - ['Thu, 29 Nov 2012 12:20:33 GMT'] + - - Content-Type + - [text/html] + socksproxy: null + test_name: test_get + test_runtime: 9.357746124267578 + test_started: 1354191606.333243 + ... + --- + input: http://torproject.org/ + report: + agent: agent + body_length_match: false + body_proportion: 1.0 + factor: 0.8 + requests: + - request: + body: null + headers: + - - User-Agent + - - &id001 [Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1), 'Internet + Explorer 7, Windows Vista'] + method: GET + url: http://torproject.org/ + response: + body: '<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN"> + + <html><head> + + <title>302 Found</title> + + </head><body> + + <h1>Found</h1> + + <p>The document has moved <a href="https://www.torproject.org/">here</a>.</p> + + <hr> + + <address>Apache Server at torproject.org Port 80</address> + + </body></html> + + ' + code: 302 + headers: + - - Content-Length + - ['275'] + - - Vary + - [Accept-Encoding] + - - Server + - [Apache] + - - Connection + - [close] + - - Location + - ['https://www.torproject.org/'] + - - Date + - ['Thu, 29 Nov 2012 12:20:08 GMT'] + - - Content-Type + - [text/html; charset=iso-8859-1] + - request: + body: null + headers: + - - User-Agent + - - *id001 + method: GET + url: shttp://torproject.org/ + response: + body: '<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN"> + + <html><head> + + <title>302 Found</title> + + </head><body> + + <h1>Found</h1> + + <p>The document has moved <a href="https://www.torproject.org/">here</a>.</p> + + <hr> + + <address>Apache Server at torproject.org Port 80</address> + + </body></html> + + ' + code: 302 + headers: + - - Content-Length + - ['275'] + - - Vary + - [Accept-Encoding] + - - Server + - [Apache] + - - Connection + - [close] + - - Location + - ['https://www.torproject.org/'] + - - Date + - ['Thu, 29 Nov 2012 12:20:16 GMT'] + - - Content-Type + - [text/html; charset=iso-8859-1] + socksproxy: null + test_name: test_get + test_runtime: 8.688138008117676 + test_started: 1354191607.287672 + ... + diff --git a/nettests/blocking/http_body_length.py b/nettests/blocking/http_body_length.py deleted file mode 100644 index 7c5b0d2..0000000 --- a/nettests/blocking/http_body_length.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -from twisted.internet import defer -from twisted.python import usage -from ooni.templates import httpt - -class UsageOptions(usage.Options): - optParameters = [ - ['url', 'u', None, 'Specify a single URL to test.'], - ['factor', 'f', 0.8, 'What factor should be used for triggering censorship (0.8 == 80%)'] - ] - -class HTTPBodyLength(httpt.HTTPTest): - """ - Performs a two GET requests to the set of sites to be tested for - censorship, one over a known good control channel (Tor), the other over the - test network. - We then look at the response body lengths and see if the control response - differs from the experiment response by a certain factor. - """ - name = "HTTP Body length test" - author = "Arturo Filastò" - version = "0.1" - - usageOptions = UsageOptions - - inputFile = ['file', 'f', None, - 'List of URLS to perform GET and POST requests to'] - - # These values are used for determining censorship based on response body - # lengths - control_body_length = None - experiment_body_length = None - - def setUp(self): - """ - Check for inputs. - """ - if self.input: - self.url = self.input - elif self.localOptions['url']: - self.url = self.localOptions['url'] - else: - raise Exception("No input specified") - - self.factor = self.localOptions['factor'] - - def compare_body_lengths(self): - body_length_a = self.control_body_length - body_length_b = self.experiment_body_length - - rel = float(body_length_a)/float(body_length_b) - if rel > 1: - rel = 1/rel - - self.report['body_proportion'] = rel - self.report['factor'] = self.factor - if rel < self.factor: - self.report['censorship'] = True - else: - self.report['censorship'] = False - - def test_get(self): - def errback(failure): - log.err("There was an error while testing %s" % self.url) - log.exception(failure) - - def control_body(result): - self.control_body_length = len(result) - if self.experiment_body_length: - self.compare_body_lengths() - - def experiment_body(result): - self.experiment_body_length = len(result) - if self.control_body_length: - self.compare_body_lengths() - - dl = [] - experiment_request = self.doRequest(self.url, method="GET", - body_processor=experiment_body) - control_request = self.doRequest(self.url, method="GET", - use_tor=True, body_processor=control_body) - dl.append(experiment_request) - dl.append(control_request) - d = defer.DeferredList(dl) - return d - diff --git a/nettests/blocking/http_requests.py b/nettests/blocking/http_requests.py new file mode 100644 index 0000000..bc2803b --- /dev/null +++ b/nettests/blocking/http_requests.py @@ -0,0 +1,120 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from twisted.internet import defer +from twisted.python import usage +from ooni.templates import httpt + +class UsageOptions(usage.Options): + optParameters = [ + ['url', 'u', None, 'Specify a single URL to test.'], + ['factor', 'f', 0.8, 'What factor should be used for triggering censorship (0.8 == 80%)'] + ] + +class HTTPRequestsTest(httpt.HTTPTest): + """ + Performs a two GET requests to the set of sites to be tested for + censorship, one over a known good control channel (Tor), the other over the + test network. + + We check to see if the response headers match and if the response body + lengths match. + """ + name = "HTTP Requests Test" + author = "Arturo Filastò" + version = "0.1" + + usageOptions = UsageOptions + + inputFile = ['file', 'f', None, + 'List of URLS to perform GET and POST requests to'] + + # These values are used for determining censorship based on response body + # lengths + control_body_length = None + experiment_body_length = None + + def setUp(self): + """ + Check for inputs. + """ + if self.input: + self.url = self.input + elif self.localOptions['url']: + self.url = self.localOptions['url'] + else: + raise Exception("No input specified") + + self.factor = self.localOptions['factor'] + + def compare_body_lengths(self): + body_length_a = self.control_body_length + body_length_b = self.experiment_body_length + + rel = float(body_length_a)/float(body_length_b) + if rel > 1: + rel = 1/rel + + self.report['body_proportion'] = rel + self.report['factor'] = self.factor + if rel < self.factor: + self.report['body_length_match'] = True + else: + self.report['body_length_match'] = False + + def compare_headers(self): + diff = TrueHeaders(self.control_headers).getDiff(self.experiment_headers) + if diff: + self.report['headers_match'] = False + else: + self.report['headers_match'] = True + + def test_get(self): + def errback(failure): + log.err("There was an error while testing %s" % self.url) + log.exception(failure) + + def control_body(result): + """ + Callback for processing the control HTTP body response. + """ + self.control_body_length = len(result) + if self.experiment_body_length: + self.compare_body_lengths() + + def experiment_body(result): + """ + Callback for processing the experiment HTTP body response. + """ + self.experiment_body_length = len(result) + if self.control_body_length: + self.compare_body_lengths() + + def control_headers(headers_dict): + """ + Callback for processing the control HTTP headers response. + """ + self.control_headers = headers_dict + + def experiment_headers(headers_dict): + """ + Callback for processing the experiment HTTP headers response. + """ + self.experiment_headers = headers_dict + + dl = [] + experiment_request = self.doRequest(self.url, method="GET", + body_processor=experiment_body, + headers_processor=control_headers) + + control_request = self.doRequest(self.url, method="GET", + use_tor=True, body_processor=control_body, + headers_processor=control_headers) + + dl.append(experiment_request) + dl.append(control_request) + d = defer.DeferredList(dl) + return d +

1 0

[ooni-probe/master] Add ooniprobe.conf to gitignore
by art＠torproject.org 29 Nov '12

29 Nov '12

commit 7135d896dfe4b8674fcb5b4c63bd42c1c3309e9b Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 29 13:42:03 2012 +0100 Add ooniprobe.conf to gitignore * Don't seem to be able to get this right --- .gitignore | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/.gitignore b/.gitignore index 2dada89..4cb0775 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ pcaps/ report*.yaml docs/build/* data/*.dat -./ooniprobe.conf +ooniprobe.conf

1 0

[ooni-probe/master] Iterate on HTTP Invalid Request Line test
by art＠torproject.org 29 Nov '12

29 Nov '12

commit 1de07f659f1393d969a1b3766baffeecb111355d Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Nov 29 15:41:01 2012 +0100 Iterate on HTTP Invalid Request Line test * Rename it to HTTP Invalid Request Line * Extend it's fuzzing capabilities to support some more specific tests * Improve TCPT test template --- nettests/blocking/http_invalid_requests.py | 63 ------------ nettests/manipulation/http_invalid_request_line.py | 106 ++++++++++++++++++++ ooni/nettest.py | 38 +++++++- ooni/templates/httpt.py | 21 +---- ooni/templates/tcpt.py | 48 ++++++---- 5 files changed, 174 insertions(+), 102 deletions(-) diff --git a/nettests/blocking/http_invalid_requests.py b/nettests/blocking/http_invalid_requests.py deleted file mode 100644 index 7e6f47f..0000000 --- a/nettests/blocking/http_invalid_requests.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- encoding: utf-8 -*- -from twisted.python import usage - -from ooni.utils import randomStr -from ooni.templates import tcpt - -class UsageOptions(usage.Options): - optParameters = [['backend', 'b', '127.0.0.1:57002', - 'The OONI backend that runs a TCP echo server (must be on port 80)']] - - optFlags = [['nopayloadmatch', 'n', - "Don't match the payload of the response. This option is used when you don't have a TCP echo server running"]] - -class HTTPInvalidRequests(tcpt.TCPTest): - name = "HTTP Invalid Requests" - version = "0.1.1" - authors = "Arturo Filastò" - - inputFile = ['file', 'f', None, - 'Input file of list of hostnames to attempt to resolve'] - - usageOptions = UsageOptions - requiredOptions = ['backend'] - - def setUp(self): - try: - self.address, self.port = self.localOptions['backend'].split(":") - self.port = int(self.port) - except: - raise usage.UsageError("Invalid backend address specified (must be address:port)") - - def test_random_invalid_request(self): - """ - We test sending data to a TCP echo server, if what we get back is not - what we have sent then there is tampering going on. - This is for example what squid will return when performing such - request: - - HTTP/1.0 400 Bad Request - Server: squid/2.6.STABLE21 - Date: Sat, 23 Jul 2011 02:22:44 GMT - Content-Type: text/html - Content-Length: 1178 - Expires: Sat, 23 Jul 2011 02:22:44 GMT - X-Squid-Error: ERR_INVALID_REQ 0 - X-Cache: MISS from cache_server - X-Cache-Lookup: NONE from cache_server:3128 - Via: 1.0 cache_server:3128 (squid/2.6.STABLE21) - Proxy-Connection: close - - """ - payload = randomStr(10) + "\n\r" - def got_all_data(received_array): - if not self.localOptions['nopayloadmatch']: - first = received_array[0] - if first != payload: - self.report['tampering'] = True - else: - self.report['tampering'] = 'unknown' - - d = self.sendPayload(payload) - d.addCallback(got_all_data) - return d diff --git a/nettests/manipulation/http_invalid_request_line.py b/nettests/manipulation/http_invalid_request_line.py new file mode 100644 index 0000000..00ab24d --- /dev/null +++ b/nettests/manipulation/http_invalid_request_line.py @@ -0,0 +1,106 @@ +# -*- encoding: utf-8 -*- +from twisted.python import usage + +from ooni.utils import randomStr +from ooni.templates import tcpt + +class UsageOptions(usage.Options): + optParameters = [['backend', 'b', '127.0.0.1', + 'The OONI backend that runs a TCP echo server']] + +class HTTPInvalidRequestLine(tcpt.TCPTest): + """ + The goal of this test is to do some very basic and not very noisy fuzzing + on the HTTP request line. We generate a series of requests that are not + valid HTTP requests. + + Unless elsewhere stated 'Xx'*N refers to N*2 random upper or lowercase ascii + letters or numbers ('XxXx' will be 4). + """ + name = "HTTP Invalid Requests" + version = "0.1.3" + authors = "Arturo Filastò" + + inputFile = ['file', 'f', None, + 'Input file of list of hostnames to attempt to resolve'] + + usageOptions = UsageOptions + requiredOptions = ['backend'] + + def setUp(self): + self.port = 80 + self.address = self.localOptions['backend'] + + def check_for_manipulation(self, response, payload): + if response != payload: + self.report['tampering'] = True + else: + self.report['tampering'] = 'unknown' + + def test_random_invalid_method(self): + """ + We test sending data to a TCP echo server listening on port 80, if what + we get back is not what we have sent then there is tampering going on. + This is for example what squid will return when performing such + request: + + HTTP/1.0 400 Bad Request + Server: squid/2.6.STABLE21 + Date: Sat, 23 Jul 2011 02:22:44 GMT + Content-Type: text/html + Content-Length: 1178 + Expires: Sat, 23 Jul 2011 02:22:44 GMT + X-Squid-Error: ERR_INVALID_REQ 0 + X-Cache: MISS from cache_server + X-Cache-Lookup: NONE from cache_server:3128 + Via: 1.0 cache_server:3128 (squid/2.6.STABLE21) + Proxy-Connection: close + + """ + payload = randomStr(10) + " / HTTP/1.1\n\r" + + d = self.sendPayload(payload) + d.addCallback(self.check_for_manipulation, payload) + return d + + def test_random_invalid_field_count(self): + """ + This generates a request that looks like this: + + XxXxX XxXxX XxXxX XxXxX + + This may trigger some bugs in the HTTP parsers of transparent HTTP + proxies. + """ + payload = ' '.join(randomStr(5) for x in range(4)) + payload += "\n\r" + + d = self.sendPayload(payload) + d.addCallback(self.check_for_manipulation, payload) + return d + + def test_random_big_request_method(self): + """ + This generates a request that looks like this: + + Xx*512 / HTTP/1.1 + """ + payload = randomStr(1024) + ' / HTTP/1.1\n\r' + + d = self.sendPayload(payload) + d.addCallback(self.check_for_manipulation, payload) + return d + + def test_random_invalid_version_number(self): + """ + This generates a request that looks like this: + + GET / HTTP/XxX + """ + payload = 'GET / HTTP/' + randomStr(3) + payload += '\n\r' + + d = self.sendPayload(payload) + d.addCallback(self.check_for_manipulation, payload) + return d + diff --git a/ooni/nettest.py b/ooni/nettest.py index e0393e7..4a54414 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -5,7 +5,6 @@ # In here is the NetTest API definition. This is how people # interested in writing ooniprobe tests will be specifying them # -# :authors: Arturo Filastò, Isis Lovecruft # :license: see included LICENSE file import sys @@ -17,12 +16,47 @@ from twisted.trial import unittest, itrial, util from twisted.internet import defer, utils from twisted.python import usage +from twisted.internet.error import ConnectionRefusedError, DNSLookupError, TCPTimedOutError + from ooni.utils import log +def failureToString(failure): + """ + Given a failure instance return a string representing the kind of error + that occurred. + + Args: + + failure: a :class:twisted.internet.error instance + + Returns: + + A string representing the HTTP response error message. + """ + if isinstance(failure.value, ConnectionRefusedError): + log.err("Connection refused. The backend may be down") + string = 'connection_refused_error' + + elif isinstance(failure.value, SOCKSError): + log.err("Sock error. The SOCKS proxy may be down") + string = 'socks_error' + + elif isinstance(failure.value, DNSLookupError): + log.err("DNS lookup failure") + string = 'dns_lookup_error' + + elif isinstance(failure.value, TCPTimedOutError): + log.err("TCP Timed Out Error") + string = 'tcp_timed_out_error' + + elif isinstance(failure.value, ResponseNeverReceived): + log.err("Response Never Received") + string = 'response_never_received' + return string + class NoPostProcessor(Exception): pass - class NetTestCase(object): """ This is the base of the OONI nettest universe. When you write a nettest diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 804a3e4..01853f3 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -23,6 +23,8 @@ from ooni import config from ooni.utils.net import BodyReceiver, StringProducer, userAgents from ooni.utils.txagentwithsocks import Agent, SOCKSError, TrueHeaders +from ooni.nettest import failureToString + class InvalidSocksProxyOption(Exception): pass @@ -131,25 +133,8 @@ class HTTPTest(NetTestCase): 'code': response.code } if failure: - if isinstance(failure.value, ConnectionRefusedError): - log.err("Connection refused. The backend may be down") - request_response['failure'] = 'connection_refused_error' - - elif isinstance(failure.value, SOCKSError): - log.err("Sock error. The SOCKS proxy may be down") - request_response['failure'] = 'socks_error' - - elif isinstance(failure.value, DNSLookupError): - log.err("DNS lookup failure") - request_response['failure'] = 'dns_lookup_error' - - elif isinstance(failure.value, TCPTimedOutError): - log.err("TCP Timed Out Error") - request_response['failure'] = 'tcp_timed_out_error' + request_response['failure'] = failureToString(failure) - elif isinstance(failure.value, ResponseNeverReceived): - log.err("Response Never Received") - request_response['failure'] = 'response_never_received' self.report['requests'].append(request_response) def _processResponseBody(self, response_body, request, response, body_processor): diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py index 77ffe3e..26f38ed 100644 --- a/ooni/templates/tcpt.py +++ b/ooni/templates/tcpt.py @@ -2,13 +2,14 @@ from twisted.internet import protocol, defer, reactor from twisted.internet.error import ConnectionDone from twisted.internet.endpoints import TCP4ClientEndpoint -from ooni.nettest import NetTestCase +from ooni.nettest import NetTestCase, failureToString from ooni.utils import log class TCPSender(protocol.Protocol): - report = None - payload_len = None - received_data = '' + def __init__(self): + self.received_data = '' + self.sent_data = '' + def dataReceived(self, data): """ We receive data until the total amount of data received reaches that @@ -27,18 +28,19 @@ class TCPSender(protocol.Protocol): """ if self.payload_len: self.received_data += data - if len(self.received_data) >= self.payload_len: - self.transport.loseConnection() - self.report['received'].append(data) - self.deferred.callback(self.report['received']) def sendPayload(self, payload): """ Write the payload to the wire and set the expected size of the payload we are to receive. + + Args: + + payload: the data to be sent on the wire. + """ self.payload_len = len(payload) - self.report['sent'].append(payload) + self.sent_data = payload self.transport.write(payload) class TCPSenderFactory(protocol.Factory): @@ -50,7 +52,7 @@ class TCPTest(NetTestCase): version = "0.1" requiresRoot = False - timeout = 2 + timeout = 5 address = None port = None @@ -61,26 +63,34 @@ class TCPTest(NetTestCase): def sendPayload(self, payload): d1 = defer.Deferred() - def closeConnection(p): - p.transport.loseConnection() + def closeConnection(proto): + self.report['sent'].append(proto.sent_data) + self.report['received'].append(proto.received_data) + proto.transport.loseConnection() log.debug("Closing connection") d1.callback(self.report['received']) + def timedOut(proto): + self.report['failure'] = 'tcp_timed_out_error' + proto.transport.loseConnection() + def errback(failure): - self.report['error'] = str(failure) + self.report['failure'] = failureToString(failure) d1.errback(failure) - def connected(p): + def connected(proto): log.debug("Connected to %s:%s" % (self.address, self.port)) - p.report = self.report - p.deferred = d1 - p.sendPayload(payload) - reactor.callLater(self.timeout, closeConnection, p) + proto.report = self.report + proto.deferred = d1 + proto.sendPayload(payload) + if self.timeout: + # XXX-Twisted this logic should probably go inside of the protocol + reactor.callLater(self.timeout, closeConnection, proto) point = TCP4ClientEndpoint(reactor, self.address, self.port) + log.debug("Connecting to %s:%s" % (self.address, self.port)) d2 = point.connect(TCPSenderFactory()) d2.addCallback(connected) d2.addErrback(errback) return d1 -

1 0