[tor-commits] [ooni-probe/master] Add better documentation to HTTP Host and url list tests

[art at torproject.org]

commit 04363333313869883c172aa0c4be18c7b776c52b
Author: Arturo Filastò <art at fuffa.org>
Date:   Mon Nov 12 10:12:43 2012 +0100

    Add better documentation to HTTP Host and url list tests
---
 nettests/core/http_host.py |   12 ++++++++----
 nettests/core/url_list.py  |   44 +++++++++++++++++++++++++++++++++++++-------
 2 files changed, 45 insertions(+), 11 deletions(-)

diff --git a/nettests/core/http_host.py b/nettests/core/http_host.py
index 662cc40..252561b 100644
--- a/nettests/core/http_host.py
+++ b/nettests/core/http_host.py
@@ -3,9 +3,6 @@
 # HTTP Host Test
 # **************
 #
-# for more details see:
-# https://trac.torproject.org/projects/tor/wiki/doc/OONI/Tests/HTTPHost
-#
 # :authors: Arturo Filastò
 # :licence: see LICENSE
 
@@ -25,6 +22,12 @@ class HTTPHost(httpt.HTTPTest):
     """
     This test is aimed at detecting the presence of a transparent HTTP proxy
     and enumerating the sites that are being censored by it.
+
+    It places inside of the Host header field the hostname of the site that is
+    to be tested for censorship and then determines if the probe is behind a
+    transparent HTTP proxy (because the response from the backend server does
+    not match) and if the site is censorsed, by checking if the page that it
+    got back matches the input block page.
     """
     name = "HTTP Host"
     author = "Arturo Filastò"
@@ -32,7 +35,8 @@ class HTTPHost(httpt.HTTPTest):
 
     usageOptions = UsageOptions
 
-    inputFile = ['file', 'f', None, 'List of hostnames to test for censorship']
+    inputFile = ['file', 'f', None, 
+            'List of hostnames to test for censorship']
 
     requiredOptions = ['backend']
 
diff --git a/nettests/core/url_list.py b/nettests/core/url_list.py
index fe678be..b6f2d43 100644
--- a/nettests/core/url_list.py
+++ b/nettests/core/url_list.py
@@ -5,20 +5,52 @@
 
 from ooni.templates import httpt
 
+class UsageOptions(usage.Options):
+    optParameters = [['content', 'c', None, 
+                        'The file to read from containing the content of a block page']]
+
 class URLList(httpt.HTTPTest):
+    """
+    Performs GET, POST and PUT requests to a list of URLs specified as
+    input and checks if the page that we get back as a result matches that
+    which we expect.
+    """
     name = "URL List"
     author = "Arturo Filastò"
-    version = 0.1
+    version = "0.1.1"
+
+    inputFile = ['file', 'f', None, 
+            'List of URLS to perform GET and POST requests to']
 
-    inputFile = ['file', 'f', None, 'List of URLS to perform GET and POST requests to']
-    
     requiredOptions = ['file']
+
+    def check_for_censorship(self, body):
+        """
+        If we have specified what a censorship page looks like here we will
+        check if the page we are looking at matches it.
+
+        XXX this is not tested, though it is basically what was used to detect
+        censorship in the palestine case.
+        """
+        if self.localOptions['content']:
+            self.report['censored'] = True
+
+            censorship_page = open(self.localOptions['content'])
+            response_page = iter(body.split("\n"))
+
+            for censorship_line in censorship_page.xreadlines():
+                response_line = response_page.next()
+                if response_line != censorship_line:
+                    self.report['censored'] = False
+                    break
+
+            censorship_page.close()
+
     def test_get(self):
         if self.input:
             self.url = self.input
         else:
             raise Exception("No input specified")
-
         return self.doRequest(self.url, method="GET")
 
     def test_post(self):
@@ -26,15 +58,13 @@ class URLList(httpt.HTTPTest):
             self.url = self.input
         else:
             raise Exception("No input specified")
-
         return self.doRequest(self.url, method="POST")
-    
+
     def test_put(self):
         if self.input:
             self.url = self.input
         else:
             raise Exception("No input specified")
-
         return self.doRequest(self.url, method="PUT")