[tor-commits] [ooni-probe/master] Merge branch 'feature/fix_cct_tests' into release/1.4.0

[art at torproject.org]

commit 397ed5506d97a17cb4d05852af06ebf3674c638b
Merge: b2dbd38 bcc2be4
Author: Arturo Filastò <arturo at filasto.net>
Date:   Tue Apr 26 23:17:01 2016 +0200

    Merge branch 'feature/fix_cct_tests' into release/1.4.0
    
    * feature/fix_cct_tests:
      Fix bug in parsing of options for Lantern
      Add support for setting the url and expected-body in tests
      Re-create the command inside the test method.
      Update user documentation for psiphon and lantern
      When GeoIP lookup is disabled it's not possible to perform sanitisations
      Always use the same URL when testing psiphon
      Add support for stripping potential inclusion of probe IPs in reports
      Refactor lantern test to use the ProcessTest template
      Add test for Vanilla Tor

 docs/source/tests/lantern.rst         |  23 ++++---
 docs/source/tests/psiphon.rst         |  24 ++++---
 ooni/nettests/blocking/vanilla_tor.py | 105 +++++++++++++++++++++++++++++
 ooni/nettests/third_party/lantern.py  | 122 ++++++++++++++++++----------------
 ooni/nettests/third_party/psiphon.py  |  50 ++++++++------
 ooni/templates/httpt.py               |   8 ++-
 ooni/templates/process.py             |   9 ++-
 ooni/utils/net.py                     |   3 +
 8 files changed, 245 insertions(+), 99 deletions(-)

diff --cc ooni/nettests/third_party/psiphon.py
index 2633f60,2b18b18..d4eb18f
--- a/ooni/nettests/third_party/psiphon.py
+++ b/ooni/nettests/third_party/psiphon.py
@@@ -30,11 -31,11 +31,11 @@@ class PsiphonTest(httpt.HTTPTest,  proc
      """
  
      name = "Psiphon Test"
 -    description = "Bootstraps Psiphon and \
 -                does a HTTP GET for the specified URL"
 +    description = ("Bootstraps Psiphon and"
 +                   "does a HTTP GET for the specified URL")
      author = "juga"
-     version = "0.0.1"
-     timeout = 20
+     version = "0.1.0"
+     timeout = 120
      usageOptions = UsageOptions
  
      def _setUp(self):
diff --cc ooni/templates/httpt.py
index edab3fa,12995bb..fd6c8ab
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@@ -163,22 -131,21 +163,26 @@@ class HTTPTest(NetTestCase)
                  'url': request['url'],
                  'method': request['method'],
                  'tor': request['tor']
--            }
++            },
++            'response': None
          }
          if response:
 +            if self.localOptions.get('withoutbody', 0) is 0:
 +                response_body = _representBody(response_body)
 +            else:
 +                response_body = ''
+             # Attempt to redact the IP address of the probe from the responses
+             if config.privacy.includeip is False and config.probe_ip.address is not None:
+                 response_body = response_body.replace(config.probe_ip.address, "[REDACTED]")
 -            request_response['response'] = {
 -                'headers': list(response.headers.getAllRawHeaders()),
 -                'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '',
 +            session['response'] = {
 +                'headers': _representHeaders(response.headers),
 +                'body': response_body,
                  'code': response.code
-         }
+             }
 +        session['failure'] = None
          if failure_string:
 -            request_response['failure'] = failure_string
 -
 -        self.report['requests'].append(request_response)
 +            session['failure'] = failure_string
 +        self.report['requests'].append(session)
  
      def _processResponseBody(self, response_body, request, response, body_processor):
          log.debug("Processing response body")