[tor-commits] [ooni-probe/master] Make SSL work in HTTPT

[isis at torproject.org]

commit ae87c2b74844372586d9c863ab63bb5f5e6bec38
Author: Arturo Filastò <arturo at filasto.net>
Date:   Fri Sep 28 23:16:44 2012 +0000

    Make SSL work in HTTPT
    * Fix misc bugs
---
 README.md                  |   10 +++++++++-
 bin/ooniprobe              |    8 ++++----
 docs/source/index.rst      |    2 +-
 nettests/example_scapyt.py |    1 +
 ooni/templates/httpt.py    |   19 +++++++++++++++++--
 ooni/utils/log.py          |    1 -
 6 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 23deb32..4837275 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,15 @@ To list the help for a specific test:
 
     virtualenv2 ENV/
     source ENV/bin/activate
-    pip install twisted Scapy
+    pip install twisted Scapy pyyaml
+
+The setup.py script of pyOpenSSL has some issues and it is therefore
+recommended to install it system wide with:
+
+    sudo pip install pyOpenSSL
+
+Note: be sure to do this over a secure network since pip does not by default
+use SSL...
 
 To install the most up to date scapy version (requires mercurial):
 
diff --git a/bin/ooniprobe b/bin/ooniprobe
index 62d60bb..c653756 100755
--- a/bin/ooniprobe
+++ b/bin/ooniprobe
@@ -11,13 +11,13 @@
 import os, sys
 import copy_reg
 
+# Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain.
+sys.path[:] = map(os.path.abspath, sys.path)
+sys.path.insert(0, os.path.abspath(os.getcwd()))
+
 # This is a hack to overcome a bug in python
 from ooni.utils.hacks import patched_reduce_ex
 copy_reg._reduce_ex = patched_reduce_ex
 
-sys.path[:] = map(os.path.abspath, sys.path)
-
-sys.path.insert(0, os.path.abspath(os.getcwd()))
-
 from ooni.oonicli import run
 run()
diff --git a/docs/source/index.rst b/docs/source/index.rst
index e744f7f..9de0dba 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -20,7 +20,7 @@ need to do the following::
 
     virtualenv ENV
     source ENV/bin/activate
-    pip install twisted Scapy pyyaml
+    pip install twisted Scapy pyyaml pyOpenSSL
 
 To get the latest version of scapy you will need mercurial. You can then install
 it with::
diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py
index 54fce9a..2e73150 100644
--- a/nettests/example_scapyt.py
+++ b/nettests/example_scapyt.py
@@ -11,3 +11,4 @@ class ExampleScapy(scapyt.ScapyTest):
     version = 0.1
 
     inputs = [IP(dst='8.8.8.8')/UDP(), IP()/TCP()]
+
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 117a77a..51775f0 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -9,6 +9,8 @@ from zope.interface import implements
 from twisted.python import usage
 from twisted.plugin import IPlugin
 from twisted.internet import protocol, defer
+from twisted.internet.ssl import ClientContextFactory
+
 from twisted.web.http_headers import Headers
 
 from ooni.nettest import TestCase
@@ -26,6 +28,11 @@ useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Geck
               ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"),
               ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")]
 
+
+class WebClientContextFactory(ClientContextFactory):
+    def getContext(self, hostname, port):
+        return ClientContextFactory.getContext(self)
+
 class BodyReceiver(protocol.Protocol):
     def __init__(self, finished):
         self.finished = finished
@@ -52,10 +59,16 @@ class HTTPTest(TestCase):
     followRedirects = False
 
     def setUp(self):
+        try:
+            import OpenSSL
+        except:
+            log.err("Warning! pyOpenSSL is not installed. https websites will"
+                     "not work")
         from twisted.web.client import Agent
         from twisted.internet import reactor
-        import yaml
+
         self.agent = Agent(reactor)
+
         if self.followRedirects:
             from twisted.web.client import RedirectAgent
             self.agent = RedirectAgent(self.agent)
@@ -96,6 +109,7 @@ class HTTPTest(TestCase):
     def doRequest(self, url):
         d = self.build_request(url)
         def finished(data):
+            #self.mainDefer.callback()
             return data
 
         d.addCallback(self._cbResponse)
@@ -144,7 +158,8 @@ class HTTPTest(TestCase):
 
         self.report['request'] = self.request
         self.report['url'] = url
-        return self.agent.request(self.request['method'], self.request['url'],
+        req = self.agent.request(self.request['method'], self.request['url'],
                                   Headers(self.request['headers']),
                                   self.request['body'])
+        return req
 
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index f79f4b6..6ff6bf9 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -91,7 +91,6 @@ def debug(message, level="debug", **kw):
     log.msg(message, logLevel=level, **kw)
 
 def msg(message, level="info", **kw):
-    print "Msg %s" % message
     log.msg(message, logLevel=level, **kw)
 
 def err(message, level="err", **kw):