commit ae87c2b74844372586d9c863ab63bb5f5e6bec38 Author: Arturo Filastò arturo@filasto.net Date: Fri Sep 28 23:16:44 2012 +0000

Make SSL work in HTTPT * Fix misc bugs --- README.md | 10 +++++++++- bin/ooniprobe | 8 ++++---- docs/source/index.rst | 2 +- nettests/example_scapyt.py | 1 + ooni/templates/httpt.py | 19 +++++++++++++++++-- ooni/utils/log.py | 1 - 6 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md index 23deb32..4837275 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,15 @@ To list the help for a specific test:

virtualenv2 ENV/ source ENV/bin/activate - pip install twisted Scapy + pip install twisted Scapy pyyaml + +The setup.py script of pyOpenSSL has some issues and it is therefore +recommended to install it system wide with: + + sudo pip install pyOpenSSL + +Note: be sure to do this over a secure network since pip does not by default +use SSL...

To install the most up to date scapy version (requires mercurial):

diff --git a/bin/ooniprobe b/bin/ooniprobe index 62d60bb..c653756 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -11,13 +11,13 @@ import os, sys import copy_reg

+# Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain. +sys.path[:] = map(os.path.abspath, sys.path) +sys.path.insert(0, os.path.abspath(os.getcwd())) + # This is a hack to overcome a bug in python from ooni.utils.hacks import patched_reduce_ex copy_reg._reduce_ex = patched_reduce_ex

-sys.path[:] = map(os.path.abspath, sys.path) - -sys.path.insert(0, os.path.abspath(os.getcwd())) - from ooni.oonicli import run run() diff --git a/docs/source/index.rst b/docs/source/index.rst index e744f7f..9de0dba 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -20,7 +20,7 @@ need to do the following::

virtualenv ENV source ENV/bin/activate - pip install twisted Scapy pyyaml + pip install twisted Scapy pyyaml pyOpenSSL

To get the latest version of scapy you will need mercurial. You can then install it with:: diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py index 54fce9a..2e73150 100644 --- a/nettests/example_scapyt.py +++ b/nettests/example_scapyt.py @@ -11,3 +11,4 @@ class ExampleScapy(scapyt.ScapyTest): version = 0.1

inputs = [IP(dst='8.8.8.8')/UDP(), IP()/TCP()] + diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 117a77a..51775f0 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -9,6 +9,8 @@ from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin from twisted.internet import protocol, defer +from twisted.internet.ssl import ClientContextFactory + from twisted.web.http_headers import Headers

from ooni.nettest import TestCase @@ -26,6 +28,11 @@ useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Geck ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")]

+ +class WebClientContextFactory(ClientContextFactory): + def getContext(self, hostname, port): + return ClientContextFactory.getContext(self) + class BodyReceiver(protocol.Protocol): def __init__(self, finished): self.finished = finished @@ -52,10 +59,16 @@ class HTTPTest(TestCase): followRedirects = False

def setUp(self): + try: + import OpenSSL + except: + log.err("Warning! pyOpenSSL is not installed. https websites will" + "not work") from twisted.web.client import Agent from twisted.internet import reactor - import yaml + self.agent = Agent(reactor) + if self.followRedirects: from twisted.web.client import RedirectAgent self.agent = RedirectAgent(self.agent) @@ -96,6 +109,7 @@ class HTTPTest(TestCase): def doRequest(self, url): d = self.build_request(url) def finished(data): + #self.mainDefer.callback() return data

d.addCallback(self._cbResponse) @@ -144,7 +158,8 @@ class HTTPTest(TestCase):

self.report['request'] = self.request self.report['url'] = url - return self.agent.request(self.request['method'], self.request['url'], + req = self.agent.request(self.request['method'], self.request['url'], Headers(self.request['headers']), self.request['body']) + return req

diff --git a/ooni/utils/log.py b/ooni/utils/log.py index f79f4b6..6ff6bf9 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -91,7 +91,6 @@ def debug(message, level="debug", **kw): log.msg(message, logLevel=level, **kw)

def msg(message, level="info", **kw): - print "Msg %s" % message log.msg(message, logLevel=level, **kw)

def err(message, level="err", **kw):