commit 6bb22e9b86a073cdbd9dc87d7ea87389f7296239 Author: Arturo Filastò arturo@filasto.net Date: Wed Oct 24 15:59:53 2012 +0000

Refactor reporter and removal of dead code --- ooni/oonicli.py | 27 ++++----- ooni/oonitests/httphost.py | 135 -------------------------------------------- ooni/oonitests/squid.py | 113 ------------------------------------ ooni/oonitests/template.py | 65 --------------------- ooni/reporter.py | 26 ++++++--- 5 files changed, 29 insertions(+), 337 deletions(-)

diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 5ea92d7..aa3cdc0 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -13,29 +13,24 @@ # original copyright (c) by Twisted Matrix Laboratories.

-import sys, os, random, gc, time, warnings - -import unittest -import inspect - -from ooni.inputunit import InputUnitFactory -from ooni.reporter import ReporterFactory -from ooni.nettest import InputTestSuite -from ooni.plugoo import tests -from ooni import nettest, runner, reporter +import sys +import os +import random +import time

from twisted.internet import defer from twisted.application import app -from twisted.python import usage, reflect, failure -from twisted.python.filepath import FilePath -from twisted import plugin +from twisted.python import usage, failure from twisted.python.util import spewer -from twisted.python.compat import set -from twisted.trial import itrial -from twisted.trial import runner as irunner

+from ooni import nettest, runner, reporter + +from ooni.inputunit import InputUnitFactory +from ooni.reporter import ReporterFactory +from ooni.nettest import InputTestSuite from ooni.utils import log

+ class Options(usage.Options, app.ReactorSelectionMixin): synopsis = """%s [options] [[file|package|module|TestCase|testmethod]...] """ % (os.path.basename(sys.argv[0]),) diff --git a/ooni/oonitests/__init__.py b/ooni/oonitests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ooni/oonitests/httphost.py b/ooni/oonitests/httphost.py deleted file mode 100644 index 25adff2..0000000 --- a/ooni/oonitests/httphost.py +++ /dev/null @@ -1,135 +0,0 @@ -""" - HTTP Host based filtering - ************************* - - This test detect HTTP Host field - based filtering. - It is used to detect censorship on - performed with Web Guard (used by - T-Mobile US). -""" -import os -from datetime import datetime -from gevent import monkey - -import urllib2 -import httplib -# WARNING! Using gevent's socket -# introduces the 0x20 DNS "feature". -# This will result is weird DNS requests -# appearing on the wire. -monkey.patch_socket() - -try: - from BeautifulSoup import BeautifulSoup -except: - print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/" - -from plugoo.assets import Asset -from plugoo.tests import Test - -__plugoo__ = "HTTP Host" -__desc__ = "This detects HTTP Host field based filtering" - -class HTTPHostAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - return line.split(',')[1].replace('\n','') - -class HTTPHost(Test): - """ - The main Test class - """ - - def check_response(self, response): - soup = BeautifulSoup(response) - if soup.head.title.string == "Content Filtered": - # Response indicates censorship - return True - else: - # Response does not indicate censorship - return False - - - def is_censored(self, response): - if response: - soup = BeautifulSoup(response) - censored = self.check_response(response) - else: - censored = "unreachable" - return censored - - def urllib2_test(self, control_server, host): - req = urllib2.Request(control_server) - req.add_header('Host', host) - try: - r = urllib2.urlopen(req) - response = r.read() - censored = self.is_censored(response) - except Exception, e: - censored = "Error! %s" % e - - return censored - - def httplib_test(self, control_server, host): - try: - conn = httplib.HTTPConnection(control_server) - conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) - conn.putheader("Host", host) - conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") - conn.endheaders() - r = conn.getresponse() - response = r.read() - censored = self.is_censored(response) - except Exception, e: - censored = "Error! %s" % e - - return censored - - - def experiment(self, *a, **kw): - """ - Try to connect to the control server with - the specified host field. - """ - host = kw['data'] - control_server = kw['control_server'] - self.logger.info("Testing %s (%s)" % (host, control_server)) - - #censored = self.urllib2_test(control_server, host) - censored = self.httplib_test(control_server, host) - - self.logger.info("%s: %s" % (host, censored)) - return {'Time': datetime.now(), - 'Host': host, - 'Censored': censored} - - -def run(ooni): - """ - This is the function that will be called by OONI - and it is responsible for instantiating and passing - the arguments to the Test class. - """ - config = ooni.config - - # This the assets array to be passed to the run function of - # the test - assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \ - "top-1m.csv"))] - - # Instantiate the Test - thetest = HTTPHost(ooni) - ooni.logger.info("starting HTTP Host Test...") - # Run the test with argument assets - thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'}) - ooni.logger.info("finished.") - - diff --git a/ooni/oonitests/squid.py b/ooni/oonitests/squid.py deleted file mode 100644 index 847135e..0000000 --- a/ooni/oonitests/squid.py +++ /dev/null @@ -1,113 +0,0 @@ -""" - Squid Proxy Detector - ******************** - -""" -import os -import httplib -import urllib2 -from urlparse import urlparse - -from plugoo import gen_headers -from plugoo.assets import Asset -from plugoo.tests import Test - -__plugoo__ = "SquidProxy" -__desc__ = "This Test aims at detecting the squid transparent proxy" - -class SquidAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - -class Squid(Test): - """ - Squid Proxy testing class. - """ - def _http_request(self, method, url, - path=None, headers=None): - """ - Perform an HTTP Request. - XXX move this function to the core OONI - code. - """ - url = urlparse(url) - host = url.netloc - - conn = httplib.HTTPConnection(host, 80) - conn.connect() - - if path is None: - path = purl.path - - conn.putrequest(method, path) - - for h in gen_headers(): - conn.putheaders(h[0], h[1]) - conn.endheaders() - - send_browser_headers(self, None, conn) - - response = conn.getresponse() - - headers = dict(response.getheaders()) - - self.headers = headers - self.data = response.read() - return True - - def invalid_request(self): - """ - This will trigger squids "Invalid Request" error. - """ - pass - - def cache_object(self): - """ - This attempts to do a GET cache_object://localhost/info on - any destination and checks to see if the response contains - is that of Squid. - """ - - pass - - def experiment(self, *a, **kw): - """ - Fill this up with the tasks that should be performed - on the "dirty" network and should be compared with the - control. - """ - - - def control(self): - """ - Fill this up with the control related code. - """ - return True - -def run(ooni): - """ - This is the function that will be called by OONI - and it is responsible for instantiating and passing - the arguments to the Test class. - """ - config = ooni.config - - # This the assets array to be passed to the run function of - # the test - assets = [TestTemplateAsset(os.path.join(config.main.assetdir, \ - "someasset.txt"))] - - # Instantiate the Test - thetest = TestTemplate(ooni) - ooni.logger.info("starting SquidProxyTest...") - # Run the test with argument assets - thetest.run(assets) - ooni.logger.info("finished.") - - diff --git a/ooni/oonitests/template.py b/ooni/oonitests/template.py deleted file mode 100644 index f5f9d93..0000000 --- a/ooni/oonitests/template.py +++ /dev/null @@ -1,65 +0,0 @@ -""" - Test Template - ************* - - This is a Test template, to be used when writing your - own OONI probe Tests. -""" -import os - -from plugoo.assets import Asset -from plugoo.tests import Test - -__plugoo__ = "Test Template" -__desc__ = "This a test template to be used to build your own tests" - -class TestTemplateAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - -class TestTemplate(Test): - """ - The main Test class - """ - - def experiment(self, *a, **kw): - """ - Fill this up with the tasks that should be performed - on the "dirty" network and should be compared with the - control. - """ - return False - - def control(self): - """ - Fill this up with the control related code. - """ - return True - -def run(ooni): - """ - This is the function that will be called by OONI - and it is responsible for instantiating and passing - the arguments to the Test class. - """ - config = ooni.config - - # This the assets array to be passed to the run function of - # the test - assets = [TestTemplateAsset(os.path.join(config.main.assetdir, \ - "someasset.txt"))] - - # Instantiate the Test - thetest = TestTemplate(ooni) - ooni.logger.info("starting TestTemplate...") - # Run the test with argument assets - thetest.run(assets) - ooni.logger.info("finished.") - - diff --git a/ooni/reporter.py b/ooni/reporter.py index c12b28f..fc5dde9 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -1,4 +1,3 @@ -import copy_reg import itertools import logging import sys @@ -12,8 +11,8 @@ from yaml.serializer import * from yaml.resolver import *

from datetime import datetime -from twisted.python.util import OrderedDict, untilConcludes -from twisted.trial import unittest, reporter, runner +from twisted.python.util import untilConcludes +from twisted.trial import reporter from twisted.internet import defer from ooni.utils import date, log, geodata

@@ -30,6 +29,11 @@ except: pyunit = __import__('unittest')

class OSafeRepresenter(SafeRepresenter): + """ + This is a custom YAML representer that allows us to represent reports + safely. + It extends the SafeRepresenter to be able to also represent complex numbers + """ def represent_complex(self, data): if data.imag == 0.0: data = u'%r' % data.real @@ -45,7 +49,10 @@ OSafeRepresenter.add_representer(complex, OSafeRepresenter.represent_complex)

class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): - + """ + This is a modification of the YAML Safe Dumper to use our own Safe + Representer that supports complex numbers. + """ def __init__(self, stream, default_style=None, default_flow_style=None, canonical=None, indent=None, width=None, @@ -64,6 +71,9 @@ class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):

def safe_dump(data, stream=None, **kw): + """ + Safely dump to a yaml file the specified data. + """ return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw)

class OReporter(pyunit.TestResult): @@ -88,8 +98,8 @@ class OReporter(pyunit.TestResult): def _getTime(self): return time.time()

- def _write(self, format, *args): - s = str(format) + def _write(self, format_string, *args): + s = str(format_string) assert isinstance(s, type('')) if args: self._stream.write(s % args) @@ -97,8 +107,8 @@ class OReporter(pyunit.TestResult): self._stream.write(s) untilConcludes(self._stream.flush)

- def _writeln(self, format, *args): - self._write(format, *args) + def _writeln(self, format_string, *args): + self._write(format_string, *args) self._write('\n')

def writeYamlLine(self, line):