[tor-commits] [ooni-probe/master] Refactor reporter and removal of dead code
art at torproject.org
art at torproject.org
Wed Oct 24 16:17:38 UTC 2012
commit 6bb22e9b86a073cdbd9dc87d7ea87389f7296239
Author: Arturo Filastò <arturo at filasto.net>
Date: Wed Oct 24 15:59:53 2012 +0000
Refactor reporter and removal of dead code
---
ooni/oonicli.py | 27 ++++-----
ooni/oonitests/httphost.py | 135 --------------------------------------------
ooni/oonitests/squid.py | 113 ------------------------------------
ooni/oonitests/template.py | 65 ---------------------
ooni/reporter.py | 26 ++++++---
5 files changed, 29 insertions(+), 337 deletions(-)
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 5ea92d7..aa3cdc0 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -13,29 +13,24 @@
# original copyright (c) by Twisted Matrix Laboratories.
-import sys, os, random, gc, time, warnings
-
-import unittest
-import inspect
-
-from ooni.inputunit import InputUnitFactory
-from ooni.reporter import ReporterFactory
-from ooni.nettest import InputTestSuite
-from ooni.plugoo import tests
-from ooni import nettest, runner, reporter
+import sys
+import os
+import random
+import time
from twisted.internet import defer
from twisted.application import app
-from twisted.python import usage, reflect, failure
-from twisted.python.filepath import FilePath
-from twisted import plugin
+from twisted.python import usage, failure
from twisted.python.util import spewer
-from twisted.python.compat import set
-from twisted.trial import itrial
-from twisted.trial import runner as irunner
+from ooni import nettest, runner, reporter
+
+from ooni.inputunit import InputUnitFactory
+from ooni.reporter import ReporterFactory
+from ooni.nettest import InputTestSuite
from ooni.utils import log
+
class Options(usage.Options, app.ReactorSelectionMixin):
synopsis = """%s [options] [[file|package|module|TestCase|testmethod]...]
""" % (os.path.basename(sys.argv[0]),)
diff --git a/ooni/oonitests/__init__.py b/ooni/oonitests/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/ooni/oonitests/httphost.py b/ooni/oonitests/httphost.py
deleted file mode 100644
index 25adff2..0000000
--- a/ooni/oonitests/httphost.py
+++ /dev/null
@@ -1,135 +0,0 @@
-"""
- HTTP Host based filtering
- *************************
-
- This test detect HTTP Host field
- based filtering.
- It is used to detect censorship on
- performed with Web Guard (used by
- T-Mobile US).
-"""
-import os
-from datetime import datetime
-from gevent import monkey
-
-import urllib2
-import httplib
-# WARNING! Using gevent's socket
-# introduces the 0x20 DNS "feature".
-# This will result is weird DNS requests
-# appearing on the wire.
-monkey.patch_socket()
-
-try:
- from BeautifulSoup import BeautifulSoup
-except:
- print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/"
-
-from plugoo.assets import Asset
-from plugoo.tests import Test
-
-__plugoo__ = "HTTP Host"
-__desc__ = "This detects HTTP Host field based filtering"
-
-class HTTPHostAsset(Asset):
- """
- This is the asset that should be used by the Test. It will
- contain all the code responsible for parsing the asset file
- and should be passed on instantiation to the test.
- """
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
- def parse_line(self, line):
- return line.split(',')[1].replace('\n','')
-
-class HTTPHost(Test):
- """
- The main Test class
- """
-
- def check_response(self, response):
- soup = BeautifulSoup(response)
- if soup.head.title.string == "Content Filtered":
- # Response indicates censorship
- return True
- else:
- # Response does not indicate censorship
- return False
-
-
- def is_censored(self, response):
- if response:
- soup = BeautifulSoup(response)
- censored = self.check_response(response)
- else:
- censored = "unreachable"
- return censored
-
- def urllib2_test(self, control_server, host):
- req = urllib2.Request(control_server)
- req.add_header('Host', host)
- try:
- r = urllib2.urlopen(req)
- response = r.read()
- censored = self.is_censored(response)
- except Exception, e:
- censored = "Error! %s" % e
-
- return censored
-
- def httplib_test(self, control_server, host):
- try:
- conn = httplib.HTTPConnection(control_server)
- conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True)
- conn.putheader("Host", host)
- conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6")
- conn.endheaders()
- r = conn.getresponse()
- response = r.read()
- censored = self.is_censored(response)
- except Exception, e:
- censored = "Error! %s" % e
-
- return censored
-
-
- def experiment(self, *a, **kw):
- """
- Try to connect to the control server with
- the specified host field.
- """
- host = kw['data']
- control_server = kw['control_server']
- self.logger.info("Testing %s (%s)" % (host, control_server))
-
- #censored = self.urllib2_test(control_server, host)
- censored = self.httplib_test(control_server, host)
-
- self.logger.info("%s: %s" % (host, censored))
- return {'Time': datetime.now(),
- 'Host': host,
- 'Censored': censored}
-
-
-def run(ooni):
- """
- This is the function that will be called by OONI
- and it is responsible for instantiating and passing
- the arguments to the Test class.
- """
- config = ooni.config
-
- # This the assets array to be passed to the run function of
- # the test
- assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \
- "top-1m.csv"))]
-
- # Instantiate the Test
- thetest = HTTPHost(ooni)
- ooni.logger.info("starting HTTP Host Test...")
- # Run the test with argument assets
- thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'})
- ooni.logger.info("finished.")
-
-
diff --git a/ooni/oonitests/squid.py b/ooni/oonitests/squid.py
deleted file mode 100644
index 847135e..0000000
--- a/ooni/oonitests/squid.py
+++ /dev/null
@@ -1,113 +0,0 @@
-"""
- Squid Proxy Detector
- ********************
-
-"""
-import os
-import httplib
-import urllib2
-from urlparse import urlparse
-
-from plugoo import gen_headers
-from plugoo.assets import Asset
-from plugoo.tests import Test
-
-__plugoo__ = "SquidProxy"
-__desc__ = "This Test aims at detecting the squid transparent proxy"
-
-class SquidAsset(Asset):
- """
- This is the asset that should be used by the Test. It will
- contain all the code responsible for parsing the asset file
- and should be passed on instantiation to the test.
- """
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
-
-class Squid(Test):
- """
- Squid Proxy testing class.
- """
- def _http_request(self, method, url,
- path=None, headers=None):
- """
- Perform an HTTP Request.
- XXX move this function to the core OONI
- code.
- """
- url = urlparse(url)
- host = url.netloc
-
- conn = httplib.HTTPConnection(host, 80)
- conn.connect()
-
- if path is None:
- path = purl.path
-
- conn.putrequest(method, path)
-
- for h in gen_headers():
- conn.putheaders(h[0], h[1])
- conn.endheaders()
-
- send_browser_headers(self, None, conn)
-
- response = conn.getresponse()
-
- headers = dict(response.getheaders())
-
- self.headers = headers
- self.data = response.read()
- return True
-
- def invalid_request(self):
- """
- This will trigger squids "Invalid Request" error.
- """
- pass
-
- def cache_object(self):
- """
- This attempts to do a GET cache_object://localhost/info on
- any destination and checks to see if the response contains
- is that of Squid.
- """
-
- pass
-
- def experiment(self, *a, **kw):
- """
- Fill this up with the tasks that should be performed
- on the "dirty" network and should be compared with the
- control.
- """
-
-
- def control(self):
- """
- Fill this up with the control related code.
- """
- return True
-
-def run(ooni):
- """
- This is the function that will be called by OONI
- and it is responsible for instantiating and passing
- the arguments to the Test class.
- """
- config = ooni.config
-
- # This the assets array to be passed to the run function of
- # the test
- assets = [TestTemplateAsset(os.path.join(config.main.assetdir, \
- "someasset.txt"))]
-
- # Instantiate the Test
- thetest = TestTemplate(ooni)
- ooni.logger.info("starting SquidProxyTest...")
- # Run the test with argument assets
- thetest.run(assets)
- ooni.logger.info("finished.")
-
-
diff --git a/ooni/oonitests/template.py b/ooni/oonitests/template.py
deleted file mode 100644
index f5f9d93..0000000
--- a/ooni/oonitests/template.py
+++ /dev/null
@@ -1,65 +0,0 @@
-"""
- Test Template
- *************
-
- This is a Test template, to be used when writing your
- own OONI probe Tests.
-"""
-import os
-
-from plugoo.assets import Asset
-from plugoo.tests import Test
-
-__plugoo__ = "Test Template"
-__desc__ = "This a test template to be used to build your own tests"
-
-class TestTemplateAsset(Asset):
- """
- This is the asset that should be used by the Test. It will
- contain all the code responsible for parsing the asset file
- and should be passed on instantiation to the test.
- """
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
-
-class TestTemplate(Test):
- """
- The main Test class
- """
-
- def experiment(self, *a, **kw):
- """
- Fill this up with the tasks that should be performed
- on the "dirty" network and should be compared with the
- control.
- """
- return False
-
- def control(self):
- """
- Fill this up with the control related code.
- """
- return True
-
-def run(ooni):
- """
- This is the function that will be called by OONI
- and it is responsible for instantiating and passing
- the arguments to the Test class.
- """
- config = ooni.config
-
- # This the assets array to be passed to the run function of
- # the test
- assets = [TestTemplateAsset(os.path.join(config.main.assetdir, \
- "someasset.txt"))]
-
- # Instantiate the Test
- thetest = TestTemplate(ooni)
- ooni.logger.info("starting TestTemplate...")
- # Run the test with argument assets
- thetest.run(assets)
- ooni.logger.info("finished.")
-
-
diff --git a/ooni/reporter.py b/ooni/reporter.py
index c12b28f..fc5dde9 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -1,4 +1,3 @@
-import copy_reg
import itertools
import logging
import sys
@@ -12,8 +11,8 @@ from yaml.serializer import *
from yaml.resolver import *
from datetime import datetime
-from twisted.python.util import OrderedDict, untilConcludes
-from twisted.trial import unittest, reporter, runner
+from twisted.python.util import untilConcludes
+from twisted.trial import reporter
from twisted.internet import defer
from ooni.utils import date, log, geodata
@@ -30,6 +29,11 @@ except:
pyunit = __import__('unittest')
class OSafeRepresenter(SafeRepresenter):
+ """
+ This is a custom YAML representer that allows us to represent reports
+ safely.
+ It extends the SafeRepresenter to be able to also represent complex numbers
+ """
def represent_complex(self, data):
if data.imag == 0.0:
data = u'%r' % data.real
@@ -45,7 +49,10 @@ OSafeRepresenter.add_representer(complex,
OSafeRepresenter.represent_complex)
class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):
-
+ """
+ This is a modification of the YAML Safe Dumper to use our own Safe
+ Representer that supports complex numbers.
+ """
def __init__(self, stream,
default_style=None, default_flow_style=None,
canonical=None, indent=None, width=None,
@@ -64,6 +71,9 @@ class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):
def safe_dump(data, stream=None, **kw):
+ """
+ Safely dump to a yaml file the specified data.
+ """
return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw)
class OReporter(pyunit.TestResult):
@@ -88,8 +98,8 @@ class OReporter(pyunit.TestResult):
def _getTime(self):
return time.time()
- def _write(self, format, *args):
- s = str(format)
+ def _write(self, format_string, *args):
+ s = str(format_string)
assert isinstance(s, type(''))
if args:
self._stream.write(s % args)
@@ -97,8 +107,8 @@ class OReporter(pyunit.TestResult):
self._stream.write(s)
untilConcludes(self._stream.flush)
- def _writeln(self, format, *args):
- self._write(format, *args)
+ def _writeln(self, format_string, *args):
+ self._write(format_string, *args)
self._write('\n')
def writeYamlLine(self, line):
More information about the tor-commits
mailing list