November 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] * Changed txscapy.sendReceivePackets() to accept **kwargs so that the
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 60cd31ec733f2c67d0fc935d7612824d228e83ec Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Oct 31 17:18:06 2012 +0000 * Changed txscapy.sendReceivePackets() to accept **kwargs so that the default timeout could be overriden. * Unittest for sorting out multiple inheritance issues. --- ooni/templates/scapyt.py | 9 ++-- tests/test-class-design.py | 101 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index b2ac148..f561d66 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -44,13 +44,12 @@ class ScapyTest(TestCase): def tearDown(self): self.reactor.stop() - def sendReceivePackets(self, packets): - d = txsr(packets, pcapfile=self.pcapfile, - timeout=10, reactor=self.reactor) - + def sendReceivePackets(self, *kw): + d = txsr(packets=self.request, pcapfile=self.pcapfile, + timeout=self.timeout, reactor=self.reactor) return d - def sendPackets(self, packets): + def sendPackets(self): return txsend(self.buildPackets(), reactor=self.reactor) def buildPackets(self): diff --git a/tests/test-class-design.py b/tests/test-class-design.py new file mode 100644 index 0000000..bb80cd3 --- /dev/null +++ b/tests/test-class-design.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# +# testing classes to test multiple inheritance. +# these are not meant to be run by trial, though they could be made to be so. +# i didn't know where to put them. --isis + +import abc +from pprint import pprint +from inspect import classify_class_attrs + +class PluginBase(object): + __metaclass__ = abc.ABCMeta + + @abc.abstractproperty + def name(self): + return 'you should not see this' + + @name.setter + def name(self, value): + return 'you should not set this' + + @name.deleter + def name(self): + return 'you should not del this' + + @abc.abstractmethod + def inputParser(self, line): + """Do something to parse something.""" + return + +class Foo(object): + woo = "this class has some shit in it" + def bar(self): + print "i'm a Foo.bar()!" + print woo + +class KwargTest(Foo): + _name = "isis" + + #def __new__(cls, *a, **kw): + # return super(KwargTest, cls).__new__(cls, *a, **kw) + + @property + def name(self): + return self._name + + @name.setter + def name(self, value): + self._name = value + + def __init__(self, *a, **kw): + super(KwargTest, self).__init__() + + ## this causes the instantion args to override the class attrs + for key, value in kw.items(): + setattr(self.__class__, key, value) + + print "%s.__init__(): self.__dict__ = %s" \ + % (type(self), pprint(type(self).__dict__)) + + for attr in classify_class_attrs(self): + print attr + + @classmethod + def sayname(cls): + print cls.name + +class KwargTestChild(KwargTest): + name = "arturo" + def __init__(self): + super(KwargTestChild, self).__init__() + print self.name + +class KwargTestChildOther(KwargTest): + def __init__(self, name="robot", does="lasers"): + super(KwargTestChildOther, self).__init__() + print self.name + + +if __name__ == "__main__": + print "class KwargTest attr name: %s" % KwargTest.name + kwargtest = KwargTest() + print "KwargTest instantiated wo args" + print "kwargtest.name: %s" % kwargtest.name + print "kwargtest.sayname(): %s" % kwargtest.sayname() + kwargtest2 = KwargTest(name="lovecruft", does="hacking") + print "KwargTest instantiated with name args" + print "kwargtest.name: %s" % kwargtest2.name + print "kwargtest.sayname(): %s" % kwargtest2.sayname() + + print "class KwargTestChild attr name: %s" % KwargTestChild.name + kwargtestchild = KwargTestChild() + print "KwargTestChild instantiated wo args" + print "kwargtestchild.name: %s" % kwargtestchild.name + print "kwargtestchild.sayname(): %s" % kwargtestchild.sayname() + + print "class KwargTestChildOther attr name: %s" % KwargTestChildOther.name + kwargtestchildother = KwargTestChildOther() + print "KwargTestChildOther instantiated wo args" + print "kwargtestchildother.name: %s" % kwargtestchildother.name + print "kwargtestchildother.sayname(): %s" % kwargtestchildother.sayname()

1 0

[ooni-probe/master] Port squid transparent HTTP proxy detector to new API
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit f5acc677c00b8ab83bc6dc41fdd09ac487acdfc0 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Oct 23 18:04:36 2012 +0000 Port squid transparent HTTP proxy detector to new API * Remove some dead code * Move authors to new directory * Make reporter not swallow all tracebacks * Fix some bugs in httpt --- AUTHORS | 4 + nettests/core/captiveportal.py | 19 +++ nettests/core/squid.py | 117 ++++++++++++++++++++ old-to-be-ported-code/AUTHORS | 3 - old-to-be-ported-code/bin/ooni-probe | 10 -- old-to-be-ported-code/ooni/output.py | 21 ---- .../ooni/plugins/captiveportal_plgoo.py | 55 --------- old-to-be-ported-code/ooni/plugins/skel_plgoo.py | 17 --- old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml | 33 ------ old-to-be-ported-code/ooni/yamlooni.py | 40 ------- ooni/nettest.py | 97 +++++++---------- ooni/reporter.py | 23 +++- ooni/runner.py | 11 ++- ooni/templates/httpt.py | 47 +++++++- ooni/utils/__init__.py | 32 ++++++ 15 files changed, 276 insertions(+), 253 deletions(-) diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..d8dc0b8 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ +Jacob Appelbaum <jacob(a)torproject.org> +Arturo Filasto <hellais(a)torproject.org> +Linus Nordberg <linus(a)torproject.org> +Isis Lovecruft <isis(a)torproject.org> diff --git a/nettests/core/captiveportal.py b/nettests/core/captiveportal.py index 46f0856..77ba3e4 100644 --- a/nettests/core/captiveportal.py +++ b/nettests/core/captiveportal.py @@ -7,6 +7,25 @@ captive portal. Code is taken, in part, from the old ooni-probe, which was written by Jacob Appelbaum and Arturo Filastò. + This module performs multiple tests that match specific vendor captive + portal tests. This is a basic internet captive portal filter tester written + for RECon 2011. + + Read the following URLs to understand the captive portal detection process + for various vendors: + + http://technet.microsoft.com/en-us/library/cc766017%28WS.10%29.aspx + http://blog.superuser.com/2011/05/16/windows-7-network-awareness/ + http://isc.sans.org/diary.html?storyid=10312& + http://src.chromium.org/viewvc/chrome?view=rev&revision=74608 + http://code.google.com/p/chromium-os/issues/detail?3281ttp, + http://crbug.com/52489 + http://crbug.com/71736 + https://bugzilla.mozilla.org/show_bug.cgi?id=562917 + https://bugzilla.mozilla.org/show_bug.cgi?id=603505 + http://lists.w3.org/Archives/Public/ietf-http-wg/2011JanMar/0086.html + http://tools.ietf.org/html/draft-nottingham-http-portal-02 + :copyright: (c) 2012 Isis Lovecruft :license: see LICENSE for more details """ diff --git a/nettests/core/squid.py b/nettests/core/squid.py new file mode 100644 index 0000000..675119c --- /dev/null +++ b/nettests/core/squid.py @@ -0,0 +1,117 @@ +# -*- encoding: utf-8 -*- +# +# Squid transparent HTTP proxy detector +# ************************************* +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni import utils +from ooni.utils import log +from ooni.templates import httpt + +class SquidTest(httpt.HTTPTest): + """ + This test aims at detecting the presence of a squid based transparent HTTP + proxy. It also tries to detect the version number. + """ + name = "Squid test" + author = "Arturo Filastò" + version = 0.1 + + optParameters = [['backend', 'b', 'http://ooni.nu/test/', 'Test backend to use']] + + #inputFile = ['urls', 'f', None, 'Urls file'] + inputs =['http://google.com'] + def test_cacheobject(self): + """ + This detects the presence of a squid transparent HTTP proxy by sending + a request for cache_object://localhost/info. + + The response to this request will usually also contain the squid + version number. + """ + log.debug("Running") + def process_body(body): + if "Access Denied." in body: + self.report['transparent_http_proxy'] = True + else: + self.report['transparent_http_proxy'] = False + + log.msg("Testing Squid proxy presence by sending a request for "\ + "cache_object") + headers = {} + #headers["Host"] = [self.input] + self.report['trans_http_proxy'] = None + method = "GET" + body = "cache_object://localhost/info" + return self.doRequest(self.localOptions['backend'], method=method, body=body, + headers=headers, body_processor=process_body) + + def test_search_bad_request(self): + """ + Attempts to perform a request with a random invalid HTTP method. + + If we are being MITMed by a Transparent Squid HTTP proxy we will get + back a response containing the X-Squid-Error header. + """ + def process_headers(headers): + log.debug("Processing headers in test_search_bad_request") + if 'X-Squid-Error' in headers: + log.msg("Detected the presence of a transparent HTTP "\ + "squid proxy") + self.report['trans_http_proxy'] = True + else: + log.msg("Did not detect the presence of transparent HTTP "\ + "squid proxy") + self.report['transparent_http_proxy'] = False + + log.msg("Testing Squid proxy presence by sending a random bad request") + headers = {} + #headers["Host"] = [self.input] + method = utils.randomSTR(10, True) + self.report['transparent_http_proxy'] = None + return self.doRequest(self.localOptions['backend'], method=method, + headers=headers, headers_processor=process_headers) + + def test_squid_headers(self): + """ + Detects the presence of a squid transparent HTTP proxy based on the + response headers it adds to the responses to requests. + """ + def process_headers(headers): + """ + Checks if any of the headers that squid is known to add match the + squid regexp. + + We are looking for something that looks like this: + + via: 1.0 cache_server:3128 (squid/2.6.STABLE21) + x-cache: MISS from cache_server + x-cache-lookup: MISS from cache_server:3128 + """ + squid_headers = {'via': r'.* \((squid.*)\)', + 'x-cache': r'MISS from (\w+)', + 'x-cache-lookup': r'MISS from (\w+:?\d+?)' + } + + self.report['transparent_http_proxy'] = False + for key in squid_headers.keys(): + if key in headers: + log.debug("Found %s in headers" % key) + m = re.search(squid_headers[key], headers[key]) + if m: + log.msg("Detected the presence of squid transparent"\ + " HTTP Proxy") + self.report['transparent_http_proxy'] = True + + log.msg("Testing Squid proxy by looking at response headers") + headers = {} + #headers["Host"] = [self.input] + method = "GET" + self.report['transparent_http_proxy'] = None + d = self.doRequest(self.localOptions['backend'], method=method, + headers=headers, headers_processor=process_headers) + return d + + diff --git a/old-to-be-ported-code/AUTHORS b/old-to-be-ported-code/AUTHORS deleted file mode 100644 index c6a4ab6..0000000 --- a/old-to-be-ported-code/AUTHORS +++ /dev/null @@ -1,3 +0,0 @@ -Jacob Appelbaum <jacob(a)torproject.org> -Arturo Filasto <hellais(a)torproject.org> -Linus Nordberg <linus(a)torproject.org> diff --git a/old-to-be-ported-code/bin/ooni-probe b/old-to-be-ported-code/bin/ooni-probe deleted file mode 100644 index 9f616bc..0000000 --- a/old-to-be-ported-code/bin/ooni-probe +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -"""\ - This is the example OONI probe command line utility -""" - -import sys - -from ooni.command import Command - -Command(sys.argv[1:]).run() diff --git a/old-to-be-ported-code/ooni/output.py b/old-to-be-ported-code/ooni/output.py deleted file mode 100644 index 48e9f1f..0000000 --- a/old-to-be-ported-code/ooni/output.py +++ /dev/null @@ -1,21 +0,0 @@ -import yaml - -class data: - def __init__(self, name=None): - if name: - self.name = name - - def output(self, data, name=None): - if name: - self.name = name - - stream = open(self.name, 'w') - yaml.dump(data, stream) - stream.close() - def append(self, data, name=None): - if name: - self.name = name - stream = open(self.name, 'a') - yaml.dump([data], stream) - stream.close() - diff --git a/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py b/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py deleted file mode 100644 index 9c0d87c..0000000 --- a/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -# Captive Portal Detection With Multi-Vendor Emulation -# by Jacob Appelbaum <jacob(a)appelbaum.net> -# -# This module performs multiple tests that match specific vendor captive -# portal tests. This is a basic internet captive portal filter tester written -# for RECon 2011. -# -# Read the following URLs to understand the captive portal detection process -# for various vendors: -# -# http://technet.microsoft.com/en-us/library/cc766017%28WS.10%29.aspx -# http://blog.superuser.com/2011/05/16/windows-7-network-awareness/ -# http://isc.sans.org/diary.html?storyid=10312& -# http://src.chromium.org/viewvc/chrome?view=rev&revision=74608 -# http://code.google.com/p/chromium-os/issues/detail?id=3281 -# http://crbug.com/52489 -# http://crbug.com/71736 -# https://bugzilla.mozilla.org/show_bug.cgi?id=562917 -# https://bugzilla.mozilla.org/show_bug.cgi?id=603505 -# http://lists.w3.org/Archives/Public/ietf-http-wg/2011JanMar/0086.html -# http://tools.ietf.org/html/draft-nottingham-http-portal-02 -# - -import sys -import ooni.http -import ooni.dnsooni -import ooni.report - -from ooni.plugooni import Plugoo - -class CaptivePortalPlugin(Plugoo): - def __init__(self): - self.in_ = sys.stdin - self.out = sys.stdout - self.debug = False - self.logger = ooni.report.Log().logger - self.name = "" - self.type = "" - self.paranoia = "" - self.modules_to_import = [] - self.output_dir = "" - self.default_args = "" - - def CaptivePortal_Tests(self): - print "Captive Portal Detection With Multi-Vendor Emulation:" - tests = self.get_tests_by_filter(("_CP_Tests"), (ooni.http, ooni.dnsooni)) - self.run_tests(tests) - - def magic_main(self): - self.run_plgoo_tests("_Tests") - - def ooni_main(self,args): - self.magic_main() diff --git a/old-to-be-ported-code/ooni/plugins/skel_plgoo.py b/old-to-be-ported-code/ooni/plugins/skel_plgoo.py deleted file mode 100644 index f365c06..0000000 --- a/old-to-be-ported-code/ooni/plugins/skel_plgoo.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/python -# This will never load it is just an example of Plugooni plgoo plugins -# -from ooni.plugooni import Plugoo - -class SkelPlugin(Plugoo): - def __init__(self): - self.name = "" - self.type = "" - self.paranoia = "" - self.modules_to_import = [] - self.output_dir = "" - - def ooni_main(self, cmd): - print "This is the main plugin function" - - diff --git a/old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml b/old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml deleted file mode 100644 index 6a91e8a..0000000 --- a/old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- -plugin: - name : Skel - author : Some Name - date_created : 2011-08-01 - modules : [tcp, udp, http] -input: - experiment: - list : ['el1', - 'el2', - 'el3', - 'el4', - 'el5'] - control: - list : ['el1', - 'el2', - 'el3', - 'el4', - 'el5'] -output: - timestamp : - experiment : - timestamp : - test : - result : - extrafield : - control : - timestamp : - test : - result : - extrafield : - - diff --git a/old-to-be-ported-code/ooni/yamlooni.py b/old-to-be-ported-code/ooni/yamlooni.py deleted file mode 100644 index a457217..0000000 --- a/old-to-be-ported-code/ooni/yamlooni.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# -# Plugooni, ooni plugin module for loading plgoo files. -# by Jacob Appelbaum <jacob(a)appelbaum.net> -# Arturo Filasto' <art(a)fuffa.org> - -import sys -import os - -class Yamlooni(): - def __init__(self, name, creator, location): - self.name = name - self.creator = creator - self.location = location - f = open(self.location) - self.ydata = yaml.load(f.read()) - - def debug_print(): - #print y.input - for i in y.iteritems(): - if i[0] == "input": - print "This is the input part:" - for j in i[1].iteritems(): - print j - print "end of the input part.\n" - - elif i[0] == "output": - print "This is the output part:" - for j in i[1].iteritems(): - print j - print "end of the output part.\n" - - elif i[0] == "plugin": - print "This is the Plugin part:" - for j in i[1].iteritems(): - print j - print "end of the plugin part.\n" - - - diff --git a/ooni/nettest.py b/ooni/nettest.py index 3c87b7f..b17117d 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -3,19 +3,9 @@ import itertools import os -from inspect import classify_class_attrs -from pprint import pprint - -from twisted.internet import defer, utils -from twisted.python import usage -from twisted.trial import unittest, itrial -from zope.interface.exceptions import BrokenImplementation - -from ooni.inputunit import InputUnitProcessor -from ooni.utils import log -from ooni.utils.assertions import isClass, isNotClass -from ooni.utils.assertions import isOldStyleClass, isNewStyleClass - +from twisted.trial import unittest, itrial, util +from twisted.internet import defer, utils +from ooni.utils import log pyunit = __import__('unittest') @@ -25,20 +15,46 @@ class InputTestSuite(pyunit.TestSuite): and the tracking of current index via idx. """ def run(self, result, idx=0): + log.debug("Running test suite") self._idx = idx while self._tests: if result.shouldStop: + log.debug("Detected that test should stop") + log.debug("Stopping...") break test = self._tests.pop(0) + try: + log.debug("Setting test attributes with %s %s" % + (self.input, self._idx)) + test.input = self.input test._idx = self._idx + except Exception, e: + log.debug("Error in some stuff") + log.debug(e) + import sys + print sys.exc_info() + + try: + log.debug("Running test") test(result) - except: + log.debug("Ran.") + except Exception, e: + log.debug("Attribute error thing") + log.debug("Had some problems with _idx") + log.debug(e) + import traceback, sys + print sys.exc_info() + traceback.print_exc() + print e + test(result) + self._idx += 1 return result + class TestCase(unittest.TestCase): """ This is the monad of the OONI nettest universe. When you write a nettest @@ -123,60 +139,23 @@ class TestCase(unittest.TestCase): writing. """ if result.reporterFactory.firstrun: + log.debug("Detecting first run. Writing report header.") d1 = result.reporterFactory.writeHeader() d2 = unittest.TestCase.deferSetUp(self, ignored, result) dl = defer.DeferredList([d1, d2]) return dl else: + log.debug("Not first run. Running test setup directly") return unittest.TestCase.deferSetUp(self, ignored, result) - def _raaun(self, methodName, result): - from twisted.internet import reactor - method = getattr(self, methodName) - log.debug("Running %s" % methodName) - d = defer.maybeDeferred( - utils.runWithWarningsSuppressed, self._getSuppress(), method) - d.addBoth(lambda x : call.active() and call.cancel() or x) - return d - - @staticmethod - def inputParser(inputs): - """Replace me with a custom function for parsing inputs.""" - return inputs - - def __input_file_processor__(self, fp): - """ - I open :attr:inputFile if there is one, and return inputs one by one - after stripping them of whitespace and running them through the parser - :meth:`inputParser`. - """ - for line in fp.readlines(): - yield self.inputParser(line.strip()) + def inputProcessor(self, fp): + log.debug("Running default input processor") + for x in fp.readlines(): + yield x.strip() fp.close() - def __get_inputs__(self): - """ - I am called from the ooni.runner and you probably should not override - me. I gather the internal inputs from an instantiated test class and - pass them to the rest of the runner. - - If you are looking for a way to parse inputs from inputFile, see - :meth:`inputParser`. - """ - processor = InputUnitProcessor(self.inputs, - input_filter=None, - catch_err=False) - processed = processor.process() - - log.msg("Received direct inputs:\n%s" % inputs) - log.debug("Our InputUnitProcessor is %s" % processor) - - #while processed is not StopIteration: - # self.inputs = processed - # yield self.inputs - #else: - # if self.inputFile: - + def getOptions(self): + log.debug("Getting options for test") if self.inputFile: try: fp = open(self.inputFile) ## xxx fixme: diff --git a/ooni/reporter.py b/ooni/reporter.py index a7b645b..c12b28f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -4,6 +4,7 @@ import logging import sys import time import yaml +import traceback from yaml.representer import * from yaml.emitter import * @@ -132,8 +133,7 @@ class ReporterFactory(OReporter): client_geodata = {} log.msg("Running geo IP lookup via check.torproject.org") - #client_ip = yield geodata.myIP() - client_ip = '127.0.0.1' + client_ip = yield geodata.myIP() try: import txtorcon client_location = txtorcon.util.NetLocation(client_ip) @@ -200,6 +200,7 @@ class OONIReporter(OReporter): if not self._startTime: self._startTime = self._getTime() + log.debug("Starting test %s" % idx) test.report = {} self._tests[idx] = {} @@ -215,6 +216,7 @@ class OONIReporter(OReporter): def stopTest(self, test): + log.debug("Stopping test") super(OONIReporter, self).stopTest(test) idx = self.getTestIndex(test) @@ -224,11 +226,14 @@ class OONIReporter(OReporter): # XXX In the future this should be removed. try: report = list(test.legacy_report) + log.debug("Set the report to be a list") except: # XXX I put a dict() here so that the object is re-instantiated and I # actually end up with the report I want. This could either be a # python bug or a yaml bug. report = dict(test.report) + log.debug("Set the report to be a dict") + log.debug("Adding to report %s" % report) self._tests[idx]['report'] = report @@ -245,6 +250,7 @@ class OONIReporter(OReporter): Expects that L{_printErrors}, L{_writeln}, L{_write}, L{_printSummary} and L{_separator} are all implemented. """ + log.debug("Test run concluded") if self._publisher is not None: self._publisher.removeObserver(self._observeWarnings) if self._startTime is not None: @@ -261,13 +267,18 @@ class OONIReporter(OReporter): super(OONIReporter, self).addSuccess(test) #self.report['result'] = {'value': 'success'} - def addError(self, *args): - super(OONIReporter, self).addError(*args) - #self.report['result'] = {'value': 'error', 'args': args} + def addError(self, test, exception): + super(OONIReporter, self).addError(test, exception) + exc_type, exc_value, exc_traceback = exception + log.err(exc_type) + log.err(str(exc_value)) + # XXX properly print out the traceback + for line in '\n'.join(traceback.format_tb(exc_traceback)).split("\n"): + log.err(line) def addFailure(self, *args): super(OONIReporter, self).addFailure(*args) - #self.report['result'] = {'value': 'failure', 'args': args} + log.warn(args) def addSkip(self, *args): super(OONIReporter, self).addSkip(*args) diff --git a/ooni/runner.py b/ooni/runner.py index b1a21ac..f352dfb 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -235,13 +235,18 @@ class ORunner(object): def runWithInputUnit(self, inputUnit): idx = 0 result = self.reporterFactory.create() - + log.debug("Running test with input unit %s" % inputUnit) for inputs in inputUnit: result.reporterFactory = self.reporterFactory + log.debug("Running with %s" % inputs) suite = self.baseSuite(self.cases) suite.input = inputs - suite(result, idx) + try: + suite(result, idx) + except Exception, e: + log.err("Error in running test!") + log.err(e) # XXX refactor all of this index bullshit to avoid having to pass # this index around. Probably what I want to do is go and make @@ -250,7 +255,9 @@ class ORunner(object): # We currently need to do this addition in order to get the number # of times the test cases that have run inside of the test suite. idx += (suite._idx - idx) + log.debug("I am now at the index %s" % idx) + log.debug("Finished") result.done() def run(self): diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 6e3163b..f453c74 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -54,6 +54,7 @@ class HTTPTest(TestCase): followRedirects = False def setUp(self): + log.debug("Setting up HTTPTest") try: import OpenSSL except: @@ -76,12 +77,17 @@ class HTTPTest(TestCase): self.request = {} self.response = {} + log.debug("Finished test setup") - def _processResponseBody(self, data): + def _processResponseBody(self, data, body_processor): + log.debug("Processing response body") self.response['body'] = data self.report['response'] = self.response - self.processResponseBody(data) + if body_processor: + body_processor(data) + else: + self.processResponseBody(data) def processResponseBody(self, data): """ @@ -108,7 +114,25 @@ class HTTPTest(TestCase): """ pass - def doRequest(self, url, method="GET", headers=None, body=None): + def doRequest(self, url, method="GET", + headers=None, body=None, headers_processor=None, + body_processor=None): + """ + Perform an HTTP request with the specified method. + + url: the full url path of the request + method: the HTTP Method to be used + headers: the request headers to be sent + body: the request body + headers_processor: a function to be used for processing the HTTP header + responses (defaults to self.processResponseHeaders). + This function takes as argument the HTTP headers as a + dict. + body_processory: a function to be used for processing the HTTP response + body (defaults to self.processResponseBody). + This function takes the response body as an argument. + """ + log.debug("Performing request %s %s %s" % (url, method, headers)) try: d = self.build_request(url, method, headers, body) except Exception, e: @@ -123,11 +147,17 @@ class HTTPTest(TestCase): return d.addErrback(errback) - d.addCallback(self._cbResponse) + d.addCallback(self._cbResponse, headers_processor, body_processor) d.addCallback(finished) return d - def _cbResponse(self, response): + def _cbResponse(self, response, headers_processor, body_processor): + log.debug("Got response %s" % response) + if not response: + self.report['response'] = None + log.err("We got an empty response") + return + self.response['headers'] = list(response.headers.getAllRawHeaders()) self.response['code'] = response.code self.response['length'] = response.length @@ -136,11 +166,14 @@ class HTTPTest(TestCase): if str(self.response['code']).startswith('3'): self.processRedirect(response.headers.getRawHeaders('Location')[0]) - self.processResponseHeaders(self.response['headers']) + if headers_processor: + headers_processor(self.response['headers']) + else: + self.processResponseHeaders(self.response['headers']) finished = defer.Deferred() response.deliverBody(BodyReceiver(finished)) - finished.addCallback(self._processResponseBody) + finished.addCallback(self._processResponseBody, body_processor) return finished diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 38239ba..cd82ab4 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -4,6 +4,9 @@ import imp import logging +import string +import random + try: import yaml except: @@ -143,3 +146,32 @@ class Log(): except: raise StopIteration +def randomSTR(length, num=True): + """ + Returns a random all uppercase alfa-numerical (if num True) string long length + """ + chars = string.ascii_uppercase + if num: + chars += string.digits + return ''.join(random.choice(chars) for x in range(length)) + +def randomstr(length, num=True): + """ + Returns a random all lowercase alfa-numerical (if num True) string long length + """ + chars = string.ascii_lowercase + if num: + chars += string.digits + return ''.join(random.choice(chars) for x in range(length)) + +def randomStr(length, num=True): + """ + Returns a random a mixed lowercase, uppercase, alfanumerical (if num True) + string long length + """ + chars = string.ascii_lowercase + string.ascii_uppercase + if num: + chars += string.digits + return ''.join(random.choice(chars) for x in range(length)) + +

1 0

[ooni-probe/master] Add a very simple test that *must* always pass.
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 4433cadaa39dd84b922329c84a7e791d81cac7c6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Oct 23 18:09:57 2012 +0000 Add a very simple test that *must* always pass. * Useful for testing the newstyle API --- nettests/simpletest.py | 19 +++++++++++++++++++ 1 files changed, 19 insertions(+), 0 deletions(-) diff --git a/nettests/simpletest.py b/nettests/simpletest.py new file mode 100644 index 0000000..7cc9318 --- /dev/null +++ b/nettests/simpletest.py @@ -0,0 +1,19 @@ +from ooni import nettest +class SimpleTest(nettest.TestCase): + inputs = range(1,100) + optParameters = [['asset', 'a', None, 'Asset file'], + ['controlserver', 'c', 'google.com', 'Specify the control server'], + ['resume', 'r', 0, 'Resume at this index'], + ['other', 'o', None, 'Other arguments']] + + def test_foo(self, *arg, **kw): + print "Running %s with %s" % ("test_foo", self.input) + self.report['test_foo'] = 'Antani' + self.report['shared'] = "sblinda" + self.assertEqual(1,1) + + def test_f4oo(self, *arg, **kw): + print "Running %s with %s" % ("test_f4oo", self.input) + self.report['test_f4oo'] = 'Antani' + self.report['shared'] = "sblinda2" + self.assertEqual(1,1)

1 0

[ooni-probe/master] Refactor runner and ooni
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 07addc1b4e1c60ede34b166a5d1208f183f47530 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Oct 24 15:32:31 2012 +0000 Refactor runner and ooni * Remove unused imports * Fix import indentation (please do not align from modules) --- ooni/__init__.py | 15 ++++++- ooni/runner.py | 121 +++++++++++++++++++++++++++++++----------------------- 2 files changed, 84 insertions(+), 52 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index a03c856..e730eb5 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1 +1,14 @@ -__all__ = ['plugoo', 'utils', 'config', 'logo', 'lib', 'log'] +from . import config +from . import input +from . import inputunit +from . import kit +from . import lib +from . import nettest +from . import oonicli +from . import reporter +from . import runner +from . import template +from . import utils +__all__ = ['config', 'input', 'inputunit', 'kit', + 'lib', 'nettest', 'oonicli', 'reporter', + 'runner', 'templates', 'utils'] diff --git a/ooni/runner.py b/ooni/runner.py index f352dfb..dd5f8a9 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -10,40 +10,21 @@ # :version: 0.1.0-pre-alpha # - -import os -import sys -import types -import time import inspect -import yaml - -from pprint import pprint -from twisted.internet import defer, reactor -from twisted.python import reflect, failure, usage -from twisted.python import log as tlog +from twisted.python import reflect, usage -from twisted.trial import unittest -from twisted.trial.runner import TrialRunner, TestLoader -from twisted.trial.runner import isPackage, isTestCase, ErrorHolder -from twisted.trial.runner import filenameToModule, _importFromFile +from twisted.trial.runner import isTestCase +from twisted.trial.runner import filenameToModule -from ooni import nettest -from ooni.inputunit import InputUnitFactory -from ooni.nettest import InputTestSuite -from ooni.plugoo import tests as oonitests -from ooni.reporter import ReporterFactory -from ooni.utils import log, geodata, date +from ooni.inputunit import InputUnitFactory +from ooni.nettest import InputTestSuite +from ooni.plugoo import tests as oonitests +from ooni.reporter import ReporterFactory +from ooni.utils import log, date from ooni.utils.legacy import LegacyOONITest from ooni.utils.legacy import start_legacy_test, adapt_legacy_test -def isTestCase(thing): - try: - return issubclass(thing, unittest.TestCase) - except TypeError: - return False - def isLegacyTest(obj): """ Returns True if the test in question is written using the OONITest legacy @@ -55,6 +36,45 @@ def isLegacyTest(obj): except TypeError: return False +def processTest(obj, config): + """ + Process the parameters and :class:`twisted.python.usage.Options` of a + :class:`ooni.nettest.Nettest`. + + :param obj: + An uninstantiated old test, which should be a subclass of + :class:`ooni.plugoo.tests.OONITest`. + :param config: + A configured and instantiated :class:`twisted.python.usage.Options` + class. + """ + + input_file = obj.inputFile + + if obj.optParameters or input_file: + if not obj.optParameters: + obj.optParameters = [] + + if input_file: + obj.optParameters.append(input_file) + + class Options(usage.Options): + optParameters = obj.optParameters + + options = Options() + options.parseOptions(config['subArgs']) + obj.localOptions = options + + if input_file: + obj.inputFile = options[input_file[0]] + try: + tmp_obj = obj() + tmp_obj.getOptions() + except usage.UsageError: + options.opt_help() + + return obj + def findTestClassesFromConfig(config): """ Takes as input the command line config parameters and returns the test @@ -82,14 +102,14 @@ def findTestClassesFromConfig(config): classes.append(adapt_legacy_test(val, config)) return classes -def makeTestCases(klass, tests, methodPrefix): +def makeTestCases(klass, tests, method_prefix): """ - Takes a class some tests and returns the test cases. methodPrefix is how + Takes a class some tests and returns the test cases. method_prefix is how the test case functions should be prefixed with. """ cases = [] for test in tests: - cases.append(klass(methodPrefix+test)) + cases.append(klass(method_prefix+test)) return cases def processTestOptions(cls, config): @@ -144,10 +164,10 @@ def loadTestsAndOptions(classes, config): Takes a list of test classes and returns their testcases and options. Legacy tests will be adapted. """ - methodPrefix = 'test' - suiteFactory = InputTestSuite + + method_prefix = 'test' options = [] - testCases = [] + test_cases = [] names = [] _old_class_type = LegacyOONITest @@ -155,8 +175,12 @@ def loadTestsAndOptions(classes, config): for cls in classes: if isinstance(cls, _old_class_type): try: - cases = start_legacy_test(cls) - testCases.append(cases) + cases = start_legacy_test(klass) + if cases: + log.debug("Processing cases") + log.debug(str(cases)) + return [], [] + test_cases.append(cases) except Exception, e: log.err(e) else: @@ -170,10 +194,10 @@ def loadTestsAndOptions(classes, config): print cases return [], [] else: - tests = reflect.prefixedMethodNames(cls, methodPrefix) + tests = reflect.prefixedMethodNames(klass, method_prefix) if tests: - cases = makeTestCases(cls, tests, methodPrefix) - testCases.append(cases) + cases = makeTestCases(klass, tests, method_prefix) + test_cases.append(cases) try: #c = cls() #cls, opts = processTestOptions(cls, config) @@ -191,8 +215,7 @@ def loadTestsAndOptions(classes, config): opts.update(inputs) options.append(opts) - log.debug("runner.loadTestsAndOptions: OPTIONS: %s" % options) - return testCases, options + return test_cases, options class ORunner(object): """ @@ -201,7 +224,7 @@ class ORunner(object): them in input units. I also create all the report instances required to run the tests. """ - def __init__(self, cases, options=None, config=None, *arg, **kw): + def __init__(self, cases, options=None, config=None): self.baseSuite = InputTestSuite self.cases = cases self.options = options @@ -232,21 +255,17 @@ class ORunner(object): reportFile, testSuite=self.baseSuite(self.cases) ) - def runWithInputUnit(self, inputUnit): + def runWithInputUnit(self, input_unit): idx = 0 result = self.reporterFactory.create() - log.debug("Running test with input unit %s" % inputUnit) - for inputs in inputUnit: + log.debug("Running test with input unit %s" % input_unit) + for inputs in input_unit: result.reporterFactory = self.reporterFactory log.debug("Running with %s" % inputs) suite = self.baseSuite(self.cases) suite.input = inputs - try: - suite(result, idx) - except Exception, e: - log.err("Error in running test!") - log.err(e) + suite(result, idx) # XXX refactor all of this index bullshit to avoid having to pass # this index around. Probably what I want to do is go and make @@ -262,5 +281,5 @@ class ORunner(object): def run(self): self.reporterFactory.options = self.options - for inputUnit in InputUnitFactory(self.inputs): - self.runWithInputUnit(inputUnit) + for input_unit in InputUnitFactory(self.inputs): + self.runWithInputUnit(input_unit)

1 0

[ooni-probe/master] Further refactoring of nettest and runner
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit d95034f8eb912771bcf2b0613d49ab307880246d Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Oct 24 15:49:06 2012 +0000 Further refactoring of nettest and runner * Add some notes on running tests in threads * Define and document _tests attribute of out nettest TestSuite * Remove unused code --- ooni/nettest.py | 69 +++++++++++++++++++----------------------------------- ooni/runner.py | 1 - 2 files changed, 24 insertions(+), 46 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index b17117d..1d46218 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -1,10 +1,17 @@ -# -*- coding: utf-8 -*- +# -*- encoding: utf-8 -*- +# +# :authors: Arturo "hellais" Filastò <art(a)fuffa.org> +# :licence: see LICENSE -import itertools +import sys import os +import itertools +import traceback from twisted.trial import unittest, itrial, util from twisted.internet import defer, utils +from twisted.python import usage + from ooni.utils import log pyunit = __import__('unittest') @@ -14,6 +21,10 @@ class InputTestSuite(pyunit.TestSuite): This in an extension of a unittest test suite. It adds support for inputs and the tracking of current index via idx. """ + + # This is used to keep track of the tests that are associated with our + # special test suite + _tests = None def run(self, result, idx=0): log.debug("Running test suite") self._idx = idx @@ -31,25 +42,16 @@ class InputTestSuite(pyunit.TestSuite): test.input = self.input test._idx = self._idx except Exception, e: - log.debug("Error in some stuff") + log.debug("Error in setting test attributes") + log.debug("This is probably because the test case you are "\ + "running is not a nettest") log.debug(e) - import sys - print sys.exc_info() - try: - log.debug("Running test") - test(result) - log.debug("Ran.") - except Exception, e: - log.debug("Attribute error thing") - log.debug("Had some problems with _idx") - log.debug(e) - import traceback, sys - print sys.exc_info() - traceback.print_exc() - print e - - test(result) + log.debug("Running test") + # XXX we may want in a future to put all of these tests inside of a + # thread pool and run them all in parallel + test(result) + log.debug("Ran.") self._idx += 1 return result @@ -95,9 +97,9 @@ class TestCase(unittest.TestCase): * version: is the version string of the test. """ - name = "I Did Not Change The Name" - author = "John Doe <foo(a)example.com>" - version = "0.0.0" + name = "I Did Not Change The Name" + author = "Jane Doe <foo(a)example.com>" + version = "0" inputFile = None inputs = [None] @@ -109,29 +111,6 @@ class TestCase(unittest.TestCase): optFlags = None subCommands = None - def setUpClass(self, *args, **kwargs): - """ - Create a TestCase instance. This function is equivalent to '__init__'. - To add futher setup steps before a set of tests in a TestCase instance - run, create a function called 'setUp'. - - Class attributes, such as `report`, `optParameters`, `name`, and - `author` should be overriden statically as class attributes in any - subclass of :class:`ooni.nettest.TestCase`, so that the calling - functions in ooni.runner can handle them correctly. - """ - methodName = 'runTest' - if kwargs: - if 'methodName' in kwargs: - methodName = kwargs['methodName'] - - super(TestCase, self).__init__(methodName=methodName) - - #for key, value in kwargs.items(): - # setattr(self.__class__, key, value) - # - #self.inputs = self.getInputs() - def deferSetUp(self, ignored, result): """ If we have the reporterFactory set we need to write the header. If diff --git a/ooni/runner.py b/ooni/runner.py index dd5f8a9..ff5dd33 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -168,7 +168,6 @@ def loadTestsAndOptions(classes, config): method_prefix = 'test' options = [] test_cases = [] - names = [] _old_class_type = LegacyOONITest

1 0

[ooni-probe/master] Add to global import table legacy modules
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 3edb0341eee6ff11b6fd7e1108a4bce00a5e89e4 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Oct 24 15:40:03 2012 +0000 Add to global import table legacy modules * Refactor test case directory structure separating legacy tests from new tests --- ooni/__init__.py | 13 ++++++++++- tests/legacy/test_plugins.py | 40 +++++++++++++++++++++++++++++++++++++ tests/legacy/test_tests.py | 45 ++++++++++++++++++++++++++++++++++++++++++ tests/test_plugins.py | 40 ------------------------------------- tests/test_tests.py | 45 ------------------------------------------ 5 files changed, 96 insertions(+), 87 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index e730eb5..9a4e60f 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -7,8 +7,17 @@ from . import nettest from . import oonicli from . import reporter from . import runner -from . import template +from . import templates from . import utils + +# XXX below are legacy related modules +from . import ooniprobe +from . import plugoo +from . import plugins +from . import oonitests + __all__ = ['config', 'input', 'inputunit', 'kit', 'lib', 'nettest', 'oonicli', 'reporter', - 'runner', 'templates', 'utils'] + 'runner', 'templates', 'utils', + # XXX below are legacy related modules + 'ooniprobe', 'plugoo', 'plugins', 'oonitests'] diff --git a/tests/legacy/test_plugins.py b/tests/legacy/test_plugins.py new file mode 100644 index 0000000..26bee51 --- /dev/null +++ b/tests/legacy/test_plugins.py @@ -0,0 +1,40 @@ +from twisted.internet import defer, reactor +from twisted.trial import unittest + +from ooni.ooniprobe import retrieve_plugoo, runTest, Options +from ooni.plugoo import work, tests + +def asset_file(filename): + import os + file_dir = os.path.normpath(os.path.join(__file__, '..')) + return os.path.join(file_dir, 'assets', filename) + +class PluginsTestCase(unittest.TestCase): + def test_plugin_blocking(self): + suboptions = {'asset': asset_file('urllist.txt')} + runTest('blocking', suboptions, Options(), reactor) + return + + def test_plugin_tcpconnect(self): + suboptions = {'asset': asset_file('ipports.txt')} + runTest('tcpconnect', suboptions, Options(), reactor) + return + + + def test_plugin_captivep(self): + runTest('blocking', None, Options(), reactor) + return + + + def test_plugin_httphost(self): + suboptions = {'asset': asset_file('urllist.txt')} + runTest('httphost', suboptions, Options(), reactor) + return + + + def test_plugin_httpt(self): + suboptions = {'urls': asset_file('urllist.txt')} + runTest('httpt', suboptions, Options(), reactor) + return + + diff --git a/tests/legacy/test_tests.py b/tests/legacy/test_tests.py new file mode 100644 index 0000000..dc89b98 --- /dev/null +++ b/tests/legacy/test_tests.py @@ -0,0 +1,45 @@ +from twisted.internet import defer +from twisted.trial import unittest + +from ooni.plugoo import work, tests + +class TestsTestCase(unittest.TestCase): + def setUp(self): + class dummyReport: + def __call__(self, *args, **kw): + pass + self.dummyreport = dummyReport() + self.callbackResults = None + self.errbackResults = None + + def _callback(self, *args, **kw): + #print args, kw + self.callbackResults = args, kw + + def _errback(self, *args, **kw): + pass + + @defer.inlineCallbacks + def test_fallThrough(self): + """ + This tests to make sure that what is returned from the experiment + method falls all the way through to control and finish. + """ + test_dict = {"hello": "world"} + class DummyTest(tests.OONITest): + blocking = False + def experiment(self, args): + def bla(a): + print a + return test_dict + d2 = defer.Deferred() + d2.addCallback(bla) + from twisted.internet import reactor + reactor.callLater(0.1, d2.callback, None) + return d2 + + test = DummyTest(None, None, self.dummyreport) + yield test.startTest(None).addCallback(self._callback) + self.assertEqual(self.callbackResults[0][0]['return_value'], test_dict) + return + diff --git a/tests/test_plugins.py b/tests/test_plugins.py deleted file mode 100644 index 26bee51..0000000 --- a/tests/test_plugins.py +++ /dev/null @@ -1,40 +0,0 @@ -from twisted.internet import defer, reactor -from twisted.trial import unittest - -from ooni.ooniprobe import retrieve_plugoo, runTest, Options -from ooni.plugoo import work, tests - -def asset_file(filename): - import os - file_dir = os.path.normpath(os.path.join(__file__, '..')) - return os.path.join(file_dir, 'assets', filename) - -class PluginsTestCase(unittest.TestCase): - def test_plugin_blocking(self): - suboptions = {'asset': asset_file('urllist.txt')} - runTest('blocking', suboptions, Options(), reactor) - return - - def test_plugin_tcpconnect(self): - suboptions = {'asset': asset_file('ipports.txt')} - runTest('tcpconnect', suboptions, Options(), reactor) - return - - - def test_plugin_captivep(self): - runTest('blocking', None, Options(), reactor) - return - - - def test_plugin_httphost(self): - suboptions = {'asset': asset_file('urllist.txt')} - runTest('httphost', suboptions, Options(), reactor) - return - - - def test_plugin_httpt(self): - suboptions = {'urls': asset_file('urllist.txt')} - runTest('httpt', suboptions, Options(), reactor) - return - - diff --git a/tests/test_tests.py b/tests/test_tests.py deleted file mode 100644 index dc89b98..0000000 --- a/tests/test_tests.py +++ /dev/null @@ -1,45 +0,0 @@ -from twisted.internet import defer -from twisted.trial import unittest - -from ooni.plugoo import work, tests - -class TestsTestCase(unittest.TestCase): - def setUp(self): - class dummyReport: - def __call__(self, *args, **kw): - pass - self.dummyreport = dummyReport() - self.callbackResults = None - self.errbackResults = None - - def _callback(self, *args, **kw): - #print args, kw - self.callbackResults = args, kw - - def _errback(self, *args, **kw): - pass - - @defer.inlineCallbacks - def test_fallThrough(self): - """ - This tests to make sure that what is returned from the experiment - method falls all the way through to control and finish. - """ - test_dict = {"hello": "world"} - class DummyTest(tests.OONITest): - blocking = False - def experiment(self, args): - def bla(a): - print a - return test_dict - d2 = defer.Deferred() - d2.addCallback(bla) - from twisted.internet import reactor - reactor.callLater(0.1, d2.callback, None) - return d2 - - test = DummyTest(None, None, self.dummyreport) - yield test.startTest(None).addCallback(self._callback) - self.assertEqual(self.callbackResults[0][0]['return_value'], test_dict) - return -

1 0

[ooni-probe/master] Refactor reporter and removal of dead code
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 71e8bc3664c7acb35d308120dc504880c3168e8d Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Oct 24 15:59:53 2012 +0000 Refactor reporter and removal of dead code --- ooni/oonicli.py | 27 ++++----- ooni/oonitests/httphost.py | 135 -------------------------------------------- ooni/oonitests/squid.py | 113 ------------------------------------ ooni/oonitests/template.py | 65 --------------------- ooni/reporter.py | 26 ++++++--- 5 files changed, 29 insertions(+), 337 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 5b0964b..9f1ec21 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -13,29 +13,24 @@ # original copyright (c) by Twisted Matrix Laboratories. -import sys, os, random, gc, time, warnings - -import unittest -import inspect - -from ooni.inputunit import InputUnitFactory -from ooni.reporter import ReporterFactory -from ooni.nettest import InputTestSuite -from ooni.plugoo import tests -from ooni import nettest, runner, reporter +import sys +import os +import random +import time from twisted.internet import defer from twisted.application import app -from twisted.python import usage, reflect, failure -from twisted.python.filepath import FilePath -from twisted import plugin +from twisted.python import usage, failure from twisted.python.util import spewer -from twisted.python.compat import set -from twisted.trial import itrial -from twisted.trial import runner as irunner +from ooni import nettest, runner, reporter + +from ooni.inputunit import InputUnitFactory +from ooni.reporter import ReporterFactory +from ooni.nettest import InputTestSuite from ooni.utils import log + class Options(usage.Options, app.ReactorSelectionMixin): synopsis = """%s [options] [[file|package|module|TestCase|testmethod]...] """ % (os.path.basename(sys.argv[0]),) diff --git a/ooni/oonitests/__init__.py b/ooni/oonitests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ooni/oonitests/httphost.py b/ooni/oonitests/httphost.py deleted file mode 100644 index 25adff2..0000000 --- a/ooni/oonitests/httphost.py +++ /dev/null @@ -1,135 +0,0 @@ -""" - HTTP Host based filtering - ************************* - - This test detect HTTP Host field - based filtering. - It is used to detect censorship on - performed with Web Guard (used by - T-Mobile US). -""" -import os -from datetime import datetime -from gevent import monkey - -import urllib2 -import httplib -# WARNING! Using gevent's socket -# introduces the 0x20 DNS "feature". -# This will result is weird DNS requests -# appearing on the wire. -monkey.patch_socket() - -try: - from BeautifulSoup import BeautifulSoup -except: - print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/" - -from plugoo.assets import Asset -from plugoo.tests import Test - -__plugoo__ = "HTTP Host" -__desc__ = "This detects HTTP Host field based filtering" - -class HTTPHostAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - return line.split(',')[1].replace('\n','') - -class HTTPHost(Test): - """ - The main Test class - """ - - def check_response(self, response): - soup = BeautifulSoup(response) - if soup.head.title.string == "Content Filtered": - # Response indicates censorship - return True - else: - # Response does not indicate censorship - return False - - - def is_censored(self, response): - if response: - soup = BeautifulSoup(response) - censored = self.check_response(response) - else: - censored = "unreachable" - return censored - - def urllib2_test(self, control_server, host): - req = urllib2.Request(control_server) - req.add_header('Host', host) - try: - r = urllib2.urlopen(req) - response = r.read() - censored = self.is_censored(response) - except Exception, e: - censored = "Error! %s" % e - - return censored - - def httplib_test(self, control_server, host): - try: - conn = httplib.HTTPConnection(control_server) - conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) - conn.putheader("Host", host) - conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") - conn.endheaders() - r = conn.getresponse() - response = r.read() - censored = self.is_censored(response) - except Exception, e: - censored = "Error! %s" % e - - return censored - - - def experiment(self, *a, **kw): - """ - Try to connect to the control server with - the specified host field. - """ - host = kw['data'] - control_server = kw['control_server'] - self.logger.info("Testing %s (%s)" % (host, control_server)) - - #censored = self.urllib2_test(control_server, host) - censored = self.httplib_test(control_server, host) - - self.logger.info("%s: %s" % (host, censored)) - return {'Time': datetime.now(), - 'Host': host, - 'Censored': censored} - - -def run(ooni): - """ - This is the function that will be called by OONI - and it is responsible for instantiating and passing - the arguments to the Test class. - """ - config = ooni.config - - # This the assets array to be passed to the run function of - # the test - assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \ - "top-1m.csv"))] - - # Instantiate the Test - thetest = HTTPHost(ooni) - ooni.logger.info("starting HTTP Host Test...") - # Run the test with argument assets - thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'}) - ooni.logger.info("finished.") - - diff --git a/ooni/oonitests/squid.py b/ooni/oonitests/squid.py deleted file mode 100644 index 847135e..0000000 --- a/ooni/oonitests/squid.py +++ /dev/null @@ -1,113 +0,0 @@ -""" - Squid Proxy Detector - ******************** - -""" -import os -import httplib -import urllib2 -from urlparse import urlparse - -from plugoo import gen_headers -from plugoo.assets import Asset -from plugoo.tests import Test - -__plugoo__ = "SquidProxy" -__desc__ = "This Test aims at detecting the squid transparent proxy" - -class SquidAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - -class Squid(Test): - """ - Squid Proxy testing class. - """ - def _http_request(self, method, url, - path=None, headers=None): - """ - Perform an HTTP Request. - XXX move this function to the core OONI - code. - """ - url = urlparse(url) - host = url.netloc - - conn = httplib.HTTPConnection(host, 80) - conn.connect() - - if path is None: - path = purl.path - - conn.putrequest(method, path) - - for h in gen_headers(): - conn.putheaders(h[0], h[1]) - conn.endheaders() - - send_browser_headers(self, None, conn) - - response = conn.getresponse() - - headers = dict(response.getheaders()) - - self.headers = headers - self.data = response.read() - return True - - def invalid_request(self): - """ - This will trigger squids "Invalid Request" error. - """ - pass - - def cache_object(self): - """ - This attempts to do a GET cache_object://localhost/info on - any destination and checks to see if the response contains - is that of Squid. - """ - - pass - - def experiment(self, *a, **kw): - """ - Fill this up with the tasks that should be performed - on the "dirty" network and should be compared with the - control. - """ - - - def control(self): - """ - Fill this up with the control related code. - """ - return True - -def run(ooni): - """ - This is the function that will be called by OONI - and it is responsible for instantiating and passing - the arguments to the Test class. - """ - config = ooni.config - - # This the assets array to be passed to the run function of - # the test - assets = [TestTemplateAsset(os.path.join(config.main.assetdir, \ - "someasset.txt"))] - - # Instantiate the Test - thetest = TestTemplate(ooni) - ooni.logger.info("starting SquidProxyTest...") - # Run the test with argument assets - thetest.run(assets) - ooni.logger.info("finished.") - - diff --git a/ooni/oonitests/template.py b/ooni/oonitests/template.py deleted file mode 100644 index f5f9d93..0000000 --- a/ooni/oonitests/template.py +++ /dev/null @@ -1,65 +0,0 @@ -""" - Test Template - ************* - - This is a Test template, to be used when writing your - own OONI probe Tests. -""" -import os - -from plugoo.assets import Asset -from plugoo.tests import Test - -__plugoo__ = "Test Template" -__desc__ = "This a test template to be used to build your own tests" - -class TestTemplateAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - -class TestTemplate(Test): - """ - The main Test class - """ - - def experiment(self, *a, **kw): - """ - Fill this up with the tasks that should be performed - on the "dirty" network and should be compared with the - control. - """ - return False - - def control(self): - """ - Fill this up with the control related code. - """ - return True - -def run(ooni): - """ - This is the function that will be called by OONI - and it is responsible for instantiating and passing - the arguments to the Test class. - """ - config = ooni.config - - # This the assets array to be passed to the run function of - # the test - assets = [TestTemplateAsset(os.path.join(config.main.assetdir, \ - "someasset.txt"))] - - # Instantiate the Test - thetest = TestTemplate(ooni) - ooni.logger.info("starting TestTemplate...") - # Run the test with argument assets - thetest.run(assets) - ooni.logger.info("finished.") - - diff --git a/ooni/reporter.py b/ooni/reporter.py index c12b28f..fc5dde9 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -1,4 +1,3 @@ -import copy_reg import itertools import logging import sys @@ -12,8 +11,8 @@ from yaml.serializer import * from yaml.resolver import * from datetime import datetime -from twisted.python.util import OrderedDict, untilConcludes -from twisted.trial import unittest, reporter, runner +from twisted.python.util import untilConcludes +from twisted.trial import reporter from twisted.internet import defer from ooni.utils import date, log, geodata @@ -30,6 +29,11 @@ except: pyunit = __import__('unittest') class OSafeRepresenter(SafeRepresenter): + """ + This is a custom YAML representer that allows us to represent reports + safely. + It extends the SafeRepresenter to be able to also represent complex numbers + """ def represent_complex(self, data): if data.imag == 0.0: data = u'%r' % data.real @@ -45,7 +49,10 @@ OSafeRepresenter.add_representer(complex, OSafeRepresenter.represent_complex) class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): - + """ + This is a modification of the YAML Safe Dumper to use our own Safe + Representer that supports complex numbers. + """ def __init__(self, stream, default_style=None, default_flow_style=None, canonical=None, indent=None, width=None, @@ -64,6 +71,9 @@ class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver): def safe_dump(data, stream=None, **kw): + """ + Safely dump to a yaml file the specified data. + """ return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw) class OReporter(pyunit.TestResult): @@ -88,8 +98,8 @@ class OReporter(pyunit.TestResult): def _getTime(self): return time.time() - def _write(self, format, *args): - s = str(format) + def _write(self, format_string, *args): + s = str(format_string) assert isinstance(s, type('')) if args: self._stream.write(s % args) @@ -97,8 +107,8 @@ class OReporter(pyunit.TestResult): self._stream.write(s) untilConcludes(self._stream.flush) - def _writeln(self, format, *args): - self._write(format, *args) + def _writeln(self, format_string, *args): + self._write(format_string, *args) self._write('\n') def writeYamlLine(self, line):

1 0

[ooni-probe/master] Use log.msg instead of print in dnstamper
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit af87339f911e0dbeaca40e017c2d1b8899e91b2f Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Oct 24 16:26:46 2012 +0000 Use log.msg instead of print in dnstamper * Minor cleanup --- nettests/core/dnstamper.py | 31 +++++++++++++++++-------------- 1 files changed, 17 insertions(+), 14 deletions(-) diff --git a/nettests/core/dnstamper.py b/nettests/core/dnstamper.py index d2eebf8..b5fcea3 100644 --- a/nettests/core/dnstamper.py +++ b/nettests/core/dnstamper.py @@ -69,7 +69,7 @@ class DNSTamperTest(nettest.TestCase): fp.close() def process_a_answers(self, answers, resolver): - print "Processing A answers for %s" % resolver + log.msg("Processing A answers for %s" % resolver) all_a = [] a_a = [] for answer in answers[0]: @@ -88,10 +88,10 @@ class DNSTamperTest(nettest.TestCase): else: self.test_a_lookups[resolver] = a_a self.report['test_lookups'][resolver] = all_a - print "Done" + log.msg("Done") def process_ptr_answers(self, answers, resolver): - print "Processing PTR answers for %s" % resolver + log.msg("Processing PTR answers for %s" % resolver) name = None for answer in answers[0]: if answer.type is 12: @@ -107,15 +107,15 @@ class DNSTamperTest(nettest.TestCase): def ptr_lookup_error(self, failure, resolver): #def ptr_lookup_error(self, *arg, **kw): - print "There was an error in PTR lookup %s" % resolver - print failure + log.msg("There was an error in PTR lookup %s" % resolver) + log.msg(failure) if resolver == 'control': self.report['control_reverse'] = None else: self.report['test_reverse'][resolver] = None def a_lookup_error(self, failure, resolver): - print "There was an error in A lookup %s" % resolver + log.msg("There was an error in A lookup %s" % resolver) if failure.type is DNSQueryRefusedError: self.report['tampering'][resolver] = 'connection-refused' @@ -147,7 +147,7 @@ class DNSTamperTest(nettest.TestCase): If they do not match then censorship is probably going on (tampering: true). """ - print "Doing the test lookups on %s" % self.input + log.msg("Doing the test lookups on %s" % self.input) list_of_ds = [] hostname = self.input @@ -159,7 +159,7 @@ class DNSTamperTest(nettest.TestCase): control_r.addErrback(self.a_lookup_error, 'control') for test_resolver in self.test_resolvers: - print "Going for %s" % test_resolver + log.msg("Going for %s" % test_resolver) resolver = [(test_resolver, 53)] res = client.createResolver(servers=resolver, resolvconf='') #res = self.createResolver(servers=resolver) @@ -181,7 +181,7 @@ class DNSTamperTest(nettest.TestCase): return r def do_reverse_lookups(self, result): - print "Doing the reverse lookups %s" % self.input + log.msg("Doing the reverse lookups %s" % self.input) list_of_ds = [] resolver = [(self.localOptions['controlresolver'], 53)] @@ -209,11 +209,11 @@ class DNSTamperTest(nettest.TestCase): return dl def compare_results(self, *arg, **kw): - print "Comparing results for %s" % self.input - print self.test_a_lookups + log.msg("Comparing results for %s" % self.input) + log.msg(self.test_a_lookups) for test, test_a_lookups in self.test_a_lookups.items(): - print "Now doing %s | %s" % (test, test_a_lookups) + log.msg("Now doing %s | %s" % (test, test_a_lookups)) if not test_a_lookups: self.report['tampering'][test] = 'unknown' continue @@ -234,5 +234,8 @@ class DNSTamperTest(nettest.TestCase): if len(self.test_a_lookups) == len(self.test_resolvers): self.end() else: - print "Still missing %s - %s" % (len(self.test_a_lookups), - len(self.test_resolvers)) + missing_tests = len(self.test_a_lookups) + missing_resolvers = len(self.test_resolvers) + log.msg("Still missing %s resolvers and %s tests" % + (missing_tests, missing_resolvers)) +

1 0

[ooni-probe/master] Implement reporter for OONIB
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 9526df1952f840cacddfa409a4afe7953919bd18 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Oct 31 14:06:19 2012 +0000 Implement reporter for OONIB * Fix bad bug in reporter for OONIC --- ooni/reporter.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 files changed, 50 insertions(+), 6 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 4d09729..14327ef 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -3,6 +3,7 @@ import logging import sys import time import yaml +import json import traceback from yaml.representer import * @@ -14,6 +15,8 @@ from datetime import datetime from twisted.python.util import untilConcludes from twisted.trial import reporter from twisted.internet import defer + +from ooni.templates.httpt import BodyReceiver, StringProducer from ooni.utils import date, log, geodata try: @@ -76,6 +79,47 @@ def safe_dump(data, stream=None, **kw): """ return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw) +class OONIBReporter(object): + def __init__(self, backend_url): + from twisted.web.client import Agent + from twisted.internet import reactor + + self.agent = Agent(reactor) + self.backend_url = backend_url + + def _newReportCreated(self, data): + #log.debug("Got this as result: %s" % data) + print "Got this as result: %s" % data + + return data + + def _processResponseBody(self, response, body_cb): + #log.debug("Got response %s" % response) + print "Got response %s" % response + + done = defer.Deferred() + response.deliverBody(BodyReceiver(done)) + done.addCallback(body_cb) + return done + + def newReport(self, test_name, test_version): + url = self.backend_url + '/new' + print "Creating report via url %s" % url + + software_version = '0.0.1' + + request = {'software_name': 'ooni-probe', + 'software_version': software_version, + 'test_name': test_name, 'test_version': test_version, + 'progress': 0} + + #log.debug("Creating report via url %s" % url) + bodyProducer = StringProducer(json.dumps(request)) + d = self.agent.request("POST", url, bodyProducer=bodyProducer) + d.addCallback(self._processResponseBody, self._newReportCreated) + return d + + class OReporter(pyunit.TestResult): """ This is an extension of the unittest TestResult. It adds support for @@ -269,11 +313,11 @@ class OONIReporter(OReporter): self.writeYamlLine(self.report) def addSuccess(self, test): - OONIReporter.addSuccess(self, test) + OReporter.addSuccess(self, test) #self.report['result'] = {'value': 'success'} def addError(self, test, exception): - OONIReporter.addError(self, test, exception) + OReporter.addError(self, test, exception) exc_type, exc_value, exc_traceback = exception log.err(exc_type) log.err(str(exc_value)) @@ -282,19 +326,19 @@ class OONIReporter(OReporter): log.err(line) def addFailure(self, *args): - OONIReporter.addFailure(self, *args) + OReporter.addFailure(self, *args) log.warn(args) def addSkip(self, *args): - OONIReporter.addSkip(self, *args) + OReporter.addSkip(self, *args) #self.report['result'] = {'value': 'skip', 'args': args} def addExpectedFailure(self, *args): - OONIReporter.addExpectedFailure(self, *args) + OReporter.addExpectedFailure(self, *args) #self.report['result'] = {'value': 'expectedFailure', 'args': args} def addUnexpectedSuccess(self, *args): - OONIReporter.addUnexpectedSuccess(self, *args) + OReporter.addUnexpectedSuccess(self, *args) #self.report['result'] = {'args': args, 'value': 'unexpectedSuccess'}

1 0

[ooni-probe/master] Avoid doing wildcard import of scapy.all
by isis＠torproject.org 01 Nov '12

01 Nov '12

commit 628684e9aee6482cf7da8bbf0999412c58dac7c9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed Oct 24 16:17:19 2012 +0000 Avoid doing wildcard import of scapy.all * Your code has been rated at 5.68/10 (previous run: 5.63/10) --- ooni/protocols/daphn3.py | 2 +- ooni/templates/scapyt.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/protocols/daphn3.py b/ooni/protocols/daphn3.py index a4cc32a..37c94c7 100644 --- a/ooni/protocols/daphn3.py +++ b/ooni/protocols/daphn3.py @@ -4,7 +4,7 @@ import yaml from twisted.internet import protocol, defer from twisted.internet.error import ConnectionDone -from scapy.all import * +from scapy.all import IP, Raw, rdpcap from ooni.utils import log from ooni.plugoo import reports diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index f561d66..77f2807 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -9,7 +9,7 @@ from twisted.python import usage from twisted.plugin import IPlugin from twisted.internet import protocol, defer -from scapy.all import * +from scapy.all import IP, TCP from ooni.nettest import TestCase from ooni.utils import log

1 0