[tor-commits] [ooni-probe/master] Port squid transparent HTTP proxy detector to new API

commit f5acc677c00b8ab83bc6dc41fdd09ac487acdfc0 Author: Arturo Filastò arturo@filasto.net Date: Tue Oct 23 18:04:36 2012 +0000

Port squid transparent HTTP proxy detector to new API * Remove some dead code * Move authors to new directory * Make reporter not swallow all tracebacks * Fix some bugs in httpt --- AUTHORS | 4 + nettests/core/captiveportal.py | 19 +++ nettests/core/squid.py | 117 ++++++++++++++++++++ old-to-be-ported-code/AUTHORS | 3 - old-to-be-ported-code/bin/ooni-probe | 10 -- old-to-be-ported-code/ooni/output.py | 21 ---- .../ooni/plugins/captiveportal_plgoo.py | 55 --------- old-to-be-ported-code/ooni/plugins/skel_plgoo.py | 17 --- old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml | 33 ------ old-to-be-ported-code/ooni/yamlooni.py | 40 ------- ooni/nettest.py | 97 +++++++---------- ooni/reporter.py | 23 +++- ooni/runner.py | 11 ++- ooni/templates/httpt.py | 47 +++++++- ooni/utils/__init__.py | 32 ++++++ 15 files changed, 276 insertions(+), 253 deletions(-)

diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..d8dc0b8 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ +Jacob Appelbaum jacob@torproject.org +Arturo Filasto hellais@torproject.org +Linus Nordberg linus@torproject.org +Isis Lovecruft isis@torproject.org diff --git a/nettests/core/captiveportal.py b/nettests/core/captiveportal.py index 46f0856..77ba3e4 100644 --- a/nettests/core/captiveportal.py +++ b/nettests/core/captiveportal.py @@ -7,6 +7,25 @@ captive portal. Code is taken, in part, from the old ooni-probe, which was written by Jacob Appelbaum and Arturo Filastò.

+ This module performs multiple tests that match specific vendor captive + portal tests. This is a basic internet captive portal filter tester written + for RECon 2011. + + Read the following URLs to understand the captive portal detection process + for various vendors: + + http://technet.microsoft.com/en-us/library/cc766017%28WS.10%29.aspx + http://blog.superuser.com/2011/05/16/windows-7-network-awareness/ + http://isc.sans.org/diary.html?storyid=10312& + http://src.chromium.org/viewvc/chrome?view=rev&revision=74608 + http://code.google.com/p/chromium-os/issues/detail?3281ttp, + http://crbug.com/52489 + http://crbug.com/71736 + https://bugzilla.mozilla.org/show_bug.cgi?id=562917 + https://bugzilla.mozilla.org/show_bug.cgi?id=603505 + http://lists.w3.org/Archives/Public/ietf-http-wg/2011JanMar/0086.html + http://tools.ietf.org/html/draft-nottingham-http-portal-02 + :copyright: (c) 2012 Isis Lovecruft :license: see LICENSE for more details """ diff --git a/nettests/core/squid.py b/nettests/core/squid.py new file mode 100644 index 0000000..675119c --- /dev/null +++ b/nettests/core/squid.py @@ -0,0 +1,117 @@ +# -*- encoding: utf-8 -*- +# +# Squid transparent HTTP proxy detector +# ************************************* +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni import utils +from ooni.utils import log +from ooni.templates import httpt + +class SquidTest(httpt.HTTPTest): + """ + This test aims at detecting the presence of a squid based transparent HTTP + proxy. It also tries to detect the version number. + """ + name = "Squid test" + author = "Arturo Filastò" + version = 0.1 + + optParameters = [['backend', 'b', 'http://ooni.nu/test/', 'Test backend to use']] + + #inputFile = ['urls', 'f', None, 'Urls file'] + inputs =['http://google.com'] + def test_cacheobject(self): + """ + This detects the presence of a squid transparent HTTP proxy by sending + a request for cache_object://localhost/info. + + The response to this request will usually also contain the squid + version number. + """ + log.debug("Running") + def process_body(body): + if "Access Denied." in body: + self.report['transparent_http_proxy'] = True + else: + self.report['transparent_http_proxy'] = False + + log.msg("Testing Squid proxy presence by sending a request for "\ + "cache_object") + headers = {} + #headers["Host"] = [self.input] + self.report['trans_http_proxy'] = None + method = "GET" + body = "cache_object://localhost/info" + return self.doRequest(self.localOptions['backend'], method=method, body=body, + headers=headers, body_processor=process_body) + + def test_search_bad_request(self): + """ + Attempts to perform a request with a random invalid HTTP method. + + If we are being MITMed by a Transparent Squid HTTP proxy we will get + back a response containing the X-Squid-Error header. + """ + def process_headers(headers): + log.debug("Processing headers in test_search_bad_request") + if 'X-Squid-Error' in headers: + log.msg("Detected the presence of a transparent HTTP "\ + "squid proxy") + self.report['trans_http_proxy'] = True + else: + log.msg("Did not detect the presence of transparent HTTP "\ + "squid proxy") + self.report['transparent_http_proxy'] = False + + log.msg("Testing Squid proxy presence by sending a random bad request") + headers = {} + #headers["Host"] = [self.input] + method = utils.randomSTR(10, True) + self.report['transparent_http_proxy'] = None + return self.doRequest(self.localOptions['backend'], method=method, + headers=headers, headers_processor=process_headers) + + def test_squid_headers(self): + """ + Detects the presence of a squid transparent HTTP proxy based on the + response headers it adds to the responses to requests. + """ + def process_headers(headers): + """ + Checks if any of the headers that squid is known to add match the + squid regexp. + + We are looking for something that looks like this: + + via: 1.0 cache_server:3128 (squid/2.6.STABLE21) + x-cache: MISS from cache_server + x-cache-lookup: MISS from cache_server:3128 + """ + squid_headers = {'via': r'.* ((squid.*))', + 'x-cache': r'MISS from (\w+)', + 'x-cache-lookup': r'MISS from (\w+:?\d+?)' + } + + self.report['transparent_http_proxy'] = False + for key in squid_headers.keys(): + if key in headers: + log.debug("Found %s in headers" % key) + m = re.search(squid_headers[key], headers[key]) + if m: + log.msg("Detected the presence of squid transparent"\ + " HTTP Proxy") + self.report['transparent_http_proxy'] = True + + log.msg("Testing Squid proxy by looking at response headers") + headers = {} + #headers["Host"] = [self.input] + method = "GET" + self.report['transparent_http_proxy'] = None + d = self.doRequest(self.localOptions['backend'], method=method, + headers=headers, headers_processor=process_headers) + return d + + diff --git a/old-to-be-ported-code/AUTHORS b/old-to-be-ported-code/AUTHORS deleted file mode 100644 index c6a4ab6..0000000 --- a/old-to-be-ported-code/AUTHORS +++ /dev/null @@ -1,3 +0,0 @@ -Jacob Appelbaum jacob@torproject.org -Arturo Filasto hellais@torproject.org -Linus Nordberg linus@torproject.org diff --git a/old-to-be-ported-code/bin/ooni-probe b/old-to-be-ported-code/bin/ooni-probe deleted file mode 100644 index 9f616bc..0000000 --- a/old-to-be-ported-code/bin/ooni-probe +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -"""\ - This is the example OONI probe command line utility -""" - -import sys - -from ooni.command import Command - -Command(sys.argv[1:]).run() diff --git a/old-to-be-ported-code/ooni/output.py b/old-to-be-ported-code/ooni/output.py deleted file mode 100644 index 48e9f1f..0000000 --- a/old-to-be-ported-code/ooni/output.py +++ /dev/null @@ -1,21 +0,0 @@ -import yaml - -class data: - def __init__(self, name=None): - if name: - self.name = name - - def output(self, data, name=None): - if name: - self.name = name - - stream = open(self.name, 'w') - yaml.dump(data, stream) - stream.close() - def append(self, data, name=None): - if name: - self.name = name - stream = open(self.name, 'a') - yaml.dump([data], stream) - stream.close() - diff --git a/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py b/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py deleted file mode 100644 index 9c0d87c..0000000 --- a/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -# Captive Portal Detection With Multi-Vendor Emulation -# by Jacob Appelbaum jacob@appelbaum.net -# -# This module performs multiple tests that match specific vendor captive -# portal tests. This is a basic internet captive portal filter tester written -# for RECon 2011. -# -# Read the following URLs to understand the captive portal detection process -# for various vendors: -# -# http://technet.microsoft.com/en-us/library/cc766017%28WS.10%29.aspx -# http://blog.superuser.com/2011/05/16/windows-7-network-awareness/ -# http://isc.sans.org/diary.html?storyid=10312& -# http://src.chromium.org/viewvc/chrome?view=rev&revision=74608 -# http://code.google.com/p/chromium-os/issues/detail?id=3281 -# http://crbug.com/52489 -# http://crbug.com/71736 -# https://bugzilla.mozilla.org/show_bug.cgi?id=562917 -# https://bugzilla.mozilla.org/show_bug.cgi?id=603505 -# http://lists.w3.org/Archives/Public/ietf-http-wg/2011JanMar/0086.html -# http://tools.ietf.org/html/draft-nottingham-http-portal-02 -# - -import sys -import ooni.http -import ooni.dnsooni -import ooni.report - -from ooni.plugooni import Plugoo - -class CaptivePortalPlugin(Plugoo): - def __init__(self): - self.in_ = sys.stdin - self.out = sys.stdout - self.debug = False - self.logger = ooni.report.Log().logger - self.name = "" - self.type = "" - self.paranoia = "" - self.modules_to_import = [] - self.output_dir = "" - self.default_args = "" - - def CaptivePortal_Tests(self): - print "Captive Portal Detection With Multi-Vendor Emulation:" - tests = self.get_tests_by_filter(("_CP_Tests"), (ooni.http, ooni.dnsooni)) - self.run_tests(tests) - - def magic_main(self): - self.run_plgoo_tests("_Tests") - - def ooni_main(self,args): - self.magic_main() diff --git a/old-to-be-ported-code/ooni/plugins/skel_plgoo.py b/old-to-be-ported-code/ooni/plugins/skel_plgoo.py deleted file mode 100644 index f365c06..0000000 --- a/old-to-be-ported-code/ooni/plugins/skel_plgoo.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/python -# This will never load it is just an example of Plugooni plgoo plugins -# -from ooni.plugooni import Plugoo - -class SkelPlugin(Plugoo): - def __init__(self): - self.name = "" - self.type = "" - self.paranoia = "" - self.modules_to_import = [] - self.output_dir = "" - - def ooni_main(self, cmd): - print "This is the main plugin function" - - diff --git a/old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml b/old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml deleted file mode 100644 index 6a91e8a..0000000 --- a/old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- -plugin: - name : Skel - author : Some Name - date_created : 2011-08-01 - modules : [tcp, udp, http] -input: - experiment: - list : ['el1', - 'el2', - 'el3', - 'el4', - 'el5'] - control: - list : ['el1', - 'el2', - 'el3', - 'el4', - 'el5'] -output: - timestamp : - experiment : - timestamp : - test : - result : - extrafield : - control : - timestamp : - test : - result : - extrafield : - - diff --git a/old-to-be-ported-code/ooni/yamlooni.py b/old-to-be-ported-code/ooni/yamlooni.py deleted file mode 100644 index a457217..0000000 --- a/old-to-be-ported-code/ooni/yamlooni.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# -# Plugooni, ooni plugin module for loading plgoo files. -# by Jacob Appelbaum jacob@appelbaum.net -# Arturo Filasto' art@fuffa.org - -import sys -import os - -class Yamlooni(): - def __init__(self, name, creator, location): - self.name = name - self.creator = creator - self.location = location - f = open(self.location) - self.ydata = yaml.load(f.read()) - - def debug_print(): - #print y.input - for i in y.iteritems(): - if i[0] == "input": - print "This is the input part:" - for j in i[1].iteritems(): - print j - print "end of the input part.\n" - - elif i[0] == "output": - print "This is the output part:" - for j in i[1].iteritems(): - print j - print "end of the output part.\n" - - elif i[0] == "plugin": - print "This is the Plugin part:" - for j in i[1].iteritems(): - print j - print "end of the plugin part.\n" - - - diff --git a/ooni/nettest.py b/ooni/nettest.py index 3c87b7f..b17117d 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -3,19 +3,9 @@ import itertools import os

-from inspect import classify_class_attrs -from pprint import pprint - -from twisted.internet import defer, utils -from twisted.python import usage -from twisted.trial import unittest, itrial -from zope.interface.exceptions import BrokenImplementation - -from ooni.inputunit import InputUnitProcessor -from ooni.utils import log -from ooni.utils.assertions import isClass, isNotClass -from ooni.utils.assertions import isOldStyleClass, isNewStyleClass - +from twisted.trial import unittest, itrial, util +from twisted.internet import defer, utils +from ooni.utils import log

pyunit = __import__('unittest')

@@ -25,20 +15,46 @@ class InputTestSuite(pyunit.TestSuite): and the tracking of current index via idx. """ def run(self, result, idx=0): + log.debug("Running test suite") self._idx = idx while self._tests: if result.shouldStop: + log.debug("Detected that test should stop") + log.debug("Stopping...") break test = self._tests.pop(0) + try: + log.debug("Setting test attributes with %s %s" % + (self.input, self._idx)) + test.input = self.input test._idx = self._idx + except Exception, e: + log.debug("Error in some stuff") + log.debug(e) + import sys + print sys.exc_info() + + try: + log.debug("Running test") test(result) - except: + log.debug("Ran.") + except Exception, e: + log.debug("Attribute error thing") + log.debug("Had some problems with _idx") + log.debug(e) + import traceback, sys + print sys.exc_info() + traceback.print_exc() + print e + test(result) + self._idx += 1 return result

+ class TestCase(unittest.TestCase): """ This is the monad of the OONI nettest universe. When you write a nettest @@ -123,60 +139,23 @@ class TestCase(unittest.TestCase): writing. """ if result.reporterFactory.firstrun: + log.debug("Detecting first run. Writing report header.") d1 = result.reporterFactory.writeHeader() d2 = unittest.TestCase.deferSetUp(self, ignored, result) dl = defer.DeferredList([d1, d2]) return dl else: + log.debug("Not first run. Running test setup directly") return unittest.TestCase.deferSetUp(self, ignored, result)

- def _raaun(self, methodName, result): - from twisted.internet import reactor - method = getattr(self, methodName) - log.debug("Running %s" % methodName) - d = defer.maybeDeferred( - utils.runWithWarningsSuppressed, self._getSuppress(), method) - d.addBoth(lambda x : call.active() and call.cancel() or x) - return d - - @staticmethod - def inputParser(inputs): - """Replace me with a custom function for parsing inputs.""" - return inputs - - def __input_file_processor__(self, fp): - """ - I open :attr:inputFile if there is one, and return inputs one by one - after stripping them of whitespace and running them through the parser - :meth:`inputParser`. - """ - for line in fp.readlines(): - yield self.inputParser(line.strip()) + def inputProcessor(self, fp): + log.debug("Running default input processor") + for x in fp.readlines(): + yield x.strip() fp.close()

- def __get_inputs__(self): - """ - I am called from the ooni.runner and you probably should not override - me. I gather the internal inputs from an instantiated test class and - pass them to the rest of the runner. - - If you are looking for a way to parse inputs from inputFile, see - :meth:`inputParser`. - """ - processor = InputUnitProcessor(self.inputs, - input_filter=None, - catch_err=False) - processed = processor.process() - - log.msg("Received direct inputs:\n%s" % inputs) - log.debug("Our InputUnitProcessor is %s" % processor) - - #while processed is not StopIteration: - # self.inputs = processed - # yield self.inputs - #else: - # if self.inputFile: - + def getOptions(self): + log.debug("Getting options for test") if self.inputFile: try: fp = open(self.inputFile) ## xxx fixme: diff --git a/ooni/reporter.py b/ooni/reporter.py index a7b645b..c12b28f 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -4,6 +4,7 @@ import logging import sys import time import yaml +import traceback

from yaml.representer import * from yaml.emitter import * @@ -132,8 +133,7 @@ class ReporterFactory(OReporter): client_geodata = {} log.msg("Running geo IP lookup via check.torproject.org")

- #client_ip = yield geodata.myIP() - client_ip = '127.0.0.1' + client_ip = yield geodata.myIP() try: import txtorcon client_location = txtorcon.util.NetLocation(client_ip) @@ -200,6 +200,7 @@ class OONIReporter(OReporter): if not self._startTime: self._startTime = self._getTime()

+ log.debug("Starting test %s" % idx) test.report = {}

self._tests[idx] = {} @@ -215,6 +216,7 @@ class OONIReporter(OReporter):

def stopTest(self, test): + log.debug("Stopping test") super(OONIReporter, self).stopTest(test)

idx = self.getTestIndex(test) @@ -224,11 +226,14 @@ class OONIReporter(OReporter): # XXX In the future this should be removed. try: report = list(test.legacy_report) + log.debug("Set the report to be a list") except: # XXX I put a dict() here so that the object is re-instantiated and I # actually end up with the report I want. This could either be a # python bug or a yaml bug. report = dict(test.report) + log.debug("Set the report to be a dict") + log.debug("Adding to report %s" % report) self._tests[idx]['report'] = report

@@ -245,6 +250,7 @@ class OONIReporter(OReporter): Expects that L{_printErrors}, L{_writeln}, L{_write}, L{_printSummary} and L{_separator} are all implemented. """ + log.debug("Test run concluded") if self._publisher is not None: self._publisher.removeObserver(self._observeWarnings) if self._startTime is not None: @@ -261,13 +267,18 @@ class OONIReporter(OReporter): super(OONIReporter, self).addSuccess(test) #self.report['result'] = {'value': 'success'}

- def addError(self, *args): - super(OONIReporter, self).addError(*args) - #self.report['result'] = {'value': 'error', 'args': args} + def addError(self, test, exception): + super(OONIReporter, self).addError(test, exception) + exc_type, exc_value, exc_traceback = exception + log.err(exc_type) + log.err(str(exc_value)) + # XXX properly print out the traceback + for line in '\n'.join(traceback.format_tb(exc_traceback)).split("\n"): + log.err(line)

def addFailure(self, *args): super(OONIReporter, self).addFailure(*args) - #self.report['result'] = {'value': 'failure', 'args': args} + log.warn(args)

def addSkip(self, *args): super(OONIReporter, self).addSkip(*args) diff --git a/ooni/runner.py b/ooni/runner.py index b1a21ac..f352dfb 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -235,13 +235,18 @@ class ORunner(object): def runWithInputUnit(self, inputUnit): idx = 0 result = self.reporterFactory.create() - + log.debug("Running test with input unit %s" % inputUnit) for inputs in inputUnit: result.reporterFactory = self.reporterFactory

+ log.debug("Running with %s" % inputs) suite = self.baseSuite(self.cases) suite.input = inputs - suite(result, idx) + try: + suite(result, idx) + except Exception, e: + log.err("Error in running test!") + log.err(e)

# XXX refactor all of this index bullshit to avoid having to pass # this index around. Probably what I want to do is go and make @@ -250,7 +255,9 @@ class ORunner(object): # We currently need to do this addition in order to get the number # of times the test cases that have run inside of the test suite. idx += (suite._idx - idx) + log.debug("I am now at the index %s" % idx)

+ log.debug("Finished") result.done()

def run(self): diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 6e3163b..f453c74 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -54,6 +54,7 @@ class HTTPTest(TestCase): followRedirects = False

def setUp(self): + log.debug("Setting up HTTPTest") try: import OpenSSL except: @@ -76,12 +77,17 @@ class HTTPTest(TestCase):

self.request = {} self.response = {} + log.debug("Finished test setup")

- def _processResponseBody(self, data): + def _processResponseBody(self, data, body_processor): + log.debug("Processing response body") self.response['body'] = data self.report['response'] = self.response

- self.processResponseBody(data) + if body_processor: + body_processor(data) + else: + self.processResponseBody(data)

def processResponseBody(self, data): """ @@ -108,7 +114,25 @@ class HTTPTest(TestCase): """ pass

- def doRequest(self, url, method="GET", headers=None, body=None): + def doRequest(self, url, method="GET", + headers=None, body=None, headers_processor=None, + body_processor=None): + """ + Perform an HTTP request with the specified method. + + url: the full url path of the request + method: the HTTP Method to be used + headers: the request headers to be sent + body: the request body + headers_processor: a function to be used for processing the HTTP header + responses (defaults to self.processResponseHeaders). + This function takes as argument the HTTP headers as a + dict. + body_processory: a function to be used for processing the HTTP response + body (defaults to self.processResponseBody). + This function takes the response body as an argument. + """ + log.debug("Performing request %s %s %s" % (url, method, headers)) try: d = self.build_request(url, method, headers, body) except Exception, e: @@ -123,11 +147,17 @@ class HTTPTest(TestCase): return

d.addErrback(errback) - d.addCallback(self._cbResponse) + d.addCallback(self._cbResponse, headers_processor, body_processor) d.addCallback(finished) return d

- def _cbResponse(self, response): + def _cbResponse(self, response, headers_processor, body_processor): + log.debug("Got response %s" % response) + if not response: + self.report['response'] = None + log.err("We got an empty response") + return + self.response['headers'] = list(response.headers.getAllRawHeaders()) self.response['code'] = response.code self.response['length'] = response.length @@ -136,11 +166,14 @@ class HTTPTest(TestCase): if str(self.response['code']).startswith('3'): self.processRedirect(response.headers.getRawHeaders('Location')[0])

- self.processResponseHeaders(self.response['headers']) + if headers_processor: + headers_processor(self.response['headers']) + else: + self.processResponseHeaders(self.response['headers'])

finished = defer.Deferred() response.deliverBody(BodyReceiver(finished)) - finished.addCallback(self._processResponseBody) + finished.addCallback(self._processResponseBody, body_processor)

return finished

diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 38239ba..cd82ab4 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -4,6 +4,9 @@

import imp import logging +import string +import random + try: import yaml except: @@ -143,3 +146,32 @@ class Log(): except: raise StopIteration

+def randomSTR(length, num=True): + """ + Returns a random all uppercase alfa-numerical (if num True) string long length + """ + chars = string.ascii_uppercase + if num: + chars += string.digits + return ''.join(random.choice(chars) for x in range(length)) + +def randomstr(length, num=True): + """ + Returns a random all lowercase alfa-numerical (if num True) string long length + """ + chars = string.ascii_lowercase + if num: + chars += string.digits + return ''.join(random.choice(chars) for x in range(length)) + +def randomStr(length, num=True): + """ + Returns a random a mixed lowercase, uppercase, alfanumerical (if num True) + string long length + """ + chars = string.ascii_lowercase + string.ascii_uppercase + if num: + chars += string.digits + return ''.join(random.choice(chars) for x in range(length)) + +