commit f4ac83c43dd27ffb05686e311ded33b850e62acf Author: Arturo Filastò arturo@filasto.net Date: Wed Oct 3 12:25:02 2012 +0000

Do some fixes to the reporter * Remove useless print lines * Re-add hacks.py --- nettests/http_host.py | 47 ++++++++++++++++++++++++++++++++++++++ nettests/myip.py | 1 - ooni/reporter.py | 12 +++++++-- ooni/templates/httpt.py | 5 ---- ooni/utils/hacks.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 114 insertions(+), 9 deletions(-)

diff --git a/nettests/http_host.py b/nettests/http_host.py new file mode 100644 index 0000000..4834ba3 --- /dev/null +++ b/nettests/http_host.py @@ -0,0 +1,47 @@ +# -*- encoding: utf-8 -*- +# +# HTTP Host Test +# ************** +# +# for more details see: +# https://trac.torproject.org/projects/tor/wiki/doc/OONI/Tests/HTTPHost +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni.templates import httpt + +good_http_server = "http://127.0.0.1:8090/" + +class HTTPHost(httpt.HTTPTest): + name = "HTTP Host" + author = "Arturo Filastò" + version = 0.1 + + + inputs = ['google.com', 'wikileaks.org', + 'torproject.org'] + + def test_send_host_header(self): + headers = {} + headers["Host"] = [self.input] + return self.doRequest(good_http_server, headers=headers) + + def processResponseBody(self, body): + # XXX here shall go your logic + # for processing the body + if 'blocked' in body: + self.report['censored'] = True + else: + self.report['censored'] = False + + def processResponseHeaders(self, headers): + # XXX place in here all the logic for handling the processing of HTTP + # Headers. + if headers.hasHeader('location'): + self.report['redirect'] = True + + server = headers.getRawHeaders("Server") + if server: + self.report['http_server'] = str(server.pop()) + diff --git a/nettests/myip.py b/nettests/myip.py index e7e96f2..40a4849 100644 --- a/nettests/myip.py +++ b/nettests/myip.py @@ -7,7 +7,6 @@ from ooni.templates import httpt class MyIP(httpt.HTTPTest): inputs = ['https://check.torproject.org'] def processResponseBody(self, body): - print "FOOOO" import re regexp = "Your IP address appears to be: <b>(.+?)</b>" match = re.search(regexp, body) diff --git a/ooni/reporter.py b/ooni/reporter.py index 3e108f9..ee8427d 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -71,14 +71,20 @@ class ReporterFactory(OReporter): self._testSuite = testSuite self._reporters = []

- def writeHeader(self, options): + def writeHeader(self, options, geodata={}): self._writeln("###########################################") self._writeln("# OONI Probe Report for %s test" % options['name']) self._writeln("# %s" % date.pretty_date()) self._writeln("###########################################")

- address = {'asn': 'XXX replace me with ASN', - 'ip': 'XXX replace me with IP'} + address = {'asn': 'unknown', + 'ip': 'unknown'} + if 'ip' in geodata: + address['ip'] = geodata['ip'] + + if 'asn' in geodata: + address['asn'] = geodata['asn'] + test_details = {'startTime': repr(date.now()), 'probeASN': address['asn'], 'testName': options['name'], diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index a537c6e..4b8fa9b 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -28,11 +28,6 @@ useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Geck ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")]

- -class WebClientContextFactory(ClientContextFactory): - def getContext(self, hostname, port): - return ClientContextFactory.getContext(self) - class BodyReceiver(protocol.Protocol): def __init__(self, finished): self.finished = finished diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py new file mode 100644 index 0000000..674f45e --- /dev/null +++ b/ooni/utils/hacks.py @@ -0,0 +1,58 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import copy_reg + +def patched_reduce_ex(self, proto): + """ + This is a hack to overcome a bug in one of pythons core functions. It is + located inside of copy_reg and is called _reduce_ex. + + Some background on the issue can be found here: + http://stackoverflow.com/questions/569754/how-to-tell-for-which-object-attri... + http://stackoverflow.com/questions/2049849/why-cant-i-pickle-this-object + + There was also an open bug on the pyyaml trac repo, but it got closed because + they could not reproduce. + http://pyyaml.org/ticket/190 + + It turned out to be easier to patch the python core library than to monkey + patch yaml. + + XXX see if there is a better way. sigh... + """ + _HEAPTYPE = 1<<9 + assert proto < 2 + for base in self.__class__.__mro__: + if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: + break + else: + base = object # not really reachable + if base is object: + state = None + elif base is int: + state = None + else: + if base is self.__class__: + raise TypeError, "can't pickle %s objects" % base.__name__ + state = base(self) + args = (self.__class__, base, state) + try: + getstate = self.__getstate__ + except AttributeError: + if getattr(self, "__slots__", None): + raise TypeError("a class that defines __slots__ without " + "defining __getstate__ cannot be pickled") + try: + dict = self.__dict__ + except AttributeError: + dict = None + else: + dict = getstate() + if dict: + return copy_reg._reconstructor, args, dict + else: + return copy_reg._reconstructor, args +