- tor-commits - lists.torproject.org

[ooni-probe/master] Make nettest and runner support test relative options.
by art＠torproject.org 06 Oct '12

06 Oct '12

commit c99b1657c35a2c83126001282fe01ecaee57b66f Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Oct 6 10:21:20 2012 +0000 Make nettest and runner support test relative options. --- nettests/httphost.py | 124 ++++++++++++++++++++++++++++++++++++++++++++++ ooni/nettest.py | 2 + ooni/plugins/httphost.py | 124 ---------------------------------------------- ooni/runner.py | 17 +++++- 4 files changed, 140 insertions(+), 127 deletions(-) diff --git a/nettests/httphost.py b/nettests/httphost.py new file mode 100644 index 0000000..b57a50d --- /dev/null +++ b/nettests/httphost.py @@ -0,0 +1,124 @@ +""" + HTTP Host based filtering + ************************* + + This test detect HTTP Host field + based filtering. +""" +import os +from datetime import datetime + +import urllib2 +import httplib +try: + from BeautifulSoup import BeautifulSoup +except: + print "Beautiful Soup not installed. HTTPHost may not work" + +# XXX reduce boilerplating. +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, OONITest + +class HTTPHostArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['controlserver', 'c', 'google.com', 'Specify the control server'], + ['resume', 'r', 0, 'Resume at this index'], + ['other', 'o', None, 'Other arguments']] + def control(self, experiment_result, args): + print "Experiment Result:", experiment_result + print "Args", args + return experiment_result + + def experiment(self, args): + import urllib + req = urllib.urlopen(args['asset']) + return {'page': req.readlines()} + +class HTTPHostAsset(Asset): + """ + This is the asset that should be used by the Test. It will + contain all the code responsible for parsing the asset file + and should be passed on instantiation to the test. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + return line.split(',')[1].replace('\n','') + + +class HTTPHostTest(OONITest): + implements(IPlugin, ITest) + + shortName = "httphost" + description = "HTTP Host plugin" + requirements = None + options = HTTPHostArgs + # Tells this to be blocking. + blocking = True + + def check_response(self, response): + soup = BeautifulSoup(response) + if soup.head.title.string == "Blocked": + # Response indicates censorship + return True + else: + # Response does not indicate censorship + return False + + def is_censored(self, response): + if response: + soup = BeautifulSoup(response) + censored = self.check_response(response) + else: + censored = "unreachable" + return censored + + def urllib2_test(self, control_server, host): + req = urllib2.Request(control_server) + req.add_header('Host', host) + try: + r = urllib2.urlopen(req) + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + def httplib_test(self, control_server, host): + censored = None + response = None + try: + conn = httplib.HTTPConnection(control_server) + conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) + conn.putheader("Host", host) + conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") + conn.endheaders() + r = conn.getresponse() + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return (censored, response) + + def experiment(self, args): + control_server = self.local_options['controlserver'] + url = 'http://torproject.org/' if not 'asset' in args else args['asset'] + censored = self.httplib_test(control_server, url) + return {'control': control_server, + 'host': url, + 'censored': censored} + + def load_assets(self): + if self.local_options and self.local_options['asset']: + return {'asset': Asset(self.local_options['asset'])} + else: + return {} + +httphost = HTTPHostTest(None, None, None) diff --git a/ooni/nettest.py b/ooni/nettest.py index d88c044..a01050e 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -80,6 +80,8 @@ class TestCase(unittest.TestCase): report = {} report['errors'] = [] + optParameters = None + def getOptions(self): if self.inputFile: fp = open(self.inputFile) diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py deleted file mode 100644 index 7c783a1..0000000 --- a/ooni/plugins/httphost.py +++ /dev/null @@ -1,124 +0,0 @@ -""" - HTTP Host based filtering - ************************* - - This test detect HTTP Host field - based filtering. -""" -import os -from datetime import datetime - -import urllib2 -import httplib -try: - from BeautifulSoup import BeautifulSoup -except: - print "Beautiful Soup not installed. HTTPHost may not work" - -# XXX reduce boilerplating. -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin - -from plugoo.assets import Asset -from plugoo.tests import ITest, OONITest - -class HTTPHostArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['controlserver', 'c', 'google.com', 'Specify the control server'], - ['resume', 'r', 0, 'Resume at this index'], - ['other', 'o', None, 'Other arguments']] - def control(self, experiment_result, args): - print "Experiment Result:", experiment_result - print "Args", args - return experiment_result - - def experiment(self, args): - import urllib - req = urllib.urlopen(args['asset']) - return {'page': req.readlines()} - -class HTTPHostAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - return line.split(',')[1].replace('\n','') - - -class HTTPHostTest(OONITest): - implements(IPlugin, ITest) - - shortName = "httphost" - description = "HTTP Host plugin" - requirements = None - options = HTTPHostArgs - # Tells this to be blocking. - blocking = True - - def check_response(self, response): - soup = BeautifulSoup(response) - if soup.head.title.string == "Blocked": - # Response indicates censorship - return True - else: - # Response does not indicate censorship - return False - - def is_censored(self, response): - if response: - soup = BeautifulSoup(response) - censored = self.check_response(response) - else: - censored = "unreachable" - return censored - - def urllib2_test(self, control_server, host): - req = urllib2.Request(control_server) - req.add_header('Host', host) - try: - r = urllib2.urlopen(req) - response = r.read() - censored = self.is_censored(response) - except Exception, e: - censored = "Error! %s" % e - - return censored - - def httplib_test(self, control_server, host): - censored = None - response = None - try: - conn = httplib.HTTPConnection(control_server) - conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) - conn.putheader("Host", host) - conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") - conn.endheaders() - r = conn.getresponse() - response = r.read() - censored = self.is_censored(response) - except Exception, e: - censored = "Error! %s" % e - - return (censored, response) - - def experiment(self, args): - control_server = self.local_options['controlserver'] - url = 'http://torproject.org/' if not 'asset' in args else args['asset'] - censored = self.httplib_test(control_server, url) - return {'control': control_server, - 'host': url, - 'censored': censored} - - def load_assets(self): - if self.local_options and self.local_options['asset']: - return {'asset': Asset(self.local_options['asset'])} - else: - return {} - -httphost = HTTPHostTest(None, None, None) diff --git a/ooni/runner.py b/ooni/runner.py index 8ed844d..c6958bc 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -5,7 +5,7 @@ import time import inspect from twisted.internet import defer, reactor -from twisted.python import reflect, failure +from twisted.python import reflect, failure, usage from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader @@ -91,6 +91,16 @@ def adaptLegacyTest(obj, config): return LegacyOONITest +def processTest(obj, config): + if obj.optParameters: + class Options(usage.Options): + optParameters = obj.optParameters + + options = Options() + options.parseOptions(config['subArgs']) + + obj.localOptions = options + return obj def findTestClassesFromConfig(config): """ @@ -106,7 +116,7 @@ def findTestClassesFromConfig(config): module = filenameToModule(filename) for name, val in inspect.getmembers(module): if isTestCase(val): - classes.append(val) + classes.append(processTest(val, config)) elif isLegacyTest(val): classes.append(adaptLegacyTest(val, config)) return classes @@ -135,7 +145,8 @@ def loadTestsAndOptions(classes): for klass in classes: try: k = klass() - options.append(k.getOptions()) + opts = k.getOptions() + options.append(opts) except AttributeError: options.append([])

1 0

[ooni-probe/master] Implement input File support for new API.
by art＠torproject.org 06 Oct '12

06 Oct '12

commit 9c7f354bcb842be19d687305c8a2e4e57cad6276 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Oct 6 10:47:03 2012 +0000 Implement input File support for new API. * Port HTTP Host test to new API --- nettests/http_host.py | 28 +++-------- nettests/httphost.py | 124 ------------------------------------------------- ooni/nettest.py | 14 +++--- ooni/runner.py | 5 ++ 4 files changed, 20 insertions(+), 151 deletions(-) diff --git a/nettests/http_host.py b/nettests/http_host.py index 4834ba3..2228c52 100644 --- a/nettests/http_host.py +++ b/nettests/http_host.py @@ -11,37 +11,25 @@ from ooni.templates import httpt -good_http_server = "http://127.0.0.1:8090/" - class HTTPHost(httpt.HTTPTest): name = "HTTP Host" author = "Arturo Filastò" version = 0.1 + optParameters = [['url', 'u', 'http://torproject.org/', 'Test single site'], + ['backend', 'b', 'http://ooni.nu/test/', 'Test backend to use'], + ] - inputs = ['google.com', 'wikileaks.org', - 'torproject.org'] + inputFile = ['urls', 'f', None, 'Urls file'] def test_send_host_header(self): headers = {} headers["Host"] = [self.input] - return self.doRequest(good_http_server, headers=headers) + return self.doRequest(self.localOptions['backend'], headers=headers) def processResponseBody(self, body): - # XXX here shall go your logic - # for processing the body - if 'blocked' in body: - self.report['censored'] = True + if 'not censored' in body: + self.report['trans_http_proxy'] = False else: - self.report['censored'] = False - - def processResponseHeaders(self, headers): - # XXX place in here all the logic for handling the processing of HTTP - # Headers. - if headers.hasHeader('location'): - self.report['redirect'] = True - - server = headers.getRawHeaders("Server") - if server: - self.report['http_server'] = str(server.pop()) + self.report['trans_http_proxy'] = True diff --git a/nettests/httphost.py b/nettests/httphost.py deleted file mode 100644 index b57a50d..0000000 --- a/nettests/httphost.py +++ /dev/null @@ -1,124 +0,0 @@ -""" - HTTP Host based filtering - ************************* - - This test detect HTTP Host field - based filtering. -""" -import os -from datetime import datetime - -import urllib2 -import httplib -try: - from BeautifulSoup import BeautifulSoup -except: - print "Beautiful Soup not installed. HTTPHost may not work" - -# XXX reduce boilerplating. -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin - -from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, OONITest - -class HTTPHostArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['controlserver', 'c', 'google.com', 'Specify the control server'], - ['resume', 'r', 0, 'Resume at this index'], - ['other', 'o', None, 'Other arguments']] - def control(self, experiment_result, args): - print "Experiment Result:", experiment_result - print "Args", args - return experiment_result - - def experiment(self, args): - import urllib - req = urllib.urlopen(args['asset']) - return {'page': req.readlines()} - -class HTTPHostAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - return line.split(',')[1].replace('\n','') - - -class HTTPHostTest(OONITest): - implements(IPlugin, ITest) - - shortName = "httphost" - description = "HTTP Host plugin" - requirements = None - options = HTTPHostArgs - # Tells this to be blocking. - blocking = True - - def check_response(self, response): - soup = BeautifulSoup(response) - if soup.head.title.string == "Blocked": - # Response indicates censorship - return True - else: - # Response does not indicate censorship - return False - - def is_censored(self, response): - if response: - soup = BeautifulSoup(response) - censored = self.check_response(response) - else: - censored = "unreachable" - return censored - - def urllib2_test(self, control_server, host): - req = urllib2.Request(control_server) - req.add_header('Host', host) - try: - r = urllib2.urlopen(req) - response = r.read() - censored = self.is_censored(response) - except Exception, e: - censored = "Error! %s" % e - - return censored - - def httplib_test(self, control_server, host): - censored = None - response = None - try: - conn = httplib.HTTPConnection(control_server) - conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) - conn.putheader("Host", host) - conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") - conn.endheaders() - r = conn.getresponse() - response = r.read() - censored = self.is_censored(response) - except Exception, e: - censored = "Error! %s" % e - - return (censored, response) - - def experiment(self, args): - control_server = self.local_options['controlserver'] - url = 'http://torproject.org/' if not 'asset' in args else args['asset'] - censored = self.httplib_test(control_server, url) - return {'control': control_server, - 'host': url, - 'censored': censored} - - def load_assets(self): - if self.local_options and self.local_options['asset']: - return {'asset': Asset(self.local_options['asset'])} - else: - return {} - -httphost = HTTPHostTest(None, None, None) diff --git a/ooni/nettest.py b/ooni/nettest.py index a01050e..89dd279 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -24,11 +24,6 @@ class InputTestSuite(pyunit.TestSuite): self._idx += 1 return result -def lineByLine(fp): - for x in fp.readlines(): - yield x.strip() - fp.close() - class TestCase(unittest.TestCase): """ This is the monad of the OONI nettest universe. When you write a nettest @@ -75,17 +70,22 @@ class TestCase(unittest.TestCase): inputs = [None] inputFile = None - inputProcessor = lineByLine + report = {} report['errors'] = [] optParameters = None + def inputProcessor(self, fp): + for x in fp.readlines(): + yield x.strip() + fp.close() + def getOptions(self): if self.inputFile: fp = open(self.inputFile) - self.inputs = inputProcessor(fp) + self.inputs = self.inputProcessor(fp) # XXX perhaps we may want to name and version to be inside of a # different object that is not called options. return {'inputs': self.inputs, diff --git a/ooni/runner.py b/ooni/runner.py index c6958bc..a753d02 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -96,10 +96,15 @@ def processTest(obj, config): class Options(usage.Options): optParameters = obj.optParameters + inputFile = obj.inputFile + Options.optParameters.append(inputFile) + options = Options() options.parseOptions(config['subArgs']) obj.localOptions = options + obj.inputFile = options[inputFile[0]] + return obj def findTestClassesFromConfig(config):

1 0

[ooni-probe/master] Fail gracefully when twisted is <= 10.x and we attempt to import RedirectAgent
by art＠torproject.org 06 Oct '12

06 Oct '12

commit 302dd54259fc6e69d3f276bcb458350344552887 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Oct 6 09:49:01 2012 +0000 Fail gracefully when twisted is <= 10.x and we attempt to import RedirectAgent --- ooni/nettest.py | 3 +++ ooni/templates/httpt.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index d7188c6..d88c044 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -77,6 +77,9 @@ class TestCase(unittest.TestCase): inputFile = None inputProcessor = lineByLine + report = {} + report['errors'] = [] + def getOptions(self): if self.inputFile: fp = open(self.inputFile) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 4b8fa9b..963189a 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -65,8 +65,15 @@ class HTTPTest(TestCase): self.agent = Agent(reactor) if self.followRedirects: - from twisted.web.client import RedirectAgent - self.agent = RedirectAgent(self.agent) + try: + from twisted.web.client import RedirectAgent + self.agent = RedirectAgent(self.agent) + except: + log.err("Warning! You are running an old version of twisted"\ + "(<= 10.1). I will not be able to follow redirects."\ + "This may make the testing less precise.") + self.report['errors'].append("Could not import RedirectAgent") + self.request = {} self.response = {}

1 0

[ooni-probe/master] Port captive portal test to new API.
by art＠torproject.org 06 Oct '12

06 Oct '12

commit 5cdc37ff80f1c8343e26f94fd7b7a29b3081ed03 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Oct 6 09:37:11 2012 +0000 Port captive portal test to new API. --- nettests/captiveportal.py | 57 +++++++++++++++++++-------------------------- ooni/runner.py | 2 +- 2 files changed, 25 insertions(+), 34 deletions(-) diff --git a/nettests/captiveportal.py b/nettests/captiveportal.py index 4bf99b3..6eed04d 100644 --- a/nettests/captiveportal.py +++ b/nettests/captiveportal.py @@ -18,13 +18,8 @@ import string import urllib2 from urlparse import urlparse -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin - -from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, OONITest -from ooni.protocols import http +from ooni import nettest +from ooni.templates import httpt from ooni.utils import log try: @@ -36,28 +31,21 @@ except ImportError: __plugoo__ = "captiveportal" __desc__ = "Captive portal detection test" -class CaptivePortalArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['resume', 'r', 0, 'Resume at this index'], - ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'], - ['user-agent', 'u', random.choice(http.useragents), - 'User agent for HTTP requests'] - ] +optParameters = [['asset', 'a', None, 'Asset file'], + ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'], + ['user-agent', 'u', random.choice(httpt.useragents), + 'User agent for HTTP requests'] + ] -class CaptivePortal(OONITest): +class CaptivePortal(nettest.TestCase): """ Compares content and status codes of HTTP responses, and attempts to determine if content has been altered. """ - implements(IPlugin, ITest) - - shortName = "captivep" + name = "captivep" description = "Captive Portal Test" requirements = None - options = CaptivePortalArgs - # Tells this to be blocking. - blocking = True def http_fetch(self, url, headers={}): """ @@ -549,7 +537,9 @@ class CaptivePortal(OONITest): in the experimental content. Returns True if matches are found, and False if otherwise. """ - experiment_url = self.local_options['experiment-url'] + # XXX put this back to being parametrized + #experiment_url = self.local_options['experiment-url'] + experiment_url = 'http://google.com/' control_result = 'XX' control_code = 200 ua = self.local_options['user-agent'] @@ -565,7 +555,8 @@ class CaptivePortal(OONITest): if status_match and content_match: log.msg("The test for '%s'" % experiment_url) log.msg("was unable to detect a captive portal.") - return experiment_result, True + + self.report['result'] = True elif status_match and not content_match: log.msg("Retrying '%s' with fuzzy match enabled." @@ -574,19 +565,20 @@ class CaptivePortal(OONITest): control_result, fuzzy=True) if fuzzy_match: - return experiment_result, True + self.report['result'] = True else: log.msg("Found modified content on '%s'," % experiment_url) log.msg("which could indicate a captive portal.") - return experiment_result, False + self.report['result'] = False else: log.msg("The content comparison test for ") log.msg("'%s'" % experiment_url) log.msg("shows that your HTTP traffic is filtered.") - return experiment_result, False - def experiment(self, args): + self.report['result'] = False + + def test_captive_portal(self): """ Runs the CaptivePortal(Test). @@ -608,22 +600,21 @@ class CaptivePortal(OONITest): Any combination of the above tests can be run. """ - report = {} log.msg("") log.msg("Running vendor tests...") - report['vendor_tests'] = self.run_vendor_tests() + self.report['vendor_tests'] = self.run_vendor_tests() log.msg("") log.msg("Running vendor DNS-based tests...") - report['vendor_dns_tests'] = self.run_vendor_dns_tests() + self.report['vendor_dns_tests'] = self.run_vendor_dns_tests() log.msg("") log.msg("Checking that DNS requests are not being tampered...") - report['check0x20'] = self.check_0x20_to_auth_ns('ooni.nu') + self.report['check0x20'] = self.check_0x20_to_auth_ns('ooni.nu') log.msg("") log.msg("Captive portal test finished!") - return report -cp = CaptivePortal(None, None, None) + self.control(self.report) + diff --git a/ooni/runner.py b/ooni/runner.py index a4c88a9..8ed844d 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -6,7 +6,7 @@ import inspect from twisted.internet import defer, reactor from twisted.python import reflect, failure -#from twisted.python import log + from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader from twisted.trial.runner import isPackage, isTestCase, ErrorHolder

1 0

[ooni-probe/master] Move captive portal test to nettest directory for porting to new API
by art＠torproject.org 06 Oct '12

06 Oct '12

commit dea9dae2d32f4ce1cd90df8a4cbf66457763dfaf Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Oct 6 09:05:23 2012 +0000 Move captive portal test to nettest directory for porting to new API * Minor change to runner to support mutliple inputs --- nettests/captiveportal.py | 629 +++++++++++++++++++++++++++++++++++++++++ ooni/plugins/captiveportal.py | 629 ----------------------------------------- ooni/runner.py | 1 + 3 files changed, 630 insertions(+), 629 deletions(-) diff --git a/nettests/captiveportal.py b/nettests/captiveportal.py new file mode 100644 index 0000000..4bf99b3 --- /dev/null +++ b/nettests/captiveportal.py @@ -0,0 +1,629 @@ +# -*- coding: utf-8 -*- +""" + captiveportal + ************* + + This test is a collection of tests to detect the presence of a + captive portal. Code is taken, in part, from the old ooni-probe, + which was written by Jacob Appelbaum and Arturo Filastò. + + :copyright: (c) 2012 Isis Lovecruft + :license: see LICENSE for more details +""" +import base64 +import os +import random +import re +import string +import urllib2 +from urlparse import urlparse + +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, OONITest +from ooni.protocols import http +from ooni.utils import log + +try: + from dns import resolver +except ImportError: + print "The dnspython module was not found. https://crate.io/packages/dnspython/" + resolver = None + +__plugoo__ = "captiveportal" +__desc__ = "Captive portal detection test" + +class CaptivePortalArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index'], + ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'], + ['user-agent', 'u', random.choice(http.useragents), + 'User agent for HTTP requests'] + ] + +class CaptivePortal(OONITest): + """ + Compares content and status codes of HTTP responses, and attempts + to determine if content has been altered. + """ + + implements(IPlugin, ITest) + + shortName = "captivep" + description = "Captive Portal Test" + requirements = None + options = CaptivePortalArgs + # Tells this to be blocking. + blocking = True + + def http_fetch(self, url, headers={}): + """ + Parses an HTTP url, fetches it, and returns a urllib2 response + object. + """ + url = urlparse(url).geturl() + request = urllib2.Request(url, None, headers) + response = urllib2.urlopen(request) + response_headers = dict(response.headers) + return response, response_headers + + def http_content_match_fuzzy_opt(self, experimental_url, control_result, + headers=None, fuzzy=False): + """ + Makes an HTTP request on port 80 for experimental_url, then + compares the response_content of experimental_url with the + control_result. Optionally, if the fuzzy parameter is set to + True, the response_content is compared with a regex of the + control_result. If the response_content from the + experimental_url and the control_result match, returns True + with the HTTP status code and headers; False, status code, and + headers if otherwise. + """ + + if headers is None: + default_ua = self.local_options['user-agent'] + headers = {'User-Agent': default_ua} + + response, response_headers = self.http_fetch(experimental_url, headers) + response_content = response.read() + response_code = response.code + if response_content is None: + log.warn("HTTP connection appears to have failed.") + return False, False, False + + if fuzzy: + pattern = re.compile(control_result) + match = pattern.search(response_content) + log.msg("Fuzzy HTTP content comparison for experiment URL") + log.msg("'%s'" % experimental_url) + if not match: + log.msg("does not match!") + return False, response_code, response_headers + else: + log.msg("and the expected control result yielded a match.") + return True, response_code, response_headers + else: + if str(response_content) != str(control_result): + log.msg("HTTP content comparison of experiment URL") + log.msg("'%s'" % experimental_url) + log.msg("and the expected control result do not match.") + return False, response_code, response_headers + else: + return True, response_code, response_headers + + def http_status_code_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they match. + """ + return int(experiment_code) == int(control_code) + + def http_status_code_no_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they do not match. + """ + return int(experiment_code) != int(control_code) + + def dns_resolve(self, hostname, nameserver=None): + """ + Resolves hostname(s) though nameserver to corresponding + address(es). hostname may be either a single hostname string, + or a list of strings. If nameserver is not given, use local + DNS resolver, and if that fails try using 8.8.8.8. + """ + if not resolver: + log.msg("dnspython is not installed.\ + Cannot perform DNS Resolve test") + return [] + if isinstance(hostname, str): + hostname = [hostname] + + if nameserver is not None: + res = resolver.Resolver(configure=False) + res.nameservers = [nameserver] + else: + res = resolver.Resolver() + + response = [] + answer = None + + for hn in hostname: + try: + answer = res.query(hn) + except resolver.NoNameservers: + res.nameservers = ['8.8.8.8'] + try: + answer = res.query(hn) + except resolver.NXDOMAIN: + log.msg("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + except resolver.NXDOMAIN: + log.msg("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + finally: + if not answer: + return response + for addr in answer: + response.append(addr.address) + return response + + def dns_resolve_match(self, experiment_hostname, control_address): + """ + Resolve experiment_hostname, and check to see that it returns + an experiment_address which matches the control_address. If + they match, returns True and experiment_address; otherwise + returns False and experiment_address. + """ + experiment_address = self.dns_resolve(experiment_hostname) + if not experiment_address: + log.debug("dns_resolve() for %s failed" % experiment_hostname) + return None, experiment_address + + if len(set(experiment_address) & set([control_address])) > 0: + return True, experiment_address + else: + log.msg("DNS comparison of control '%s' does not" % control_address) + log.msg("match experiment response '%s'" % experiment_address) + return False, experiment_address + + def get_auth_nameservers(self, hostname): + """ + Many CPs set a nameserver to be used. Let's query that + nameserver for the authoritative nameservers of hostname. + + The equivalent of: + $ dig +short NS ooni.nu + """ + if not resolver: + log.msg("dnspython not installed.") + log.msg("Cannot perform test.") + return [] + + res = resolver.Resolver() + answer = res.query(hostname, 'NS') + auth_nameservers = [] + for auth in answer: + auth_nameservers.append(auth.to_text()) + return auth_nameservers + + def hostname_to_0x20(self, hostname): + """ + MaKEs yOur HOsTnaME lOoK LiKE THis. + + For more information, see: + D. Dagon, et. al. "Increased DNS Forgery Resistance + Through 0x20-Bit Encoding". Proc. CSS, 2008. + """ + hostname_0x20 = '' + for char in hostname: + l33t = random.choice(['caps', 'nocaps']) + if l33t == 'caps': + hostname_0x20 += char.capitalize() + else: + hostname_0x20 += char.lower() + return hostname_0x20 + + def check_0x20_to_auth_ns(self, hostname, sample_size=None): + """ + Resolve a 0x20 DNS request for hostname over hostname's + authoritative nameserver(s), and check to make sure that + the capitalization in the 0x20 request matches that of the + response. Also, check the serial numbers of the SOA (Start + of Authority) records on the authoritative nameservers to + make sure that they match. + + If sample_size is given, a random sample equal to that number + of authoritative nameservers will be queried; default is 5. + """ + log.msg("") + log.msg("Testing random capitalization of DNS queries...") + log.msg("Testing that Start of Authority serial numbers match...") + + auth_nameservers = self.get_auth_nameservers(hostname) + + if sample_size is None: + sample_size = 5 + resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers), + sample_size) + + querynames = [] + answernames = [] + serials = [] + + # Even when gevent monkey patching is on, the requests here + # are sent without being 0x20'd, so we need to 0x20 them. + hostname = self.hostname_to_0x20(hostname) + + for auth_ns in resolved_auth_ns: + res = resolver.Resolver(configure=False) + res.nameservers = [auth_ns] + try: + answer = res.query(hostname, 'SOA') + except resolver.Timeout: + continue + querynames.append(answer.qname.to_text()) + answernames.append(answer.rrset.name.to_text()) + for soa in answer: + serials.append(str(soa.serial)) + + if len(set(querynames).intersection(answernames)) == 1: + log.msg("Capitalization in DNS queries and responses match.") + name_match = True + else: + log.msg("The random capitalization '%s' used in" % hostname) + log.msg("DNS queries to that hostname's authoritative") + log.msg("nameservers does not match the capitalization in") + log.msg("the response.") + name_match = False + + if len(set(serials)) == 1: + log.msg("Start of Authority serial numbers all match.") + serial_match = True + else: + log.msg("Some SOA serial numbers did not match the rest!") + serial_match = False + + ret = name_match, serial_match, querynames, answernames, serials + + if name_match and serial_match: + log.msg("Your DNS queries do not appear to be tampered.") + return ret + elif name_match or serial_match: + log.msg("Something is tampering with your DNS queries.") + return ret + elif not name_match and not serial_match: + log.msg("Your DNS queries are definitely being tampered with.") + return ret + + def get_random_url_safe_string(self, length): + """ + Returns a random url-safe string of specified length, where + 0 < length <= 256. The returned string will always start with + an alphabetic character. + """ + if (length <= 0): + length = 1 + elif (length > 256): + length = 256 + + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + while not random_ascii[:1].isalpha(): + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + three_quarters = int((len(random_ascii)) * (3.0/4.0)) + random_string = random_ascii[:three_quarters] + return random_string + + def get_random_hostname(self, length=None): + """ + Returns a random hostname with SLD of specified length. If + length is unspecified, length=32 is used. + + These *should* all resolve to NXDOMAIN. If they actually + resolve to a box that isn't part of a captive portal that + would be rather interesting. + """ + if length is None: + length = 32 + + random_sld = self.get_random_url_safe_string(length) + + # if it doesn't start with a letter, chuck it. + while not random_sld[:1].isalpha(): + random_sld = self.get_random_url_safe_string(length) + + tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] + random_tld = urllib2.random.choice(tld_list) + random_hostname = random_sld + random_tld + return random_hostname + + def compare_random_hostnames(self, hostname_count=None, hostname_length=None): + """ + Get hostname_count number of random hostnames with SLD length + of hostname_length, and then attempt DNS resolution. If no + arguments are given, default to three hostnames of 32 bytes + each. These random hostnames *should* resolve to NXDOMAIN, + except in the case where a user is presented with a captive + portal and remains unauthenticated, in which case the captive + portal may return the address of the authentication page. + + If the cardinality of the intersection of the set of resolved + random hostnames and the single element control set + (['NXDOMAIN']) are equal to one, then DNS properly resolved. + + Returns true if only NXDOMAINs were returned, otherwise returns + False with the relative complement of the control set in the + response set. + """ + if hostname_count is None: + hostname_count = 3 + + log.msg("Generating random hostnames...") + log.msg("Resolving DNS for %d random hostnames..." % hostname_count) + + control = ['NXDOMAIN'] + responses = [] + + for x in range(hostname_count): + random_hostname = self.get_random_hostname(hostname_length) + response_match, response_address = self.dns_resolve_match(random_hostname, + control[0]) + for address in response_address: + if response_match is False: + log.msg("Strangely, DNS resolution of the random hostname") + log.msg("%s actually points to %s" + % (random_hostname, response_address)) + responses = responses + [address] + else: + responses = responses + [address] + + intersection = set(responses) & set(control) + relative_complement = set(responses) - set(control) + r = set(responses) + + if len(intersection) == 1: + log.msg("All %d random hostnames properly resolved to NXDOMAIN." + % hostname_count) + return True, relative_complement + elif (len(intersection) == 1) and (len(r) > 1): + log.msg("Something odd happened. Some random hostnames correctly") + log.msg("resolved to NXDOMAIN, but several others resolved to") + log.msg("to the following addresses: %s" % relative_complement) + return False, relative_complement + elif (len(intersection) == 0) and (len(r) == 1): + log.msg("All random hostnames resolved to the IP address ") + log.msg("'%s', which is indicative of a captive portal." % r) + return False, relative_complement + else: + log.debug("Apparently, pigs are flying on your network, 'cause a") + log.debug("bunch of hostnames made from 32-byte random strings") + log.debug("just magically resolved to a bunch of random addresses.") + log.debug("That is definitely highly improbable. In fact, my napkin") + log.debug("tells me that the probability of just one of those") + log.debug("hostnames resolving to an address is 1.68e-59, making") + log.debug("it nearly twice as unlikely as an MD5 hash collision.") + log.debug("Either someone is seriously messing with your network,") + log.debug("or else you are witnessing the impossible. %s" % r) + return False, relative_complement + + def google_dns_cp_test(self): + """ + Google Chrome resolves three 10-byte random hostnames. + """ + subtest = "Google Chrome DNS-based" + + log.msg("") + log.msg("Running the Google Chrome DNS-based captive portal test...") + + gmatch, google_dns_result = self.compare_random_hostnames(3, 10) + + if gmatch: + log.msg("Google Chrome DNS-based captive portal test did not") + log.msg("detect a captive portal.") + return google_dns_result + else: + log.msg("Google Chrome DNS-based captive portal test believes") + log.msg("you are in a captive portal, or else something very") + log.msg("odd is happening with your DNS.") + return google_dns_result + + def ms_dns_cp_test(self): + """ + Microsoft "phones home" to a server which will always resolve + to the same address. + """ + subtest = "Microsoft NCSI DNS-based" + + log.msg("") + log.msg("Running the Microsoft NCSI DNS-based captive portal") + log.msg("test...") + + msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com", + "131.107.255.255") + if msmatch: + log.msg("Microsoft NCSI DNS-based captive portal test did not") + log.msg("detect a captive portal.") + return ms_dns_result + else: + log.msg("Microsoft NCSI DNS-based captive portal test ") + log.msg("believes you are in a captive portal.") + return ms_dns_result + + def run_vendor_dns_tests(self): + """ + Run the vendor DNS tests. + """ + report = {} + report['google_dns_cp'] = self.google_dns_cp_test() + report['ms_dns_cp'] = self.ms_dns_cp_test() + + return report + + def run_vendor_tests(self, *a, **kw): + """ + These are several vendor tests used to detect the presence of + a captive portal. Each test compares HTTP status code and + content to the control results and has its own User-Agent + string, in order to emulate the test as it would occur on the + device it was intended for. Vendor tests are defined in the + format: + [exp_url, ctrl_result, ctrl_code, ua, test_name] + """ + + vendor_tests = [['http://www.apple.com/library/test/success.html', + 'Success', + '200', + 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3', + 'Apple HTTP Captive Portal'], + ['http://tools.ietf.org/html/draft-nottingham-http-portal-02', + '428 Network Authentication Required', + '428', + 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0', + 'W3 Captive Portal'], + ['http://www.msftncsi.com/ncsi.txt', + 'Microsoft NCSI', + '200', + 'Microsoft NCSI', + 'MS HTTP Captive Portal',]] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + + def compare_content(status_func, fuzzy, experiment_url, control_result, + control_code, headers, test_name): + log.msg("") + log.msg("Running the %s test..." % test_name) + + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + headers, fuzzy) + status_match = status_func(experiment_code, control_code) + + if status_match and content_match: + log.msg("The %s test was unable to detect" % test_name) + log.msg("a captive portal.") + return True + else: + log.msg("The %s test shows that your network" % test_name) + log.msg("is filtered.") + return False + + result = [] + for vt in vendor_tests: + report = {} + report['vt'] = vt + + experiment_url = vt[0] + control_result = vt[1] + control_code = vt[2] + headers = {'User-Agent': vt[3]} + test_name = vt[4] + + args = (experiment_url, control_result, control_code, headers, test_name) + + if test_name == "MS HTTP Captive Portal": + report['result'] = compare_content(sm, False, *args) + + elif test_name == "Apple HTTP Captive Portal": + report['result'] = compare_content(sm, True, *args) + + elif test_name == "W3 Captive Portal": + report['result'] = compare_content(snm, True, *args) + + else: + log.warn("Ooni is trying to run an undefined CP vendor test.") + result.append(report) + return result + + def control(self, experiment_result, args): + """ + Compares the content and status code of the HTTP response for + experiment_url with the control_result and control_code + respectively. If the status codes match, but the experimental + content and control_result do not match, fuzzy matching is enabled + to determine if the control_result is at least included somewhere + in the experimental content. Returns True if matches are found, + and False if otherwise. + """ + experiment_url = self.local_options['experiment-url'] + control_result = 'XX' + control_code = 200 + ua = self.local_options['user-agent'] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + + log.msg("Running test for '%s'..." % experiment_url) + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result) + status_match = sm(experiment_code, control_code) + if status_match and content_match: + log.msg("The test for '%s'" % experiment_url) + log.msg("was unable to detect a captive portal.") + return experiment_result, True + + elif status_match and not content_match: + log.msg("Retrying '%s' with fuzzy match enabled." + % experiment_url) + fuzzy_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + fuzzy=True) + if fuzzy_match: + return experiment_result, True + else: + log.msg("Found modified content on '%s'," % experiment_url) + log.msg("which could indicate a captive portal.") + + return experiment_result, False + else: + log.msg("The content comparison test for ") + log.msg("'%s'" % experiment_url) + log.msg("shows that your HTTP traffic is filtered.") + return experiment_result, False + + def experiment(self, args): + """ + Runs the CaptivePortal(Test). + + CONFIG OPTIONS + -------------- + + If "do_captive_portal_vendor_tests" is set to "true", then vendor + specific captive portal HTTP-based tests will be run. + + If "do_captive_portal_dns_tests" is set to "true", then vendor + specific captive portal DNS-based tests will be run. + + If "check_dns_requests" is set to "true", then Ooni-probe will + attempt to check that your DNS requests are not being tampered with + by a captive portal. + + If "captive_portal" = "yourfilename.txt", then user-specified tests + will be run. + + Any combination of the above tests can be run. + """ + report = {} + + log.msg("") + log.msg("Running vendor tests...") + report['vendor_tests'] = self.run_vendor_tests() + + log.msg("") + log.msg("Running vendor DNS-based tests...") + report['vendor_dns_tests'] = self.run_vendor_dns_tests() + + log.msg("") + log.msg("Checking that DNS requests are not being tampered...") + report['check0x20'] = self.check_0x20_to_auth_ns('ooni.nu') + + log.msg("") + log.msg("Captive portal test finished!") + return report + +cp = CaptivePortal(None, None, None) diff --git a/ooni/plugins/captiveportal.py b/ooni/plugins/captiveportal.py deleted file mode 100644 index 4bf99b3..0000000 --- a/ooni/plugins/captiveportal.py +++ /dev/null @@ -1,629 +0,0 @@ -# -*- coding: utf-8 -*- -""" - captiveportal - ************* - - This test is a collection of tests to detect the presence of a - captive portal. Code is taken, in part, from the old ooni-probe, - which was written by Jacob Appelbaum and Arturo Filastò. - - :copyright: (c) 2012 Isis Lovecruft - :license: see LICENSE for more details -""" -import base64 -import os -import random -import re -import string -import urllib2 -from urlparse import urlparse - -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin - -from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, OONITest -from ooni.protocols import http -from ooni.utils import log - -try: - from dns import resolver -except ImportError: - print "The dnspython module was not found. https://crate.io/packages/dnspython/" - resolver = None - -__plugoo__ = "captiveportal" -__desc__ = "Captive portal detection test" - -class CaptivePortalArgs(usage.Options): - optParameters = [['asset', 'a', None, 'Asset file'], - ['resume', 'r', 0, 'Resume at this index'], - ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'], - ['user-agent', 'u', random.choice(http.useragents), - 'User agent for HTTP requests'] - ] - -class CaptivePortal(OONITest): - """ - Compares content and status codes of HTTP responses, and attempts - to determine if content has been altered. - """ - - implements(IPlugin, ITest) - - shortName = "captivep" - description = "Captive Portal Test" - requirements = None - options = CaptivePortalArgs - # Tells this to be blocking. - blocking = True - - def http_fetch(self, url, headers={}): - """ - Parses an HTTP url, fetches it, and returns a urllib2 response - object. - """ - url = urlparse(url).geturl() - request = urllib2.Request(url, None, headers) - response = urllib2.urlopen(request) - response_headers = dict(response.headers) - return response, response_headers - - def http_content_match_fuzzy_opt(self, experimental_url, control_result, - headers=None, fuzzy=False): - """ - Makes an HTTP request on port 80 for experimental_url, then - compares the response_content of experimental_url with the - control_result. Optionally, if the fuzzy parameter is set to - True, the response_content is compared with a regex of the - control_result. If the response_content from the - experimental_url and the control_result match, returns True - with the HTTP status code and headers; False, status code, and - headers if otherwise. - """ - - if headers is None: - default_ua = self.local_options['user-agent'] - headers = {'User-Agent': default_ua} - - response, response_headers = self.http_fetch(experimental_url, headers) - response_content = response.read() - response_code = response.code - if response_content is None: - log.warn("HTTP connection appears to have failed.") - return False, False, False - - if fuzzy: - pattern = re.compile(control_result) - match = pattern.search(response_content) - log.msg("Fuzzy HTTP content comparison for experiment URL") - log.msg("'%s'" % experimental_url) - if not match: - log.msg("does not match!") - return False, response_code, response_headers - else: - log.msg("and the expected control result yielded a match.") - return True, response_code, response_headers - else: - if str(response_content) != str(control_result): - log.msg("HTTP content comparison of experiment URL") - log.msg("'%s'" % experimental_url) - log.msg("and the expected control result do not match.") - return False, response_code, response_headers - else: - return True, response_code, response_headers - - def http_status_code_match(self, experiment_code, control_code): - """ - Compare two HTTP status codes, returns True if they match. - """ - return int(experiment_code) == int(control_code) - - def http_status_code_no_match(self, experiment_code, control_code): - """ - Compare two HTTP status codes, returns True if they do not match. - """ - return int(experiment_code) != int(control_code) - - def dns_resolve(self, hostname, nameserver=None): - """ - Resolves hostname(s) though nameserver to corresponding - address(es). hostname may be either a single hostname string, - or a list of strings. If nameserver is not given, use local - DNS resolver, and if that fails try using 8.8.8.8. - """ - if not resolver: - log.msg("dnspython is not installed.\ - Cannot perform DNS Resolve test") - return [] - if isinstance(hostname, str): - hostname = [hostname] - - if nameserver is not None: - res = resolver.Resolver(configure=False) - res.nameservers = [nameserver] - else: - res = resolver.Resolver() - - response = [] - answer = None - - for hn in hostname: - try: - answer = res.query(hn) - except resolver.NoNameservers: - res.nameservers = ['8.8.8.8'] - try: - answer = res.query(hn) - except resolver.NXDOMAIN: - log.msg("DNS resolution for %s returned NXDOMAIN" % hn) - response.append('NXDOMAIN') - except resolver.NXDOMAIN: - log.msg("DNS resolution for %s returned NXDOMAIN" % hn) - response.append('NXDOMAIN') - finally: - if not answer: - return response - for addr in answer: - response.append(addr.address) - return response - - def dns_resolve_match(self, experiment_hostname, control_address): - """ - Resolve experiment_hostname, and check to see that it returns - an experiment_address which matches the control_address. If - they match, returns True and experiment_address; otherwise - returns False and experiment_address. - """ - experiment_address = self.dns_resolve(experiment_hostname) - if not experiment_address: - log.debug("dns_resolve() for %s failed" % experiment_hostname) - return None, experiment_address - - if len(set(experiment_address) & set([control_address])) > 0: - return True, experiment_address - else: - log.msg("DNS comparison of control '%s' does not" % control_address) - log.msg("match experiment response '%s'" % experiment_address) - return False, experiment_address - - def get_auth_nameservers(self, hostname): - """ - Many CPs set a nameserver to be used. Let's query that - nameserver for the authoritative nameservers of hostname. - - The equivalent of: - $ dig +short NS ooni.nu - """ - if not resolver: - log.msg("dnspython not installed.") - log.msg("Cannot perform test.") - return [] - - res = resolver.Resolver() - answer = res.query(hostname, 'NS') - auth_nameservers = [] - for auth in answer: - auth_nameservers.append(auth.to_text()) - return auth_nameservers - - def hostname_to_0x20(self, hostname): - """ - MaKEs yOur HOsTnaME lOoK LiKE THis. - - For more information, see: - D. Dagon, et. al. "Increased DNS Forgery Resistance - Through 0x20-Bit Encoding". Proc. CSS, 2008. - """ - hostname_0x20 = '' - for char in hostname: - l33t = random.choice(['caps', 'nocaps']) - if l33t == 'caps': - hostname_0x20 += char.capitalize() - else: - hostname_0x20 += char.lower() - return hostname_0x20 - - def check_0x20_to_auth_ns(self, hostname, sample_size=None): - """ - Resolve a 0x20 DNS request for hostname over hostname's - authoritative nameserver(s), and check to make sure that - the capitalization in the 0x20 request matches that of the - response. Also, check the serial numbers of the SOA (Start - of Authority) records on the authoritative nameservers to - make sure that they match. - - If sample_size is given, a random sample equal to that number - of authoritative nameservers will be queried; default is 5. - """ - log.msg("") - log.msg("Testing random capitalization of DNS queries...") - log.msg("Testing that Start of Authority serial numbers match...") - - auth_nameservers = self.get_auth_nameservers(hostname) - - if sample_size is None: - sample_size = 5 - resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers), - sample_size) - - querynames = [] - answernames = [] - serials = [] - - # Even when gevent monkey patching is on, the requests here - # are sent without being 0x20'd, so we need to 0x20 them. - hostname = self.hostname_to_0x20(hostname) - - for auth_ns in resolved_auth_ns: - res = resolver.Resolver(configure=False) - res.nameservers = [auth_ns] - try: - answer = res.query(hostname, 'SOA') - except resolver.Timeout: - continue - querynames.append(answer.qname.to_text()) - answernames.append(answer.rrset.name.to_text()) - for soa in answer: - serials.append(str(soa.serial)) - - if len(set(querynames).intersection(answernames)) == 1: - log.msg("Capitalization in DNS queries and responses match.") - name_match = True - else: - log.msg("The random capitalization '%s' used in" % hostname) - log.msg("DNS queries to that hostname's authoritative") - log.msg("nameservers does not match the capitalization in") - log.msg("the response.") - name_match = False - - if len(set(serials)) == 1: - log.msg("Start of Authority serial numbers all match.") - serial_match = True - else: - log.msg("Some SOA serial numbers did not match the rest!") - serial_match = False - - ret = name_match, serial_match, querynames, answernames, serials - - if name_match and serial_match: - log.msg("Your DNS queries do not appear to be tampered.") - return ret - elif name_match or serial_match: - log.msg("Something is tampering with your DNS queries.") - return ret - elif not name_match and not serial_match: - log.msg("Your DNS queries are definitely being tampered with.") - return ret - - def get_random_url_safe_string(self, length): - """ - Returns a random url-safe string of specified length, where - 0 < length <= 256. The returned string will always start with - an alphabetic character. - """ - if (length <= 0): - length = 1 - elif (length > 256): - length = 256 - - random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) - - while not random_ascii[:1].isalpha(): - random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) - - three_quarters = int((len(random_ascii)) * (3.0/4.0)) - random_string = random_ascii[:three_quarters] - return random_string - - def get_random_hostname(self, length=None): - """ - Returns a random hostname with SLD of specified length. If - length is unspecified, length=32 is used. - - These *should* all resolve to NXDOMAIN. If they actually - resolve to a box that isn't part of a captive portal that - would be rather interesting. - """ - if length is None: - length = 32 - - random_sld = self.get_random_url_safe_string(length) - - # if it doesn't start with a letter, chuck it. - while not random_sld[:1].isalpha(): - random_sld = self.get_random_url_safe_string(length) - - tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] - random_tld = urllib2.random.choice(tld_list) - random_hostname = random_sld + random_tld - return random_hostname - - def compare_random_hostnames(self, hostname_count=None, hostname_length=None): - """ - Get hostname_count number of random hostnames with SLD length - of hostname_length, and then attempt DNS resolution. If no - arguments are given, default to three hostnames of 32 bytes - each. These random hostnames *should* resolve to NXDOMAIN, - except in the case where a user is presented with a captive - portal and remains unauthenticated, in which case the captive - portal may return the address of the authentication page. - - If the cardinality of the intersection of the set of resolved - random hostnames and the single element control set - (['NXDOMAIN']) are equal to one, then DNS properly resolved. - - Returns true if only NXDOMAINs were returned, otherwise returns - False with the relative complement of the control set in the - response set. - """ - if hostname_count is None: - hostname_count = 3 - - log.msg("Generating random hostnames...") - log.msg("Resolving DNS for %d random hostnames..." % hostname_count) - - control = ['NXDOMAIN'] - responses = [] - - for x in range(hostname_count): - random_hostname = self.get_random_hostname(hostname_length) - response_match, response_address = self.dns_resolve_match(random_hostname, - control[0]) - for address in response_address: - if response_match is False: - log.msg("Strangely, DNS resolution of the random hostname") - log.msg("%s actually points to %s" - % (random_hostname, response_address)) - responses = responses + [address] - else: - responses = responses + [address] - - intersection = set(responses) & set(control) - relative_complement = set(responses) - set(control) - r = set(responses) - - if len(intersection) == 1: - log.msg("All %d random hostnames properly resolved to NXDOMAIN." - % hostname_count) - return True, relative_complement - elif (len(intersection) == 1) and (len(r) > 1): - log.msg("Something odd happened. Some random hostnames correctly") - log.msg("resolved to NXDOMAIN, but several others resolved to") - log.msg("to the following addresses: %s" % relative_complement) - return False, relative_complement - elif (len(intersection) == 0) and (len(r) == 1): - log.msg("All random hostnames resolved to the IP address ") - log.msg("'%s', which is indicative of a captive portal." % r) - return False, relative_complement - else: - log.debug("Apparently, pigs are flying on your network, 'cause a") - log.debug("bunch of hostnames made from 32-byte random strings") - log.debug("just magically resolved to a bunch of random addresses.") - log.debug("That is definitely highly improbable. In fact, my napkin") - log.debug("tells me that the probability of just one of those") - log.debug("hostnames resolving to an address is 1.68e-59, making") - log.debug("it nearly twice as unlikely as an MD5 hash collision.") - log.debug("Either someone is seriously messing with your network,") - log.debug("or else you are witnessing the impossible. %s" % r) - return False, relative_complement - - def google_dns_cp_test(self): - """ - Google Chrome resolves three 10-byte random hostnames. - """ - subtest = "Google Chrome DNS-based" - - log.msg("") - log.msg("Running the Google Chrome DNS-based captive portal test...") - - gmatch, google_dns_result = self.compare_random_hostnames(3, 10) - - if gmatch: - log.msg("Google Chrome DNS-based captive portal test did not") - log.msg("detect a captive portal.") - return google_dns_result - else: - log.msg("Google Chrome DNS-based captive portal test believes") - log.msg("you are in a captive portal, or else something very") - log.msg("odd is happening with your DNS.") - return google_dns_result - - def ms_dns_cp_test(self): - """ - Microsoft "phones home" to a server which will always resolve - to the same address. - """ - subtest = "Microsoft NCSI DNS-based" - - log.msg("") - log.msg("Running the Microsoft NCSI DNS-based captive portal") - log.msg("test...") - - msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com", - "131.107.255.255") - if msmatch: - log.msg("Microsoft NCSI DNS-based captive portal test did not") - log.msg("detect a captive portal.") - return ms_dns_result - else: - log.msg("Microsoft NCSI DNS-based captive portal test ") - log.msg("believes you are in a captive portal.") - return ms_dns_result - - def run_vendor_dns_tests(self): - """ - Run the vendor DNS tests. - """ - report = {} - report['google_dns_cp'] = self.google_dns_cp_test() - report['ms_dns_cp'] = self.ms_dns_cp_test() - - return report - - def run_vendor_tests(self, *a, **kw): - """ - These are several vendor tests used to detect the presence of - a captive portal. Each test compares HTTP status code and - content to the control results and has its own User-Agent - string, in order to emulate the test as it would occur on the - device it was intended for. Vendor tests are defined in the - format: - [exp_url, ctrl_result, ctrl_code, ua, test_name] - """ - - vendor_tests = [['http://www.apple.com/library/test/success.html', - 'Success', - '200', - 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3', - 'Apple HTTP Captive Portal'], - ['http://tools.ietf.org/html/draft-nottingham-http-portal-02', - '428 Network Authentication Required', - '428', - 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0', - 'W3 Captive Portal'], - ['http://www.msftncsi.com/ncsi.txt', - 'Microsoft NCSI', - '200', - 'Microsoft NCSI', - 'MS HTTP Captive Portal',]] - - cm = self.http_content_match_fuzzy_opt - sm = self.http_status_code_match - snm = self.http_status_code_no_match - - def compare_content(status_func, fuzzy, experiment_url, control_result, - control_code, headers, test_name): - log.msg("") - log.msg("Running the %s test..." % test_name) - - content_match, experiment_code, experiment_headers = cm(experiment_url, - control_result, - headers, fuzzy) - status_match = status_func(experiment_code, control_code) - - if status_match and content_match: - log.msg("The %s test was unable to detect" % test_name) - log.msg("a captive portal.") - return True - else: - log.msg("The %s test shows that your network" % test_name) - log.msg("is filtered.") - return False - - result = [] - for vt in vendor_tests: - report = {} - report['vt'] = vt - - experiment_url = vt[0] - control_result = vt[1] - control_code = vt[2] - headers = {'User-Agent': vt[3]} - test_name = vt[4] - - args = (experiment_url, control_result, control_code, headers, test_name) - - if test_name == "MS HTTP Captive Portal": - report['result'] = compare_content(sm, False, *args) - - elif test_name == "Apple HTTP Captive Portal": - report['result'] = compare_content(sm, True, *args) - - elif test_name == "W3 Captive Portal": - report['result'] = compare_content(snm, True, *args) - - else: - log.warn("Ooni is trying to run an undefined CP vendor test.") - result.append(report) - return result - - def control(self, experiment_result, args): - """ - Compares the content and status code of the HTTP response for - experiment_url with the control_result and control_code - respectively. If the status codes match, but the experimental - content and control_result do not match, fuzzy matching is enabled - to determine if the control_result is at least included somewhere - in the experimental content. Returns True if matches are found, - and False if otherwise. - """ - experiment_url = self.local_options['experiment-url'] - control_result = 'XX' - control_code = 200 - ua = self.local_options['user-agent'] - - cm = self.http_content_match_fuzzy_opt - sm = self.http_status_code_match - snm = self.http_status_code_no_match - - log.msg("Running test for '%s'..." % experiment_url) - content_match, experiment_code, experiment_headers = cm(experiment_url, - control_result) - status_match = sm(experiment_code, control_code) - if status_match and content_match: - log.msg("The test for '%s'" % experiment_url) - log.msg("was unable to detect a captive portal.") - return experiment_result, True - - elif status_match and not content_match: - log.msg("Retrying '%s' with fuzzy match enabled." - % experiment_url) - fuzzy_match, experiment_code, experiment_headers = cm(experiment_url, - control_result, - fuzzy=True) - if fuzzy_match: - return experiment_result, True - else: - log.msg("Found modified content on '%s'," % experiment_url) - log.msg("which could indicate a captive portal.") - - return experiment_result, False - else: - log.msg("The content comparison test for ") - log.msg("'%s'" % experiment_url) - log.msg("shows that your HTTP traffic is filtered.") - return experiment_result, False - - def experiment(self, args): - """ - Runs the CaptivePortal(Test). - - CONFIG OPTIONS - -------------- - - If "do_captive_portal_vendor_tests" is set to "true", then vendor - specific captive portal HTTP-based tests will be run. - - If "do_captive_portal_dns_tests" is set to "true", then vendor - specific captive portal DNS-based tests will be run. - - If "check_dns_requests" is set to "true", then Ooni-probe will - attempt to check that your DNS requests are not being tampered with - by a captive portal. - - If "captive_portal" = "yourfilename.txt", then user-specified tests - will be run. - - Any combination of the above tests can be run. - """ - report = {} - - log.msg("") - log.msg("Running vendor tests...") - report['vendor_tests'] = self.run_vendor_tests() - - log.msg("") - log.msg("Running vendor DNS-based tests...") - report['vendor_dns_tests'] = self.run_vendor_dns_tests() - - log.msg("") - log.msg("Checking that DNS requests are not being tampered...") - report['check0x20'] = self.check_0x20_to_auth_ns('ooni.nu') - - log.msg("") - log.msg("Captive portal test finished!") - return report - -cp = CaptivePortal(None, None, None) diff --git a/ooni/runner.py b/ooni/runner.py index 0738bf8..a4c88a9 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -69,6 +69,7 @@ def adaptLegacyTest(obj, config): test_class.local_options = subOptions assets = test_class.load_assets() + inputs = [None] # XXX here we are only taking assets that are set to one item only. for key, inputs in assets.items(): pass

1 0

r25839: {website} someone noticed andrew used to sign rpms too, list out the f (in website/trunk: about/en docs/en)
by Andrew Lewman 06 Oct '12

06 Oct '12

Author: phobos Date: 2012-10-06 01:48:05 +0000 (Sat, 06 Oct 2012) New Revision: 25839 Modified: website/trunk/about/en/financials.wml website/trunk/docs/en/signing-keys.wml Log: someone noticed andrew used to sign rpms too, list out the first two 2011 financial docs. Modified: website/trunk/about/en/financials.wml =================================================================== --- website/trunk/about/en/financials.wml 2012-10-05 21:54:42 UTC (rev 25838) +++ website/trunk/about/en/financials.wml 2012-10-06 01:48:05 UTC (rev 25839) @@ -13,18 +13,33 @@ <div id="maincol"> <h1>Tor: Financial Reports</h1> <dl> + <dt>Fiscal Year 2011</dt> + <dd><a href="findoc/2011-TorProject-Amended-Final-Report.pdf">2011 + Financial Statements and Audit Report</a></dd> + <dd><a href="findoc/2011-TorProject-DCF.pdf">2011 Dept of Commerce + Data Collection Form</a></dd> <dt>Fiscal Year 2010</dt> - <dd><a href="findoc/2010-TorProject-combined-Form990_PC_Audit_Results.pdf">2010 IRS Form 990, State of MA Form PC, and Independent Audit Results</a></dd> + <dd><a + href="findoc/2010-TorProject-combined-Form990_PC_Audit_Results.pdf">2010 + IRS Form 990, State of MA Form PC, and Independent Audit + Results</a></dd> <dt>Fiscal Year 2009</dt> - <dd><a href="findoc/2009-TorProject-Annual-Report.pdf">2009 Annual Report</a></dd> - <dd><a href="findoc/2009-TorProject-Form990andPC.pdf">2009 IRS Form 990 and State of MA Form PC</a></dd> - <dd><a href="findoc/2009-TorProject-FinancialStatements.pdf">2009 Financial Statements and Audit results</a></dd> - <dd><a href="findoc/2009-TorProject-DCF.pdf">2009 Dept of Commerce Data Collection Form</a></dd> + <dd><a href="findoc/2009-TorProject-Annual-Report.pdf">2009 + Annual Report</a></dd> + <dd><a href="findoc/2009-TorProject-Form990andPC.pdf">2009 IRS + Form 990 and State of MA Form PC</a></dd> + <dd><a href="findoc/2009-TorProject-FinancialStatements.pdf">2009 + Financial Statements and Audit results</a></dd> + <dd><a href="findoc/2009-TorProject-DCF.pdf">2009 Dept of Commerce + Data Collection Form</a></dd> <dt>Fiscal Year 2008</dt> - <dd><a href="findoc/2008-TorProject-Form990.pdf">2008 IRS Form 990</a></dd> - <dd><a href="findoc/2008-TorProject-FinancialStatements.pdf">2008 Financial Statements and Audit Results</a></dd> + <dd><a href="findoc/2008-TorProject-Form990.pdf">2008 IRS Form + 990</a></dd> + <dd><a href="findoc/2008-TorProject-FinancialStatements.pdf">2008 + Financial Statements and Audit Results</a></dd> <dt>Fiscal Year 2007</dt> - <dd><a href="findoc/2007-TorProject-Form990.pdf">2007 IRS Form 990</a></dd> + <dd><a href="findoc/2007-TorProject-Form990.pdf">2007 IRS Form + 990</a></dd> </dl> </div>  Modified: website/trunk/docs/en/signing-keys.wml =================================================================== --- website/trunk/docs/en/signing-keys.wml 2012-10-05 21:54:42 UTC (rev 25838) +++ website/trunk/docs/en/signing-keys.wml 2012-10-06 01:48:05 UTC (rev 25839) @@ -19,8 +19,8 @@ tarballs.</li> <li>Erinn Clark (0x63FEE659) signs the Tor Browser Bundles, Vidalia bundles, and many other packages. She signs RPMs with her other key - (0xF1F5C9B5). Andrew Lewman (0x31B0974B, 0x6B4D6475) used to sign packages for - Windows and OS X.</li> + (0xF1F5C9B5). Andrew Lewman (0x31B0974B, 0x6B4D6475) used to sign + packages for RPMs, Windows, and OS X.</li> <li>Tor Project Archive (0x886DDD89) signs the deb.torproject.org repositories and archives.</li> <li>Tomás Touceda (0x9A753A6B) signs current Vidalia tarballs. Matt @@ -79,9 +79,7 @@ pub 4096R/6B4D6475 2012-02-29 Key fingerprint = 0291 ECCB E42B 2206 8E68 5545 627D EE28 6B4D 6475 - uid Andrew Lewman <andrew(a)lewman.is> uid Andrew Lewman <andrew(a)torproject.org> - uid Andrew Lewman <andrew(a)lewman.com> uid Andrew Lewman <andrew(a)torproject.is> sub 4096R/BE713AC6 2012-02-29

1 0

r25838: {website} mike points out another characteristic that would be nice to (website/trunk/about/en)
by Roger Dingledine 05 Oct '12

05 Oct '12

Author: arma Date: 2012-10-05 21:54:42 +0000 (Fri, 05 Oct 2012) New Revision: 25838 Modified: website/trunk/about/en/jobs-projectcoordinator.wml Log: mike points out another characteristic that would be nice to have Modified: website/trunk/about/en/jobs-projectcoordinator.wml =================================================================== --- website/trunk/about/en/jobs-projectcoordinator.wml 2012-10-05 16:20:39 UTC (rev 25837) +++ website/trunk/about/en/jobs-projectcoordinator.wml 2012-10-05 21:54:42 UTC (rev 25838) @@ -61,6 +61,7 @@ working with distributed teams across different time-zones containing employees and volunteers of differing skill levels over multiple mediums, including email, instant messaging, and IRC.</li> +<li>Have experience maintaining long-term software projects. <li>Have basic familiarity with distributed version control systems.</li> </ul>

1 0

r25837: {} seed the audit and dcf reports. (website/trunk/about/findoc)
by Andrew Lewman 05 Oct '12

05 Oct '12

Author: phobos Date: 2012-10-05 16:20:39 +0000 (Fri, 05 Oct 2012) New Revision: 25837 Added: website/trunk/about/findoc/2011-TorProject-Amended-Final-Report.pdf website/trunk/about/findoc/2011-TorProject-Amended-Final-Report.pdf.asc website/trunk/about/findoc/2011-TorProject-DCF.pdf website/trunk/about/findoc/2011-TorProject-DCF.pdf.asc Log: seed the audit and dcf reports. Added: website/trunk/about/findoc/2011-TorProject-Amended-Final-Report.pdf =================================================================== (Binary files differ) Property changes on: website/trunk/about/findoc/2011-TorProject-Amended-Final-Report.pdf ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: website/trunk/about/findoc/2011-TorProject-Amended-Final-Report.pdf.asc =================================================================== --- website/trunk/about/findoc/2011-TorProject-Amended-Final-Report.pdf.asc (rev 0) +++ website/trunk/about/findoc/2011-TorProject-Amended-Final-Report.pdf.asc 2012-10-05 16:20:39 UTC (rev 25837) @@ -0,0 +1,16 @@ +-----BEGIN PGP SIGNATURE----- + +iQIcBAABCAAGBQJQbwfdAAoJEGJ97ihrTWR1a4oQAKHa7GEYTU9hld8UznLLQ989 +ZuzZVNKlxzBXc264AM1As7xjyZb6jZ7JF3s3Leor7bgnC4c3re3wIgCteDpde04h +5UxaJwuvIbCo5hJBEQ0eas/5UDphNroqwypWJznoy3BPK0vuSy9eCenFMa3PNO1w +BiKneQ4H/TU50lU4ToLsB5t6r2j9es+92ZOnIYVn3UpKEceNFFGtOIbiGlRMu6+I +UY8tyDS1ewVOT89D3XygjIU0iyVeiWDicHMl1RUkxMWg7hM8KCC2VpLujjWLuWBs +N5ZfpdQaKJMKSSMUwXfQZU4TN3ffhfeBhfRNg8zJ3ryozfWtBjRYLtAGTzKEHCeJ +eN2ZUXFCU2PeTwgUA7Q8Zl41VoGDIr/OG3OYri2PCbFOmXpe6AMpuGp4eEHPmB7H +KitqFn4gHpdfYqGUQPsSU4RgyKPg7sArhRGTCetTouu/w2JjZ6W7hLVYzkMjmZaS +4IYt2+rTjinkBXii27M/Pv+FWBPlLn5bg5tawt4SrbNxDkFZZ+L7CF+MO4KfTTan +ChMg5VjrX+zjQmPwyp+GFfohq2OSnScAFeLBx/ScwZm5NJVFXwih5X96pYgAUnq7 +VBIWvN29TMls1msVJhAtWLQlpaJb/J/FCBUi+I7H6SiYCXrUWZGE2l3Iax6Kj8Gp +x+7uQDsngWNCdQQ59RGB +=h7ca +-----END PGP SIGNATURE----- Property changes on: website/trunk/about/findoc/2011-TorProject-Amended-Final-Report.pdf.asc ___________________________________________________________________ Added: svn:mime-type + text/plain Added: website/trunk/about/findoc/2011-TorProject-DCF.pdf =================================================================== (Binary files differ) Property changes on: website/trunk/about/findoc/2011-TorProject-DCF.pdf ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: website/trunk/about/findoc/2011-TorProject-DCF.pdf.asc =================================================================== --- website/trunk/about/findoc/2011-TorProject-DCF.pdf.asc (rev 0) +++ website/trunk/about/findoc/2011-TorProject-DCF.pdf.asc 2012-10-05 16:20:39 UTC (rev 25837) @@ -0,0 +1,16 @@ +-----BEGIN PGP SIGNATURE----- + +iQIcBAABCAAGBQJQbwflAAoJEGJ97ihrTWR1QQkP/3CEEXtGhaf+7/xypcN1ZulI +R9m1EPmXpxQbTbA8YbrdCfLJ0ct9tcqG+zUoeNX6RZ1KIohiuBbvKrwIPpUvBmN+ +B7YBqNg1Wahk58JpOCCHloonLqhMtZTPIkyp5cBCbgtsIQrAV8Jwfzkmr+ur9ALn +TXU4w7nk7B8GkS65K+rVNwR+eKo73/zBFj4XFuWl7mAK7WbYVt1/Qz6ZjHZvPNtu +dcRSLhAIHq5yDJO+5w3PhMddq3njGQB97lhE/eMS2/HkbPpwUjpDnXf9Yc8suhcV +I4LZMT1htQz+BltgJ1Viu8EZ4kH2wjYGMHvlqsOs2afqp3syWzlU3Gqhn/A/9brS +UwcKqB56nP+g5Yf+Z6vGiXXz9nEH6v90NfxAsUMJMz8LLqv8qUXIcoNuRtkEB/f0 +uggRUY87Mn7BvihuZoAEwQ3aqKtCMPrCbjkO3my1c4HXCTE5GRv/qKUMhmB+lWbJ +Zl2OdXJb9zTMuZKsH/KswJXsS3rru6fPNmVPBJS0m0+a0E353DGiyTofUPdSaea2 +zzqcv91eVnzJT15U497tTlevr0TkCsuB1nvFDhpiigzxsjrHV3jcMoIXozozswla +bLxDbz+/e+6IwTEscSIR1BlD+Jr5UnG87kPNsX39vwSIWqPN1iMwrEem8qEBMOjk +R6dHsG3HVLJm4WDzRdV3 +=lJMW +-----END PGP SIGNATURE----- Property changes on: website/trunk/about/findoc/2011-TorProject-DCF.pdf.asc ___________________________________________________________________ Added: svn:mime-type + text/plain

1 0

[ooni-probe/master] Remove hardcoded sys.path from reporter
by art＠torproject.org 05 Oct '12

05 Oct '12

commit bf3a4c6648cfe987a759331637cc42e7bfeef567 Author: Arturo Filastò <arturo(a)filasto.net> Date: Thu Oct 4 13:38:54 2012 +0000 Remove hardcoded sys.path from reporter --- ooni/reporter.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index ee8427d..86c1fc1 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -3,7 +3,6 @@ import sys import yaml import itertools -sys.path.insert(0, '/home/x/Documents/pen_drive_bitcoin2012/ooni-probe/ENV/lib/python2.7/site-packages') from datetime import datetime from twisted.python.util import OrderedDict, untilConcludes from twisted.trial import unittest, reporter, runner

1 0

[ooni-probe/master] Fix date structure in reporting sytem.
by art＠torproject.org 05 Oct '12

05 Oct '12

commit 43cf06c1d2c5aa410aa8b220463d84f6ac7f0a26 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue Oct 2 23:45:07 2012 +0000 Fix date structure in reporting sytem. * Collect errors in httpt template * Rename testhelpers to avoid namespace conflicts * Miscelaneous bug fixing --- bin/ooniprobe | 2 +- nettests/example_httpt.py | 9 ++++- nettests/example_scapyt.py | 2 +- ooni/lib/txscapy.py | 2 +- ooni/nettest.py | 7 ++- ooni/reporter.py | 20 +++++---- ooni/runner.py | 6 ++- ooni/templates/httpt.py | 28 ++++++------- ooni/utils/date.py | 4 +- ooni/utils/log.py | 2 +- oonib/oonibackend.py | 5 ++- oonib/testhelpers/http.py | 74 ----------------------------------- oonib/testhelpers/httph.py | 92 ++++++++++++++++++++++++++++++++++++++++++++ 13 files changed, 143 insertions(+), 110 deletions(-) diff --git a/bin/ooniprobe b/bin/ooniprobe index c653756..4e400ce 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/env python # -*- encoding: utf-8 -*- # # startup script based on twisted trial diff --git a/nettests/example_httpt.py b/nettests/example_httpt.py index 6501ca7..b113b23 100644 --- a/nettests/example_httpt.py +++ b/nettests/example_httpt.py @@ -10,7 +10,14 @@ class ExampleHTTP(httpt.HTTPTest): version = 0.1 inputs = ['http://google.com/', 'http://wikileaks.org/', - 'http://torproject.org/'] + 'http://torproject.org/'] + + def test_http(self): + if self.input: + url = self.input + return self.doRequest(url) + else: + raise Exception("No input specified") def processResponseBody(self, body): # XXX here shall go your logic diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py index c7f342d..1cc8054 100644 --- a/nettests/example_scapyt.py +++ b/nettests/example_scapyt.py @@ -11,5 +11,5 @@ class ExampleScapy(scapyt.ScapyTest): version = 0.1 inputs = [IP(dst="8.8.8.8")/TCP(dport=31337), - IP(dst="ooni.nu")/TCP(dport=31337)] + IP(dst="ooni.nu")/TCP(dport=31337)] diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py index ec605e5..d4f146e 100644 --- a/ooni/lib/txscapy.py +++ b/ooni/lib/txscapy.py @@ -84,7 +84,7 @@ class Scapy(object): """ min = 2 max = 6 - debug = True + debug = False write_only_answers = False pcapwriter = None recv = False diff --git a/ooni/nettest.py b/ooni/nettest.py index 403a022..d7188c6 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -81,8 +81,11 @@ class TestCase(unittest.TestCase): if self.inputFile: fp = open(self.inputFile) self.inputs = inputProcessor(fp) - - return {'inputs': self.inputs} + # XXX perhaps we may want to name and version to be inside of a + # different object that is not called options. + return {'inputs': self.inputs, + 'name': self.name, + 'version': self.version} def __repr__(self): return "<%s inputs=%s>" % (self.__class__, self.inputs) diff --git a/ooni/reporter.py b/ooni/reporter.py index 51a0166..3e108f9 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -7,6 +7,7 @@ sys.path.insert(0, '/home/x/Documents/pen_drive_bitcoin2012/ooni-probe/ENV/lib/p from datetime import datetime from twisted.python.util import OrderedDict, untilConcludes from twisted.trial import unittest, reporter, runner +from ooni.utils import date try: from scapy.all import packet @@ -70,19 +71,19 @@ class ReporterFactory(OReporter): self._testSuite = testSuite self._reporters = [] - def writeHeader(self): - pretty_date = "XXX Replace me with date.pretty_date()" + def writeHeader(self, options): self._writeln("###########################################") - self._writeln("# OONI Probe Report for Test %s" % "XXX replace with with the test suite name") - self._writeln("# %s" % pretty_date) + self._writeln("# OONI Probe Report for %s test" % options['name']) + self._writeln("# %s" % date.pretty_date()) self._writeln("###########################################") address = {'asn': 'XXX replace me with ASN', 'ip': 'XXX replace me with IP'} - test_details = {'start_time': datetime.now(), - 'asn': address['asn'], - 'test_name': 'XXX replace me with the test name', - 'addr': address['ip']} + test_details = {'startTime': repr(date.now()), + 'probeASN': address['asn'], + 'testName': options['name'], + 'testVersion': options['version'], + 'probeIP': address['ip']} self.writeYamlLine(test_details) self._writeln('') @@ -135,8 +136,9 @@ class OONIReporter(OReporter): test_input = repr(test.input) else: test_input = test.input + self._tests[idx]['input'] = test_input - self._tests[idx]['idx'] = idx + #self._tests[idx]['idx'] = idx self._tests[idx]['name'] = test.name #self._tests[idx]['test'] = test print "Now starting %s" % self._tests[idx] diff --git a/ooni/runner.py b/ooni/runner.py index 7883f07..0738bf8 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -129,6 +129,8 @@ def loadTestsAndOptions(classes): suiteFactory = InputTestSuite options = [] testCases = [] + names = [] + for klass in classes: try: k = klass() @@ -184,8 +186,8 @@ class ORunner(object): def run(self): #log.startLogging(sys.stdout) - log.start(True) - self.reporterFactory.writeHeader() + log.start() + self.reporterFactory.writeHeader(self.options) for inputUnit in InputUnitFactory(self.inputs): self.runWithInputUnit(inputUnit) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 51775f0..a537c6e 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -106,27 +106,25 @@ class HTTPTest(TestCase): """ pass - def doRequest(self, url): - d = self.build_request(url) + def doRequest(self, url, method="GET", headers=None, body=None): + try: + d = self.build_request(url, method, headers, body) + except Exception, e: + print e + self.report['error'] = e + + def errback(data): + print data + #self.report["error"] = data + def finished(data): - #self.mainDefer.callback() - return data + return + d.addErrback(errback) d.addCallback(self._cbResponse) d.addCallback(finished) return d - def test_http(self): - log.msg("Running experiment") - - if self.input: - url = self.input - else: - raise Exception("No input supplied") - - self.mainDefer = self.doRequest(url) - return self.mainDefer - def _cbResponse(self, response): self.response['headers'] = response.headers self.response['code'] = response.code diff --git a/ooni/utils/date.py b/ooni/utils/date.py index 59ec1f8..dcd8e36 100644 --- a/ooni/utils/date.py +++ b/ooni/utils/date.py @@ -3,10 +3,10 @@ from datetime import datetime class odate(datetime): def __str__(self): - return rfc3339(self) + return "%s" % rfc3339(self) def __repr__(self): - return "'%s'" % rfc3339(self) + return "%s" % rfc3339(self) def from_rfc(self, datestr): pass diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 6ff6bf9..06bc3fa 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -73,7 +73,7 @@ class OONILogObserver(log.FileLogObserver): util.untilConcludes(self.write, timeStr + " " + msgStr) util.untilConcludes(self.flush) -def start(logstdout, logfile=None, verbosity=None): +def start(logfile=None, logstdout=True, verbosity=None): if log.defaultObserver: verbosity = _get_log_level(verbosity) diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py index d6ba3f0..0232829 100644 --- a/oonib/oonibackend.py +++ b/oonib/oonibackend.py @@ -17,8 +17,8 @@ from twisted.web.microdom import escape from twisted.names import dns from oonib.lib import config -from oonib.testhelpers.http import HTTPBackend from oonib.lib.ssl import SSLContext +from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer from oonib.testhelpers.dns import ProxyDNSServer from oonib.testhelpers.daphn3 import Daphn3Server @@ -37,6 +37,9 @@ internet.SSLServer(int(config.main.ssl_port), SSLContext(config), ).setServiceParent(serviceCollection) +debugHTTPServer = DebugHTTPServer() +internet.TCPServer(8090, debugHTTPServer).setServiceParent(serviceCollection) + # Start the DNS Server related services TCPDNSServer = ProxyDNSServer() internet.TCPServer(int(config.main.dns_tcp_port), TCPDNSServer).setServiceParent(serviceCollection) diff --git a/oonib/testhelpers/http.py b/oonib/testhelpers/http.py deleted file mode 100644 index 79a487b..0000000 --- a/oonib/testhelpers/http.py +++ /dev/null @@ -1,74 +0,0 @@ -import json -import random -import string - -from twisted.application import internet, service -from twisted.internet import protocol, reactor, defer -from twisted.protocols import basic -from twisted.web import resource, server, static -from twisted.web.microdom import escape - -server.version = "Apache" - -class HTTPRandomPage(resource.Resource): - """ - This generates a random page of arbitrary length and containing the string - selected by the user. - The format is the following: - /random/<length>/<keyword> - """ - isLeaf = True - def _gen_random_string(self, length): - return ''.join(random.choice(string.letters) for x in range(length)) - - def genRandomPage(self, length=100, keyword=None): - data = self._gen_random_string(length/2) - if keyword: - data += keyword - data += self._gen_random_string(length - length/2) - data += '\n' - return data - - def render(self, request): - length = 100 - keyword = None - path_parts = request.path.split('/') - if len(path_parts) > 2: - length = int(path_parts[2]) - if length > 100000: - length = 100000 - - if len(path_parts) > 3: - keyword = escape(path_parts[3]) - - return self.genRandomPage(length, keyword) - -class HTTPReturnHeaders(resource.Resource): - """ - This returns the headers being sent by the client in JSON format. - """ - isLeaf = True - def render(self, request): - req_headers = request.getAllHeaders() - return json.dumps(req_headers) - -class HTTPSendHeaders(resource.Resource): - """ - This sends to the client the headers that they send inside of the POST - request encoded in json. - """ - isLeaf = True - def render_POST(self, request): - headers = json.loads(request.content.read()) - for header, value in headers.items(): - request.setHeader(str(header), str(value)) - return '' - -class HTTPBackend(resource.Resource): - def __init__(self): - resource.Resource.__init__(self) - self.putChild('random', HTTPRandomPage()) - self.putChild('returnheaders', HTTPReturnHeaders()) - self.putChild('sendheaders', HTTPSendHeaders()) - - diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py new file mode 100644 index 0000000..217dbd5 --- /dev/null +++ b/oonib/testhelpers/httph.py @@ -0,0 +1,92 @@ +import json +import random +import string + +from twisted.application import internet, service +from twisted.internet import protocol, reactor, defer +from twisted.protocols import basic +from twisted.web import resource, server, static, http +from twisted.web.microdom import escape + +server.version = "Apache" + +class HTTPRandomPage(resource.Resource): + """ + This generates a random page of arbitrary length and containing the string + selected by the user. + The format is the following: + /random/<length>/<keyword> + """ + isLeaf = True + def _gen_random_string(self, length): + return ''.join(random.choice(string.letters) for x in range(length)) + + def genRandomPage(self, length=100, keyword=None): + data = self._gen_random_string(length/2) + if keyword: + data += keyword + data += self._gen_random_string(length - length/2) + data += '\n' + return data + + def render(self, request): + length = 100 + keyword = None + path_parts = request.path.split('/') + if len(path_parts) > 2: + length = int(path_parts[2]) + if length > 100000: + length = 100000 + + if len(path_parts) > 3: + keyword = escape(path_parts[3]) + + return self.genRandomPage(length, keyword) + +class HTTPReturnHeaders(resource.Resource): + """ + This returns the headers being sent by the client in JSON format. + """ + isLeaf = True + def render(self, request): + req_headers = request.getAllHeaders() + return json.dumps(req_headers) + +class HTTPSendHeaders(resource.Resource): + """ + This sends to the client the headers that they send inside of the POST + request encoded in json. + """ + isLeaf = True + def render_POST(self, request): + headers = json.loads(request.content.read()) + for header, value in headers.items(): + request.setHeader(str(header), str(value)) + return '' + +class HTTPBackend(resource.Resource): + def __init__(self): + resource.Resource.__init__(self) + self.putChild('random', HTTPRandomPage()) + self.putChild('returnheaders', HTTPReturnHeaders()) + self.putChild('sendheaders', HTTPSendHeaders()) + +class DebugProtocol(http.HTTPChannel): + def headerReceived(self, line): + print "[HEADER] %s" % line + http.HTTPChannel.headerReceived(self, line) + + def allContentReceived(self): + print self.requests[-1].getAllHeaders() + self.transport.loseConnection() + self.connectionLost("Normal closure") + +class DebugHTTPServer(http.HTTPFactory): + protocol = DebugProtocol + + def buildProtocol(self, addr): + print "Got connection from %s" % addr + p = protocol.ServerFactory.buildProtocol(self, addr) + p.timeOut = self.timeOut + return p +

1 0