July 2012 - tor-commits - lists.torproject.org

[stem/master] Aesthetic changes to __init__.py files
by atagar＠torproject.org 09 Jul '12

09 Jul '12

commit ac0d2183f04838790c64ff3194ae29f90fd4ed93 Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Mon Jul 9 09:55:38 2012 +0530 Aesthetic changes to __init__.py files Making __all__ lists multiline when they're long and awkward. --- stem/__init__.py | 12 +++++++++++- stem/descriptor/__init__.py | 9 ++++++++- stem/response/__init__.py | 10 +++++++++- stem/util/__init__.py | 11 ++++++++++- test/__init__.py | 7 ++++++- test/integ/__init__.py | 10 +++++++++- test/unit/__init__.py | 8 +++++++- 7 files changed, 60 insertions(+), 7 deletions(-) diff --git a/stem/__init__.py b/stem/__init__.py index 8f5add1..a2f00a1 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -8,5 +8,15 @@ __contact__ = 'atagar(a)torproject.org' __url__ = 'http://www.atagar.com/stem/' __license__ = 'LGPLv3' -__all__ = ["descriptor", "response", "util", "connection", "control", "prereq", "process", "socket", "version"] +__all__ = [ + "descriptor", + "response", + "util", + "connection", + "control", + "prereq", + "process", + "socket", + "version" +] diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 4bddbc5..2328bf5 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -12,7 +12,14 @@ Package for parsing and processing descriptor data. +- __str__ - string that the descriptor was made from """ -__all__ = ["descriptor", "reader", "extrainfo_descriptor", "server_descriptor", "parse_file", "Descriptor"] +__all__ = [ + "descriptor", + "reader", + "extrainfo_descriptor", + "server_descriptor", + "parse_file", + "Descriptor" +] import os import re diff --git a/stem/response/__init__.py b/stem/response/__init__.py index 1d96988..995cc69 100644 --- a/stem/response/__init__.py +++ b/stem/response/__init__.py @@ -25,7 +25,15 @@ Parses replies from the control socket. from __future__ import with_statement -__all__ = ["getinfo", "getconf", "protocolinfo", "authchallenge", "convert", "ControlMessage", "ControlLine"] +__all__ = [ + "getinfo", + "getconf", + "protocolinfo", + "authchallenge", + "convert", + "ControlMessage", + "ControlLine" +] import re import threading diff --git a/stem/util/__init__.py b/stem/util/__init__.py index 5f6bb24..6bc70e4 100644 --- a/stem/util/__init__.py +++ b/stem/util/__init__.py @@ -2,5 +2,14 @@ Utility functions used by the stem library. """ -__all__ = ["conf", "connection", "enum", "log", "proc", "system", "term", "tor_tools"] +__all__ = [ + "conf", + "connection", + "enum", + "log", + "proc", + "system", + "term", + "tor_tools" +] diff --git a/test/__init__.py b/test/__init__.py index eb03299..adb29e4 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -2,5 +2,10 @@ Unit and integration tests for the stem library. """ -__all__ = ["check_whitespace", "mocking", "output", "runner"] +__all__ = [ + "check_whitespace", + "mocking", + "output", + "runner" +] diff --git a/test/integ/__init__.py b/test/integ/__init__.py index a9bb9ef..ec9d1ca 100644 --- a/test/integ/__init__.py +++ b/test/integ/__init__.py @@ -2,5 +2,13 @@ Integration tests for the stem library. """ -__all__ = ["connection", "control", "descriptor", "process", "socket", "util", "version"] +__all__ = [ + "connection", + "control", + "descriptor", + "process", + "socket", + "util", + "version" +] diff --git a/test/unit/__init__.py b/test/unit/__init__.py index e3dbe9a..6c77f6f 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -2,5 +2,11 @@ Unit tests for the stem library. """ -__all__ = ["connection", "descriptor", "socket", "util", "version"] +__all__ = [ + "connection", + "descriptor", + "socket", + "util", + "version" +]

1 0

[stem/master] Implement Controller.load_conf
by atagar＠torproject.org 09 Jul '12

09 Jul '12

commit 6cc4e195452bc573e34f0b2de295b1028a20eb4a Author: Ravi Chandra Padmala <neenaoffline(a)gmail.com> Date: Fri Jun 29 00:25:17 2012 +0530 Implement Controller.load_conf --- stem/control.py | 20 ++++++++++++++++++++ test/integ/control/controller.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 0 deletions(-) diff --git a/stem/control.py b/stem/control.py index 2c12d43..f68183e 100644 --- a/stem/control.py +++ b/stem/control.py @@ -756,6 +756,26 @@ class Controller(BaseController): raise stem.socket.InvalidRequest(response.code, response.message) else: raise stem.socket.ProtocolError("%s returned unexpected status code" % command) + + def load_conf(self, configtext): + """ + Sends the configuration text to Tor and loads it as if it has been read from + disk. + + :param str configtext: the configuration text + + :raises: :class:`stem.socket.ControllerError` if the call fails + """ + + response = self.msg("LOADCONF\n%s" % configtext) + stem.response.convert("SINGLELINE", response) + + if response.code in ("552", "553"): + if response.code == "552" and response.message.startswith("Invalid config file: Failed to parse/validate config: Unknown option"): + raise stem.socket.InvalidArguments(response.code, response.message, [response.message[70:response.message.find('.', 70) - 1]]) + raise stem.socket.InvalidRequest(response.code, response.message) + elif not response.is_ok(): + raise stem.socket.ProtocolError("+LOADCONF Received unexpected response\n%s" % str(response)) def _case_insensitive_lookup(entries, key, default = UNDEFINED): """ diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index 65dfe18..bcd76fb 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -267,4 +267,32 @@ class TestController(unittest.TestCase): ), reset = True) shutil.rmtree(tmpdir) + + def test_loadconf(self): + """ + Exercises Controller.load_conf with valid and invalid requests. + """ + + if test.runner.require_control(self): return + + runner = test.runner.get_runner() + + with runner.get_tor_controller() as controller: + oldconf = runner.get_torrc_contents() + + # invalid requests + self.assertRaises(stem.socket.InvalidRequest, controller.load_conf, "ContactInfo confloaded") + try: + controller.load_conf("Blahblah blah") + except stem.socket.InvalidArguments, exc: + self.assertEqual(["Blahblah"], exc.arguments) + else: + self.fail() + + # valid config + controller.load_conf(runner.get_torrc_contents() + "\nContactInfo confloaded\n") + self.assertEqual("confloaded", controller.get_conf("ContactInfo")) + + # reload original valid config + controller.load_conf(oldconf)

1 0

[ooni-probe/master] Rewrote captive portal tests and added requisite options to the .conf file.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 3002dd3a1afbe92d02734fb5120aa80b6c8c38eb Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Sun Apr 8 02:10:38 2012 -0700 Rewrote captive portal tests and added requisite options to the .conf file. --- ooni-probe.conf | 10 +++ tests/captiveportal.py | 191 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 0 deletions(-) diff --git a/ooni-probe.conf b/ooni-probe.conf index 161a008..a744fcf 100644 --- a/ooni-probe.conf +++ b/ooni-probe.conf @@ -38,6 +38,16 @@ dns_control_server = 91.191.136.152 # on positive results. dns_reverse_lookup = true +### captiveportal testing configuration parameters + +# The default User Agent that ooni-probe should send for +# HTTP requests (pretend we're a Windows box running FF10): +default_ua = Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.2) Gecko/20100101 Firefox/10.0.2 + +# This is the list of captive portal tests, in the format: +# test_name, experiment_url, control_result, control_code +captive_portal = captive_portal_tests.txt + ### traceroute testing related config parameters # This is the list of ips to traceroute to diff --git a/tests/captiveportal.py b/tests/captiveportal.py new file mode 100644 index 0000000..324d3cc --- /dev/null +++ b/tests/captiveportal.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +""" + captiveportal + ************* + + This test is a collection of tests to detect the presence of a + captive portal. Code is taken, in part from the old ooni-probe, + which was written by Jacob Appelbaum and Arturo Filastò. + + :copyright: (c) 2012 Isis Lovecruft + :license: see LICENSE for more details +""" +import os +import re +import urllib2 +from urlparse import urlparse + +from plugoo.assets import Asset +from plugoo.tests import Test + +__plugoo__ = "captiveportal" +__desc__ = "Captive portal detection test" + +class CaptivePortalAsset(Asset): + """ + Parses captive_portal_test.txt into an Asset. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + self = Asset.parse_line(self, line) + return line.replace('\n', '').split(', ') + + ''' + def next_asset(self): + self = Asset.next_asset(self) + with self.fh as fh: + asset_list = [] + lines = fh.readlines() + for line in lines: + parsed_line = self.parse_line(line) + if parsed_line: + asset_list.append(parsed_line) + else: + fh.seek(0) + raise StopIteration + return asset_list + ''' + +class CaptivePortal(Test): + """ + Compares content and status codes of HTTP responses, and attempts + to determine if content has been altered. + + TODO: compare headers + """ + def __init__(self, ooni): + Test.__init__(self, ooni, name='test') + self.default_ua = ooni.config.tests.default_ua + + def http_fetch(self, url, headers=None): + """ + Parses an HTTP url, fetches it, and returns a urllib2 response + object. + """ + url = urlparse(url).geturl() + request = urllib2.Request(url, None, headers) + response = urllib2.urlopen(request) + return response + + def http_content_match_fuzzy_opt(self, experimental_url, control_result, + fuzzy=False): + """ + Makes an HTTP request on port 80 for experimental_url, then + compares the response_content of experimental_url with the + control_result. Optionally, if the fuzzy parameter is set to + True, the response_content is compared with a regex of the + control_result. If the response_content from the + experimental_url and the control_result match, returns True + with the HTTP status code, False if otherwise. + """ + log = self.logger + default_ua = self.default_ua + + response = self.http_fetch(experimental_url, + headers={'User-Agent': default_ua}) + response_content = response.read() + response_code = response.code + if response_content is not None: + if fuzzy: + pattern = re.compile(control_result) + match = pattern.search(response_content) + if not match: + log.info("Fuzzy HTTP content comparison of experiment" \ + " URL '%s' and the expected control result" \ + " do not match." % experimental_url) + return False, response_code + else: + log.info("Fuzzy HTTP content comparison of experiment" \ + " URL '%s' and the expected control result" \ + " yielded a match." % experimental_url) + return True, response_code + else: + if str(response_content) != str(control_result): + log.info("HTTP content comparison of experiment URL" \ + " '%s' and the expected control result" \ + " do not match." % experimental_url) + return False, response_code + else: + return True, response_code + else: + log.warn("HTTP connection appears to have failed.") + return False, False + + def http_status_code_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they match. + """ + if int(experiment_code) != int(control_code): + return False + return True + + def http_status_code_no_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they do not match. + """ + if self.http_status_code_match(experiment_code, control_code): + return False + return True + + def experiment(self, *a, **kw): + """ + Compares the content and status code of the HTTP response for + experiment_url with the control_result and control_code + respectively. If the status codes match, but the experimental + content and control_result do not match, fuzzy matching is enabled + to determine if the control_result is at least included somewhere + in the experimental content. Returns True if matches are found, + and False if otherwise. + """ + test_name = kw['data'][0] + experiment_url = kw['data'][1] + control_result = kw['data'][2] + control_code = kw['data'][3] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + + log = self.logger + log.info("Running the %s test..." % test_name) + + content_match, experiment_code = cm(experiment_url, control_result) + status_match = sm(experiment_code, control_code) + + if status_match and content_match: + log.info("The %s test was unable to detect a captive portal." + % test_name) + return True + elif status_match and not content_match: + log.info("The %s test detected mismatched content, retrying with " \ + "fuzzy match enabled." % test_name) + content_fuzzy_match, experiment_code = cm(experiment_url, + control_result, + fuzzy=True) + if content_fuzzy_match: + return True + else: + return False + else: + log.info("The %s test shows that your network is filtered, possibly " \ + "due to a captive portal." % test_name) + return False + + return False + +def run(ooni): + """ + Run the CaptivePortal(Test). + """ + config = ooni.config + log = ooni.logger + assets = [CaptivePortalAsset(os.path.join(config.main.assetdir, + config.tests.captive_portal))] + + captiveportal = CaptivePortal(ooni) + log.info("Starting captive portal test...") + captiveportal.run(assets, {'index': 1}) + log.info("Captive portal test finished!") + +

1 0

[ooni-probe/master] Hardcoded vendor tests in separate function which optionally runs through setting in config, user-defined tests are optionally specified in asset file.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 6d57e84a19699a6f0dd2c6152ecafc87edacec9e Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Mon Apr 9 19:32:12 2012 -0700 Hardcoded vendor tests in separate function which optionally runs through setting in config, user-defined tests are optionally specified in asset file. --- assets/captive_portal_tests.txt | 4 + ooni-probe.conf | 13 ++- tests/captiveportal.py | 208 ++++++++++++++++++++++++++++----------- 3 files changed, 166 insertions(+), 59 deletions(-) diff --git a/assets/captive_portal_tests.txt b/assets/captive_portal_tests.txt new file mode 100644 index 0000000..1bd016f --- /dev/null +++ b/assets/captive_portal_tests.txt @@ -0,0 +1,4 @@ + +http://ooni.nu, Open Observatory of Network Interference, 200 +http://www.patternsinthevoid.net/2CDB8B35pub.asc, mQINBE5qkHABEADVnasCm9w9hUff1E4iKnzcAdp4lx6XU5USmYdwKg2RQt2VFqWQ, 200 +http://www.google.com, Search the world's information, 200 diff --git a/ooni-probe.conf b/ooni-probe.conf index a744fcf..a19c1d0 100644 --- a/ooni-probe.conf +++ b/ooni-probe.conf @@ -40,13 +40,20 @@ dns_reverse_lookup = true ### captiveportal testing configuration parameters +# This is an optional list of user defined captive portal tests, +# one per line, with each line in the format: +# experiment_url, control_result, control_code +# where experiment_url is the test page to retrieve, +# control_result is some unique text found on the test page, +# and control_code is the expected HTTP status code. +captive_portal = captive_portal_tests.txt + # The default User Agent that ooni-probe should send for # HTTP requests (pretend we're a Windows box running FF10): default_ua = Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.2) Gecko/20100101 Firefox/10.0.2 -# This is the list of captive portal tests, in the format: -# test_name, experiment_url, control_result, control_code -captive_portal = captive_portal_tests.txt +# Enable vendor tests for captive portals: +do_captive_portal_vendor_tests = true ### traceroute testing related config parameters diff --git a/tests/captiveportal.py b/tests/captiveportal.py index 324d3cc..494a4c3 100644 --- a/tests/captiveportal.py +++ b/tests/captiveportal.py @@ -23,7 +23,7 @@ __desc__ = "Captive portal detection test" class CaptivePortalAsset(Asset): """ - Parses captive_portal_test.txt into an Asset. + Parses captive_portal_tests.txt into an Asset. """ def __init__(self, file=None): self = Asset.__init__(self, file) @@ -32,28 +32,13 @@ class CaptivePortalAsset(Asset): self = Asset.parse_line(self, line) return line.replace('\n', '').split(', ') - ''' - def next_asset(self): - self = Asset.next_asset(self) - with self.fh as fh: - asset_list = [] - lines = fh.readlines() - for line in lines: - parsed_line = self.parse_line(line) - if parsed_line: - asset_list.append(parsed_line) - else: - fh.seek(0) - raise StopIteration - return asset_list - ''' - class CaptivePortal(Test): """ Compares content and status codes of HTTP responses, and attempts to determine if content has been altered. - TODO: compare headers + TODO: compare headers, random URL requests with control obtained + through Tor. """ def __init__(self, ooni): Test.__init__(self, ooni, name='test') @@ -70,7 +55,7 @@ class CaptivePortal(Test): return response def http_content_match_fuzzy_opt(self, experimental_url, control_result, - fuzzy=False): + headers=None, fuzzy=False): """ Makes an HTTP request on port 80 for experimental_url, then compares the response_content of experimental_url with the @@ -78,13 +63,15 @@ class CaptivePortal(Test): True, the response_content is compared with a regex of the control_result. If the response_content from the experimental_url and the control_result match, returns True - with the HTTP status code, False if otherwise. + with the HTTP status code, False and status code if otherwise. """ log = self.logger - default_ua = self.default_ua - response = self.http_fetch(experimental_url, - headers={'User-Agent': default_ua}) + if headers is None: + default_ua = self.default_ua + headers = {'User-Agent': default_ua} + + response = self.http_fetch(experimental_url, headers) response_content = response.read() response_code = response.code if response_content is not None: @@ -129,6 +116,89 @@ class CaptivePortal(Test): return False return True + def run_vendor_tests(self, *a, **kw): + """ + These are several vendor tests used to detect the presence of + a captive portal. Each test compares HTTP status code and + content to the control results and has its own User-Agent + string, in order to emulate the test as it would occur on the + device it was intended for. Vendor tests are defined in the + format: + [experimental_url, control_response, control_code, ua, test_name] + """ + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + + log = self.logger + + vendor_tests = [['http://www.apple.com/library/test/success.html', + 'Success', + '200', + 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3', + 'Apple HTTP Captive Portal'], + ['http://tools.ietf.org/html/draft-nottingham-http-portal-02', + '428 Network Authentication Required', + '428', + 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0', + 'W3 Captive Portal'], + ['http://www.msftncsi.com/ncsi.txt', + 'Microsoft NCSI', + '200', + 'Microsoft NCSI', + 'MS HTTP Captive Portal',]] + + log.debug("Getting vendor test data") + + for vt in vendor_tests: + experiment_url = vt[0] + control_result = vt[1] + control_code = vt[2] + ua = vt[3] + test_name = vt[4] + + if test_name == "MS HTTP Captive Portal": + log.info("Running the %s test..." % test_name) + content_match, experiment_code = cm(experiment_url, control_result, + headers={'User-Agent': ua}) + status_match = sm(experiment_code, control_code) + if status_match and content_match: + log.info("The %s test was unable to detect a captive portal." + % test_name) + else: + log.info("The %s test shows that your network is filtered." + % test_name) + + elif test_name == "Apple HTTP Captive Portal": + log.info("Running the %s test..." % test_name) + content_fuzzy_match, experiment_code = cm(experiment_url, + control_result, + headers={'User-Agent': ua}, + fuzzy=True) + status_match = sm(experiment_code, control_code) + if status_match and content_fuzzy_match: + log.info("The %s test was unable to detect a captive portal." + % test_name) + else: + log.info("The %s test shows that your network is filtered." + % test_name) + + elif test_name == "W3 Captive Portal": + log.info("Running the %s test..." % test_name) + content_fuzzy_match, experiment_code = cm(experiment_url, + control_result, + headers={'User-Agent': ua}, + fuzzy=True) + status_no_match = snm(experiment_code, control_code) + if status_no_match and content_fuzzy_match: + log.info("The %s test was unable to detect a captive portal." + % test_name) + else: + log.info("The %s test shows that your network is filtered." + % test_name) + else: + log.warn("Ooni is trying to run an undefined CP vendor test.") + def experiment(self, *a, **kw): """ Compares the content and status code of the HTTP response for @@ -139,53 +209,79 @@ class CaptivePortal(Test): in the experimental content. Returns True if matches are found, and False if otherwise. """ - test_name = kw['data'][0] - experiment_url = kw['data'][1] - control_result = kw['data'][2] - control_code = kw['data'][3] - + if (os.path.isfile(os.path.join(self.config.main.assetdir, + self.config.tests.captive_portal))): + kw['data'].append(None) + kw['data'].append('user defined') + + experiment_url = kw['data'][0] + control_result = kw['data'][1] + control_code = kw['data'][2] + ua = kw['data'][3] + test_name = kw['data'][4] + cm = self.http_content_match_fuzzy_opt sm = self.http_status_code_match - + snm = self.http_status_code_no_match + log = self.logger - log.info("Running the %s test..." % test_name) - content_match, experiment_code = cm(experiment_url, control_result) - status_match = sm(experiment_code, control_code) - - if status_match and content_match: - log.info("The %s test was unable to detect a captive portal." - % test_name) - return True - elif status_match and not content_match: - log.info("The %s test detected mismatched content, retrying with " \ - "fuzzy match enabled." % test_name) - content_fuzzy_match, experiment_code = cm(experiment_url, - control_result, - fuzzy=True) - if content_fuzzy_match: - return True + if test_name == "user defined": + log.info("Running the %s test for %s..." % (test_name, experiment_url)) + content_match, experiment_code = cm(experiment_url, control_result) + status_match = sm(experiment_code, control_code) + if status_match and content_match: + log.info("The %s test was unable to detect a captive portal." + % test_name) + return True, test_name + elif status_match and not content_match: + log.info("The %s test detected mismatched content, retrying " + "with fuzzy match enabled." % test_name) + content_fuzzy_match, experiment_code = cm(experiment_url, + control_result, + fuzzy=True) + if content_fuzzy_match: + return True, test_name + else: + return False, test_name else: - return False + log.info("The %s test shows that your network is filtered." + % test_name) + return False, test_name + else: - log.info("The %s test shows that your network is filtered, possibly " \ - "due to a captive portal." % test_name) - return False - - return False + log.warn("Ooni is trying to run an undefined captive portal test.") + return False, test_name + def run(ooni): """ - Run the CaptivePortal(Test). + Runs the CaptivePortal(Test). + + If do_captive_portal_vendor_tests is set to true, then vendor + specific captive portal tests will be run. + + If captive_portal = filename.txt, then user-specified tests + will be run. + + Either vendor tests or user-defined tests can be run, or both. """ config = ooni.config log = ooni.logger - assets = [CaptivePortalAsset(os.path.join(config.main.assetdir, - config.tests.captive_portal))] + assets = [] + if (os.path.isfile(os.path.join(config.main.assetdir, + config.tests.captive_portal))): + assets.append(CaptivePortalAsset(os.path.join(config.main.assetdir, + config.tests.captive_portal))) + captiveportal = CaptivePortal(ooni) log.info("Starting captive portal test...") + log.info("Running user defined tests...") captiveportal.run(assets, {'index': 1}) - log.info("Captive portal test finished!") - + + if config.tests.do_captive_portal_vendor_tests: + log.info("Running vendor tests...") + captiveportal.run_vendor_tests() + log.info("Captive portal test finished!")

1 0

[ooni-probe/master] Vendor test logic moved to one simple function.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 8f0b3385041f00c979298da3784c64627c526da4 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Tue Apr 10 01:58:53 2012 -0700 Vendor test logic moved to one simple function. --- tests/captiveportal.py | 80 +++++++++++++++++++++--------------------------- 1 files changed, 35 insertions(+), 45 deletions(-) diff --git a/tests/captiveportal.py b/tests/captiveportal.py index 494a4c3..7b46094 100644 --- a/tests/captiveportal.py +++ b/tests/captiveportal.py @@ -18,6 +18,12 @@ from urlparse import urlparse from plugoo.assets import Asset from plugoo.tests import Test +try: + from gevent import monkey + monkey.patch_socket() +except ImportError: + print "The gevent module was not found. https://crate.io/packages/gevent/" + __plugoo__ = "captiveportal" __desc__ = "Captive portal detection test" @@ -124,13 +130,8 @@ class CaptivePortal(Test): string, in order to emulate the test as it would occur on the device it was intended for. Vendor tests are defined in the format: - [experimental_url, control_response, control_code, ua, test_name] + [exp_url, ctrl_result, ctrl_code, ua, test_name] """ - cm = self.http_content_match_fuzzy_opt - sm = self.http_status_code_match - snm = self.http_status_code_no_match - - log = self.logger vendor_tests = [['http://www.apple.com/library/test/success.html', 'Success', @@ -148,54 +149,43 @@ class CaptivePortal(Test): 'Microsoft NCSI', 'MS HTTP Captive Portal',]] - log.debug("Getting vendor test data") + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + log = self.logger + def compare_content(status_func, exp_url, ctrl_result, ctrl_code, headers, + test_name, fuzzy): + log.info("Running the %s test..." % test_name) + content_match, exp_code = cm(exp_url, ctrl_result, headers, fuzzy) + status_match = status_func(exp_code, ctrl_code) + if status_match and content_match: + log.info("The %s test was unable to detect a captive portal." % test_name) + else: + log.info("The %s test shows that your network is filtered." % test_name) + for vt in vendor_tests: - experiment_url = vt[0] - control_result = vt[1] - control_code = vt[2] - ua = vt[3] + exp_url = vt[0] + ctrl_result = vt[1] + ctrl_code = vt[2] + headers = {'User-Agent': vt[3]} test_name = vt[4] if test_name == "MS HTTP Captive Portal": - log.info("Running the %s test..." % test_name) - content_match, experiment_code = cm(experiment_url, control_result, - headers={'User-Agent': ua}) - status_match = sm(experiment_code, control_code) - if status_match and content_match: - log.info("The %s test was unable to detect a captive portal." - % test_name) - else: - log.info("The %s test shows that your network is filtered." - % test_name) + fuzzy = False + compare_content(sm, exp_url, ctrl_result, ctrl_code, headers, + test_name, fuzzy) elif test_name == "Apple HTTP Captive Portal": - log.info("Running the %s test..." % test_name) - content_fuzzy_match, experiment_code = cm(experiment_url, - control_result, - headers={'User-Agent': ua}, - fuzzy=True) - status_match = sm(experiment_code, control_code) - if status_match and content_fuzzy_match: - log.info("The %s test was unable to detect a captive portal." - % test_name) - else: - log.info("The %s test shows that your network is filtered." - % test_name) + fuzzy = True + compare_content(sm, exp_url, ctrl_result, ctrl_code, headers, + test_name, fuzzy) elif test_name == "W3 Captive Portal": - log.info("Running the %s test..." % test_name) - content_fuzzy_match, experiment_code = cm(experiment_url, - control_result, - headers={'User-Agent': ua}, - fuzzy=True) - status_no_match = snm(experiment_code, control_code) - if status_no_match and content_fuzzy_match: - log.info("The %s test was unable to detect a captive portal." - % test_name) - else: - log.info("The %s test shows that your network is filtered." - % test_name) + fuzzy = True + compare_content(snm, exp_url, ctrl_result, ctrl_code, headers, + test_name, fuzzy) + else: log.warn("Ooni is trying to run an undefined CP vendor test.")

1 0

[ooni-probe/master] Because that declaration is still good 15 years later, and it really needed to be in the source code somewhere.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit bbfb34399d1cbddd312e663711b02f4a82207c5b Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Fri Apr 13 00:08:08 2012 -0700 Because that declaration is still good 15 years later, and it really needed to be in the source code somewhere. --- assets/captive_portal_tests.txt | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/captive_portal_tests.txt b/assets/captive_portal_tests.txt index 1bd016f..7007411 100644 --- a/assets/captive_portal_tests.txt +++ b/assets/captive_portal_tests.txt @@ -1,4 +1,4 @@ http://ooni.nu, Open Observatory of Network Interference, 200 -http://www.patternsinthevoid.net/2CDB8B35pub.asc, mQINBE5qkHABEADVnasCm9w9hUff1E4iKnzcAdp4lx6XU5USmYdwKg2RQt2VFqWQ, 200 -http://www.google.com, Search the world's information, 200 +https://w2.eff.org/Censorship/Internet_censorship_bills/barlow_0296.declaration, let us now take our leave of them, 200 +https://www.torproject.org, Defend yourself against network surveillance and traffic analysis, 200

1 0

[ooni-probe/master] Added random hostname testing and vendor DNS tests.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 95bbf24df68c13f16fd1d231326f89934133f5ae Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Thu Apr 12 16:32:42 2012 -0700 Added random hostname testing and vendor DNS tests. --- ooni-probe.conf | 3 + tests/captiveportal.py | 310 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 290 insertions(+), 23 deletions(-) diff --git a/ooni-probe.conf b/ooni-probe.conf index a19c1d0..7a456d8 100644 --- a/ooni-probe.conf +++ b/ooni-probe.conf @@ -55,6 +55,9 @@ default_ua = Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.2) Gecko/20100101 Firef # Enable vendor tests for captive portals: do_captive_portal_vendor_tests = true +# Enable DNS-based vendor tests for captive portals: +do_captive_portal_vendor_dns_tests = true + ### traceroute testing related config parameters # This is the list of ips to traceroute to diff --git a/tests/captiveportal.py b/tests/captiveportal.py index 7b46094..e74490c 100644 --- a/tests/captiveportal.py +++ b/tests/captiveportal.py @@ -4,14 +4,16 @@ ************* This test is a collection of tests to detect the presence of a - captive portal. Code is taken, in part from the old ooni-probe, + captive portal. Code is taken, in part, from the old ooni-probe, which was written by Jacob Appelbaum and Arturo Filastò. :copyright: (c) 2012 Isis Lovecruft :license: see LICENSE for more details """ +import base64 import os import re +import string import urllib2 from urlparse import urlparse @@ -19,14 +21,24 @@ from plugoo.assets import Asset from plugoo.tests import Test try: + from dns import resolver +except ImportError: + print "The dnspython module was not found. https://crate.io/packages/dnspython/" + +try: from gevent import monkey - monkey.patch_socket() + monkey.patch_socket(dns=False) except ImportError: print "The gevent module was not found. https://crate.io/packages/gevent/" __plugoo__ = "captiveportal" __desc__ = "Captive portal detection test" +# TODO make tally marker system to display all detected +# censorship event at the end of the test. +#tally = 0 +#tally_marks = [] + class CaptivePortalAsset(Asset): """ Parses captive_portal_tests.txt into an Asset. @@ -85,20 +97,20 @@ class CaptivePortal(Test): pattern = re.compile(control_result) match = pattern.search(response_content) if not match: - log.info("Fuzzy HTTP content comparison of experiment" \ - " URL '%s' and the expected control result" \ - " do not match." % experimental_url) + log.info("Fuzzy HTTP content comparison for experiment URL") + log.info("'%s'" % experimental_url) + log.info("does not match!") return False, response_code else: - log.info("Fuzzy HTTP content comparison of experiment" \ - " URL '%s' and the expected control result" \ - " yielded a match." % experimental_url) + log.info("Fuzzy HTTP content comparison of experiment URL") + log.info("'%s'" % experimental_url) + log.info("and the expected control result yielded a match.") return True, response_code else: if str(response_content) != str(control_result): - log.info("HTTP content comparison of experiment URL" \ - " '%s' and the expected control result" \ - " do not match." % experimental_url) + log.info("HTTP content comparison of experiment URL") + log.info("'%s'" % experimental_url) + log.info("and the expected control result do not match.") return False, response_code else: return True, response_code @@ -122,6 +134,216 @@ class CaptivePortal(Test): return False return True + def dns_resolve(self, hostname, nameserver=None): + """ + Resolves hostname though nameserver ns to its corresponding + address(es). If ns is not given, use local DNS resolver. + """ + log = self.logger + + if nameserver is not None: + res = resolver.Resolver(configure=False) + res.nameservers = [nameserver] + else: + res = resolver.Resolver() + + try: + answer = res.query(hostname) + response = [] + for addr in answer: + response.append(addr.address) + return response + except resolver.NXDOMAIN as e: + log.info("DNS resolution for %s returned NXDOMAIN" % hostname) + response = ['NXDOMAIN'] + return response + except: + return False + + def dns_resolve_match(self, experiment_hostname, control_address): + """ + Resolve experiment_hostname, and check to see that it returns + an experiment_address which matches the control_address. If + they match, returns True and experiment_address; otherwise + returns False and experiment_address. + """ + log = self.logger + + experiment_address = self.dns_resolve(experiment_hostname) + if experiment_address: + if len(set(experiment_address) & set([control_address])) > 0: + return True, experiment_address + else: + log.info("DNS comparison of control '%s' does not match " \ + "experiment response '%s'" % control_address, address) + return False, experiment_address + else: + return None + + def get_random_url_safe_string(self, length): + """ + Returns a random url-safe string of specified length, where + 0 < length <= 256. The returned string will always start with + an alphabetic character. + """ + if (length <= 0): + length = 1 + elif (length > 256): + length = 256 + + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + while not random_ascii[:1].isalpha(): + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + three_quarters = int((len(random_ascii)) * (3.0/4.0)) + random_string = random_ascii[:three_quarters] + return random_string + + def get_random_hostname(self, length=None): + """ + Returns a random hostname with SLD of specified length. If + length is unspecified, length=32 is used. + + These *should* all resolve to NXDOMAIN. If they actually + resolve to a box that isn't part of a captive portal that + would be rather interesting. + """ + log = self.logger + + if length is None: + length = 32 + + random_sld = self.get_random_url_safe_string(length) + + # if it doesn't start with a letter, chuck it. + while not random_sld[:1].isalpha(): + random_sld = self.get_random_url_safe_string(length) + + tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] + random_tld = urllib2.random.choice(tld_list) + random_hostname = random_sld + random_tld + return random_hostname + + def compare_random_hostnames(self, hostname_count=None, hostname_length=None): + """ + Get hostname_count number of random hostnames with SLD length + of hostname_length, and then attempt DNS resolution. If no + arguments are given, default to three hostnames of 32 bytes + each. These random hostnames *should* resolve to NXDOMAIN, + except in the case where a user is presented with a captive + portal and remains unauthenticated, in which case the captive + portal may return the address of the authentication page. + + If the cardinality of the intersection of the set of resolved + random hostnames and the single element control set + (['NXDOMAIN']) are equal to one, then DNS properly resolved. + + Returns true if only NXDOMAINs were returned, otherwise returns + False with the relative complement of the control set in the + response set. + """ + log = self.logger + + if hostname_count is None: + hostname_count = 3 + + log.info("Generating random hostnames...") + log.info("Resolving DNS for %d random hostnames..." % hostname_count) + + control = ['NXDOMAIN'] + responses = [] + + for x in range(hostname_count): + random_hostname = self.get_random_hostname(hostname_length) + response_match, response_address = self.dns_resolve_match(random_hostname, + control[0]) + if response_match is False: + log.info("Strangely, DNS resolution of the random hostname") + log.ingo("%s actually points to %s" + % (random_hostname, response_address)) + responses = responses + response_address + else: + responses = responses + response_address + + intersection = set(responses) & set(control) + relative_complement = set(responses) - set(control) + r = set(responses) + + if len(intersection) == 1: + log.info("All %d random hostnames properly resolved to NXDOMAIN." + % hostname_count) + return True, relative_complement + elif (len(intersection) == 1) and (len(r) > 1): + log.info("Something odd happened. Some random hostnames correctly " \ + "resolved to NXDOMAIN, but several others resolved " \ + "to the following addresses: %s" % relative_complement) + return False, relative_complement + elif (len(intersection) == 0) and (len(r) == 1): + log.info("All random hostnames resolved to the IP address ") + log.info("'%s', which is indicative of a captive portal." % r) + return False, relative_complement + else: + log.debug("Apparently, pigs are flying on your network, 'cause a " \ + "bunch of hostnames made from 32-byte random strings " \ + "just magically resolved to a bunch of random addresses. " \ + "That is definitely highly improbable. In fact, my napkin " \ + "tells me that the probability of just one of those " \ + "hostnames resolving to an address is 1.68e-59, making" \ + "it nearly twice as unlikely as an MD5 hash collision. " \ + "Either someone is seriously messing with your network, " \ + "or else you are witnessing the impossible. %s" % r) + return False, relative_complement + + def google_dns_cp_test(self): + """ + Google Chrome resolves three 10-byte random hostnames. + """ + log = self.logger + log.info("") + log.info("Running the Google Chrome DNS-based captive portal test...") + + gmatch, g_dns_result = self.compare_random_hostnames(3, 10) + + if gmatch: + log.info("Google Chrome DNS-based captive portal test did not") + log.info("detect a captive portal.") + return g_dns_result + else: + log.info("Google Chrome DNS-based captive portal test believes") + log.info("you are in a captive portal, or else something very") + log.info("odd is happening with your DNS.") + return g_dns_result + + def ms_dns_cp_test(self): + """ + Microsoft "phones home" to a server which will always resolve + to the same address. + """ + log = self.logger + log.info("") + log.info("Running the Microsoft NCSI DNS-based captive") + log.info("portal test...") + + msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com", + "131.107.255.255") + if msmatch: + log.info("Microsoft NCSI DNS-based captive portal test did not") + log.info("detect a captive portal.") + return ms_dns_result + else: + log.info("Microsoft NCSI DNS-based captive portal test ") + log.info("believes you are in a captive portal.") + return ms_dns_result + + def run_vendor_dns_tests(self): + """ + Run the vendor DNS tests. + """ + self.google_dns_cp_test() + self.ms_dns_cp_test() + return + def run_vendor_tests(self, *a, **kw): """ These are several vendor tests used to detect the presence of @@ -156,13 +378,16 @@ class CaptivePortal(Test): def compare_content(status_func, exp_url, ctrl_result, ctrl_code, headers, test_name, fuzzy): + log.info("") log.info("Running the %s test..." % test_name) content_match, exp_code = cm(exp_url, ctrl_result, headers, fuzzy) status_match = status_func(exp_code, ctrl_code) if status_match and content_match: - log.info("The %s test was unable to detect a captive portal." % test_name) + log.info("The %s test was unable to detect " % test_name) + log.info("a captive portal.") else: - log.info("The %s test shows that your network is filtered." % test_name) + log.info("The %s test shows that your network" % test_name) + log.info("is filtered.") for vt in vendor_tests: exp_url = vt[0] @@ -202,7 +427,7 @@ class CaptivePortal(Test): if (os.path.isfile(os.path.join(self.config.main.assetdir, self.config.tests.captive_portal))): kw['data'].append(None) - kw['data'].append('user defined') + kw['data'].append('user-defined') experiment_url = kw['data'][0] control_result = kw['data'][1] @@ -215,34 +440,65 @@ class CaptivePortal(Test): snm = self.http_status_code_no_match log = self.logger + + #tally = kw['tally'] + #tally_marks = kw['tally_marks'] - if test_name == "user defined": - log.info("Running the %s test for %s..." % (test_name, experiment_url)) + if test_name == "user-defined": + log.info("Running %s test for '%s'..." % (test_name, experiment_url)) content_match, experiment_code = cm(experiment_url, control_result) status_match = sm(experiment_code, control_code) if status_match and content_match: - log.info("The %s test was unable to detect a captive portal." - % test_name) + log.info("The %s test for '%s'" % (test_name, experiment_url)) + log.info("was unable to detect a captive portal.") return True, test_name elif status_match and not content_match: - log.info("The %s test detected mismatched content, retrying " - "with fuzzy match enabled." % test_name) + log.info("Retrying '%s' with fuzzy match enabled." + % experiment_url) content_fuzzy_match, experiment_code = cm(experiment_url, control_result, fuzzy=True) if content_fuzzy_match: return True, test_name else: + log.info("Found modified content on '%s'," % experiment_url) + log.info("which could indicate a captive portal.") + + ## TODO return exp_content and compare HTTP headers + #tally = tally + 1 + #tally_marks.append([experiment_url, experiment_code, + # control_result, control_code]) return False, test_name else: - log.info("The %s test shows that your network is filtered." - % test_name) + log.info("The content comparison test for ") + log.info("'%s'" % experiment_url) + log.info("shows that your HTTP traffic is filtered.") + #tally = tally + 1 + #tally_marks.append([experiment_url, experiment_code, + # control_result, control_code]) return False, test_name else: log.warn("Ooni is trying to run an undefined captive portal test.") return False, test_name + def confirmed_kill_count(self, *a, **kw): + """ + Yeah, sounds scary. And it is. + + This returns a tally count for detected censorship events and the + experiment results which upped the count. + """ + log = self.logger + + tally = kw['tally'] + tally_marks = kw['tally_marks'] + + log.info("") + log.info("OONI-probe captive portal test detected %d potential " % tally) + log.info("censorship events.") + log.info("Events which were flagged as potential censorship:") + log.info("%s" % tally_marks) def run(ooni): """ @@ -267,11 +523,19 @@ def run(ooni): captiveportal = CaptivePortal(ooni) log.info("Starting captive portal test...") - log.info("Running user defined tests...") captiveportal.run(assets, {'index': 1}) + #captiveportal.run(assets, {'index': 1, 'tally': tally, + # 'tally_marks': tally_marks}) if config.tests.do_captive_portal_vendor_tests: log.info("Running vendor tests...") captiveportal.run_vendor_tests() + if config.tests.do_captive_portal_vendor_dns_tests: + log.info("Running vendor DNS-based tests...") + captiveportal.run_vendor_dns_tests() + + #captiveportal.confirmed_kill_count({'tally': tally, + # 'tally_marks': tally_marks}) + log.info("Captive portal test finished!")

1 0

[ooni-probe/master] * Added gevent monkey patching for SSL
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 98d62ca235a260ab0e0ca708098200dec33168a6 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Fri Apr 13 20:31:10 2012 -0700 * Added gevent monkey patching for SSL * Fixed NoneType headers for SSL fetches * Disabled asynchronous select function becauses dnspython requires blocking select * Added horribly ugly multiple exception handling for DNS queries * Changed the logic of compare_random_hostnames() to handle cases where they don't properly resolve to NXDOMAIN * tally_mark system is still broken --- tests/captiveportal.py | 115 ++++++++++++++++++++++++++++++++---------------- 1 files changed, 77 insertions(+), 38 deletions(-) diff --git a/tests/captiveportal.py b/tests/captiveportal.py index e74490c..2ca7a18 100644 --- a/tests/captiveportal.py +++ b/tests/captiveportal.py @@ -27,7 +27,8 @@ except ImportError: try: from gevent import monkey - monkey.patch_socket(dns=False) + monkey.patch_all(socket=True, dns=False, time=True, select=False, thread=True, + os=True, ssl=True, httplib=False, aggressive=True) except ImportError: print "The gevent module was not found. https://crate.io/packages/gevent/" @@ -62,7 +63,7 @@ class CaptivePortal(Test): Test.__init__(self, ooni, name='test') self.default_ua = ooni.config.tests.default_ua - def http_fetch(self, url, headers=None): + def http_fetch(self, url, headers={}): """ Parses an HTTP url, fetches it, and returns a urllib2 response object. @@ -134,6 +135,13 @@ class CaptivePortal(Test): return False return True + def http_headers(self, experiment_url): + """ + Return only the headers of an HTTP response. + """ + response = self.http_fetch(url) + return response.headers + def dns_resolve(self, hostname, nameserver=None): """ Resolves hostname though nameserver ns to its corresponding @@ -147,18 +155,40 @@ class CaptivePortal(Test): else: res = resolver.Resolver() + def __addresses__(hostname): + answer = res.query(hostname) + response = [] + for addr in answer: + response.append(addr.address) + return response + + # This is gross and needs to be cleaned up, but it + # was the best way I could find to handle all the + # exceptions properly. try: answer = res.query(hostname) response = [] for addr in answer: response.append(addr.address) + #response = __addresses__(hostname) return response - except resolver.NXDOMAIN as e: + except resolver.NoNameservers as nns: + res.nameservers = ['8.8.8.8'] + try: + answer = res.query(hostname) + response = [] + for addr in answer: + response.append(addr.address) + #response = __addresses__(hostname) + return response + except resolver.NXDOMAIN as nx: + log.info("DNS resolution for %s returned NXDOMAIN" % hostname) + response = ['NXDOMAIN'] + return response + except resolver.NXDOMAIN as nx: log.info("DNS resolution for %s returned NXDOMAIN" % hostname) response = ['NXDOMAIN'] return response - except: - return False def dns_resolve_match(self, experiment_hostname, control_address): """ @@ -178,7 +208,8 @@ class CaptivePortal(Test): "experiment response '%s'" % control_address, address) return False, experiment_address else: - return None + log.debug("dns_resolve() for %s failed" % experiment_hostname) + return None, experiment_address def get_random_url_safe_string(self, length): """ @@ -258,13 +289,14 @@ class CaptivePortal(Test): random_hostname = self.get_random_hostname(hostname_length) response_match, response_address = self.dns_resolve_match(random_hostname, control[0]) - if response_match is False: - log.info("Strangely, DNS resolution of the random hostname") - log.ingo("%s actually points to %s" - % (random_hostname, response_address)) - responses = responses + response_address - else: - responses = responses + response_address + for address in response_address: + if response_match is False: + log.info("Strangely, DNS resolution of the random hostname") + log.info("%s actually points to %s" + % (random_hostname, response_address)) + responses = responses + [address] + else: + responses = responses + [address] intersection = set(responses) & set(control) relative_complement = set(responses) - set(control) @@ -275,24 +307,24 @@ class CaptivePortal(Test): % hostname_count) return True, relative_complement elif (len(intersection) == 1) and (len(r) > 1): - log.info("Something odd happened. Some random hostnames correctly " \ - "resolved to NXDOMAIN, but several others resolved " \ - "to the following addresses: %s" % relative_complement) + log.info("Something odd happened. Some random hostnames correctly") + log.info("resolved to NXDOMAIN, but several others resolved to") + log.info("to the following addresses: %s" % relative_complement) return False, relative_complement elif (len(intersection) == 0) and (len(r) == 1): log.info("All random hostnames resolved to the IP address ") log.info("'%s', which is indicative of a captive portal." % r) return False, relative_complement else: - log.debug("Apparently, pigs are flying on your network, 'cause a " \ - "bunch of hostnames made from 32-byte random strings " \ - "just magically resolved to a bunch of random addresses. " \ - "That is definitely highly improbable. In fact, my napkin " \ - "tells me that the probability of just one of those " \ - "hostnames resolving to an address is 1.68e-59, making" \ - "it nearly twice as unlikely as an MD5 hash collision. " \ - "Either someone is seriously messing with your network, " \ - "or else you are witnessing the impossible. %s" % r) + log.debug("Apparently, pigs are flying on your network, 'cause a") + log.debug("bunch of hostnames made from 32-byte random strings") + log.debug("just magically resolved to a bunch of random addresses.") + log.debug("That is definitely highly improbable. In fact, my napkin") + log.debug("tells me that the probability of just one of those") + log.degug("hostnames resolving to an address is 1.68e-59, making") + log.debug("it nearly twice as unlikely as an MD5 hash collision.") + log.debug("Either someone is seriously messing with your network,") + log.debug("or else you are witnessing the impossible. %s" % r) return False, relative_complement def google_dns_cp_test(self): @@ -342,6 +374,7 @@ class CaptivePortal(Test): """ self.google_dns_cp_test() self.ms_dns_cp_test() + return def run_vendor_tests(self, *a, **kw): @@ -380,8 +413,10 @@ class CaptivePortal(Test): test_name, fuzzy): log.info("") log.info("Running the %s test..." % test_name) + content_match, exp_code = cm(exp_url, ctrl_result, headers, fuzzy) status_match = status_func(exp_code, ctrl_code) + if status_match and content_match: log.info("The %s test was unable to detect " % test_name) log.info("a captive portal.") @@ -441,8 +476,8 @@ class CaptivePortal(Test): log = self.logger - #tally = kw['tally'] - #tally_marks = kw['tally_marks'] + tally = kw['tally'] + tally_marks = kw['tally_marks'] if test_name == "user-defined": log.info("Running %s test for '%s'..." % (test_name, experiment_url)) @@ -465,17 +500,17 @@ class CaptivePortal(Test): log.info("which could indicate a captive portal.") ## TODO return exp_content and compare HTTP headers - #tally = tally + 1 - #tally_marks.append([experiment_url, experiment_code, - # control_result, control_code]) + tally = tally + 1 + tally_marks.append([experiment_url, experiment_code, + control_result, control_code]) return False, test_name else: log.info("The content comparison test for ") log.info("'%s'" % experiment_url) log.info("shows that your HTTP traffic is filtered.") - #tally = tally + 1 - #tally_marks.append([experiment_url, experiment_code, - # control_result, control_code]) + tally = tally + 1 + tally_marks.append([experiment_url, experiment_code, + control_result, control_code]) return False, test_name else: @@ -512,6 +547,9 @@ def run(ooni): Either vendor tests or user-defined tests can be run, or both. """ + #tally = Storage(tally=0) + #tally_marks = Storage(tally_marks=[]) + config = ooni.config log = ooni.logger @@ -523,19 +561,20 @@ def run(ooni): captiveportal = CaptivePortal(ooni) log.info("Starting captive portal test...") - captiveportal.run(assets, {'index': 1}) - #captiveportal.run(assets, {'index': 1, 'tally': tally, - # 'tally_marks': tally_marks}) + #captiveportal.run(assets, {'index': 1}) + captiveportal.run(assets, {'index': 1, 'tally': 0, + 'tally_marks': []}) if config.tests.do_captive_portal_vendor_tests: + log.info("") log.info("Running vendor tests...") captiveportal.run_vendor_tests() if config.tests.do_captive_portal_vendor_dns_tests: + log.info("") log.info("Running vendor DNS-based tests...") captiveportal.run_vendor_dns_tests() - #captiveportal.confirmed_kill_count({'tally': tally, - # 'tally_marks': tally_marks}) + #captiveportal.confirmed_kill_count() log.info("Captive portal test finished!")

1 0

[ooni-probe/master] Added 0x20 query/response check and authoritative nameserver SOA serial number check.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 9313c75efaeda3aa4a971a20749aa5e402a2ca4a Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Thu Apr 19 05:29:08 2012 -0700 Added 0x20 query/response check and authoritative nameserver SOA serial number check. --- TODO | 2 + ooni-probe.conf | 3 + tests/captiveportal.py | 194 ++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 161 insertions(+), 38 deletions(-) diff --git a/TODO b/TODO index 9ecae4b..8cc5b90 100644 --- a/TODO +++ b/TODO @@ -18,6 +18,8 @@ effort: low, skill: medium * All the captive portal detection tests (old/ooni/http.py and old/ooni/dnsooni.py) effort: low, skill: low + - The captiveportal test is a port of the old code, plus + a couple new methods for testing. * All the DNS censorship detection tests (old/ooni/dnsooni.py) effort: medium, skill: medium diff --git a/ooni-probe.conf b/ooni-probe.conf index 7a456d8..bae6910 100644 --- a/ooni-probe.conf +++ b/ooni-probe.conf @@ -58,6 +58,9 @@ do_captive_portal_vendor_tests = true # Enable DNS-based vendor tests for captive portals: do_captive_portal_vendor_dns_tests = true +# Enable checking of DNS requests for tampering: +check_dns_requests = true + ### traceroute testing related config parameters # This is the list of ips to traceroute to diff --git a/tests/captiveportal.py b/tests/captiveportal.py index 8703c76..ffa86c4 100644 --- a/tests/captiveportal.py +++ b/tests/captiveportal.py @@ -12,6 +12,7 @@ """ import base64 import os +import random import re import string import urllib2 @@ -51,9 +52,6 @@ class CaptivePortal(Test): """ Compares content and status codes of HTTP responses, and attempts to determine if content has been altered. - - TODO: compare headers, compare 0x20 dns requests with authoritative - server answers. """ def __init__(self, ooni, name=__plugoo__): Test.__init__(self, ooni, name) @@ -135,42 +133,43 @@ class CaptivePortal(Test): def dns_resolve(self, hostname, nameserver=None): """ - Resolves hostname though nameserver ns to its corresponding - address(es). If ns is not given, use local DNS resolver. + Resolves hostname(s) though nameserver to corresponding + address(es). hostname may be either a single hostname string, + or a list of strings. If nameserver is not given, use local + DNS resolver, and if that fails try using 8.8.8.8. """ log = self.logger + if isinstance(hostname, str): + hostname = [hostname] + if nameserver is not None: res = resolver.Resolver(configure=False) res.nameservers = [nameserver] else: res = resolver.Resolver() - # This is gross and needs to be cleaned up, but it - # was the best way I could find to handle all the - # exceptions properly. - try: - answer = res.query(hostname) - response = [] - for addr in answer: - response.append(addr.address) - return response - except resolver.NoNameservers as nns: - res.nameservers = ['8.8.8.8'] + response = [] + answer = None + + for hn in hostname: try: - answer = res.query(hostname) - response = [] - for addr in answer: - response.append(addr.address) - return response - except resolver.NXDOMAIN as nx: - log.info("DNS resolution for %s returned NXDOMAIN" % hostname) - response = ['NXDOMAIN'] - return response - except resolver.NXDOMAIN as nx: - log.info("DNS resolution for %s returned NXDOMAIN" % hostname) - response = ['NXDOMAIN'] - return response + answer = res.query(hn) + except resolver.NoNameservers: + res.nameservers = ['8.8.8.8'] + try: + answer = res.query(hn) + except resolver.NXDOMAIN: + log.info("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + except resolver.NXDOMAIN: + log.info("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + finally: + if answer: + for addr in answer: + response.append(addr.address) + return response def dns_resolve_match(self, experiment_hostname, control_address): """ @@ -186,13 +185,118 @@ class CaptivePortal(Test): if len(set(experiment_address) & set([control_address])) > 0: return True, experiment_address else: - log.info("DNS comparison of control '%s' does not match " \ - "experiment response '%s'" % control_address, address) + log.info("DNS comparison of control '%s' does not" % control_address) + log.info("match experiment response '%s'" % experiment_address) return False, experiment_address else: log.debug("dns_resolve() for %s failed" % experiment_hostname) return None, experiment_address + def get_auth_nameservers(self, hostname): + """ + Many CPs set a nameserver to be used. Let's query that + nameserver for the authoritative nameservers of hostname. + + The equivalent of: + $ dig +short NS ooni.nu + """ + res = resolver.Resolver() + answer = res.query(hostname, 'NS') + auth_nameservers = [] + for auth in answer: + auth_nameservers.append(auth.to_text()) + return auth_nameservers + + def hostname_to_0x20(self, hostname): + """ + MaKEs yOur HOsTnaME lOoK LiKE THis. + + For more information, see: + D. Dagon, et. al. "Increased DNS Forgery Resistance + Through 0x20-Bit Encoding". Proc. CSS, 2008. + """ + hostname_0x20 = '' + for char in hostname: + l33t = random.choice(['caps', 'nocaps']) + if l33t == 'caps': + hostname_0x20 = hostname_0x20 + char.capitalize() + else: + hostname_0x20 = hostname_0x20 + char.lower() + return hostname_0x20 + + def check_0x20_to_auth_ns(self, hostname, sample_size=None): + """ + Resolve a 0x20 DNS request for hostname over hostname's + authoritative nameserver(s), and check to make sure that + the capitalization in the 0x20 request matches that of the + response. Also, check the serial numbers of the SOA (Start + of Authority) records on the authoritative nameservers to + make sure that they match. + + If sample_size is given, a random sample equal to that number + of authoritative nameservers will be queried; default is 5. + """ + log = self.logger + log.info("") + log.info("Testing random capitalization of DNS queries...") + log.info("Testing that Start of Authority serial numbers match...") + + auth_nameservers = self.get_auth_nameservers(hostname) + + if sample_size is None: + sample_size = 5 + resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers), + sample_size) + + querynames = [] + answernames = [] + serials = [] + + # Even when gevent monkey patching is on, the requests here + # are sent without being 0x20'd, so we need to 0x20 them. + hostname = self.hostname_to_0x20(hostname) + + for auth_ns in resolved_auth_ns: + res = resolver.Resolver(configure=False) + res.nameservers = [auth_ns] + try: + answer = res.query(hostname, 'SOA') + except resolver.Timeout: + continue + querynames.append(answer.qname.to_text()) + answernames.append(answer.rrset.name.to_text()) + for soa in answer: + serials.append(str(soa.serial)) + + if len(set(querynames).intersection(answernames)) == 1: + log.info("Capitalization in DNS queries and responses match.") + name_match = True + else: + log.info("The random capitalization '%s' used in" % hostname) + log.info("DNS queries to that hostname's authoritative") + log.info("nameservers does not match the capitalization in") + log.info("the response.") + name_match = False + + if len(set(serials)) == 1: + log.info("Start of Authority serial numbers all match.") + serial_match = True + else: + log.info("Some SOA serial numbers did not match the rest!") + serial_match = False + + ret = name_match, serial_match, querynames, answernames, serials + + if name_match and serial_match: + log.info("Your DNS queries do not appear to be tampered.") + return ret + elif name_match or serial_match: + log.info("Something is tampering with your DNS queries.") + return ret + elif not name_match and not serial_match: + log.info("Your DNS queries are definitely being tampered with.") + return ret + def get_random_url_safe_string(self, length): """ Returns a random url-safe string of specified length, where @@ -497,17 +601,26 @@ def run(ooni): """ Runs the CaptivePortal(Test). - If do_captive_portal_vendor_tests is set to true, then vendor - specific captive portal tests will be run. + CONFIG OPTIONS + -------------- + + If "do_captive_portal_vendor_tests" is set to "true", then vendor + specific captive portal HTTP-based tests will be run. + + If "do_captive_portal_dns_tests" is set to "true", then vendor + specific captive portal DNS-based tests will be run. - If captive_portal = filename.txt, then user-specified tests + If "check_dns_requests" is set to "true", then Ooni-probe will + attempt to check that your DNS requests are not being tampered with + by a captive portal. + + If "captive_portal" = "yourfilename.txt", then user-specified tests will be run. - Either vendor tests or user-defined tests can be run, or both. + Any combination of the above tests can be run. """ config = ooni.config log = ooni.logger - tally = ooni.tally assets = [] if (os.path.isfile(os.path.join(config.main.assetdir, @@ -517,8 +630,7 @@ def run(ooni): captiveportal = CaptivePortal(ooni) log.info("Starting captive portal test...") - captiveportal.run(assets, {'index': 1, 'tally': tally.count, - 'tally_marks': tally.marks}) + captiveportal.run(assets, {'index': 1}) if config.tests.do_captive_portal_vendor_tests: log.info("") @@ -530,4 +642,10 @@ def run(ooni): log.info("Running vendor DNS-based tests...") captiveportal.run_vendor_dns_tests() + if config.tests.check_dns_requests: + log.info("") + log.info("Checking that DNS requests are not being tampered...") + captiveportal.check_0x20_to_auth_ns('ooni.nu') + + log.info("") log.info("Captive portal test finished!")

1 0

[ooni-probe/master] Minor fixes before redesigning the tally mark system.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit a661a279fcfc88067d1f4c5a88eb091201885ea0 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Mon Apr 16 23:07:48 2012 -0700 Minor fixes before redesigning the tally mark system. --- ooniprobe.py | 4 ++ plugoo/tests.py | 1 + tests/captiveportal.py | 141 ++++++++++++++++-------------------------------- 3 files changed, 52 insertions(+), 94 deletions(-) diff --git a/ooniprobe.py b/ooniprobe.py index 9d9c0ac..ced3f91 100755 --- a/ooniprobe.py +++ b/ooniprobe.py @@ -51,6 +51,10 @@ class ooni(object): self.tests = Storage() #self.load_tests() + self.tally = Storage() + self.tally.count = 0 + self.tally.marks = Storage() + self.runtests = self.config.tests.run.split(",") diff --git a/plugoo/tests.py b/plugoo/tests.py index f94ba04..825cea8 100644 --- a/plugoo/tests.py +++ b/plugoo/tests.py @@ -15,6 +15,7 @@ class Test: def __init__(self, ooni, name="test"): self.config = ooni.config self.logger = ooni.logger + self.tally = ooni.tally self.name = name self.report = Report(ooni, scp=ooni.config.report.ssh, diff --git a/tests/captiveportal.py b/tests/captiveportal.py index 2ca7a18..8703c76 100644 --- a/tests/captiveportal.py +++ b/tests/captiveportal.py @@ -35,10 +35,6 @@ except ImportError: __plugoo__ = "captiveportal" __desc__ = "Captive portal detection test" -# TODO make tally marker system to display all detected -# censorship event at the end of the test. -#tally = 0 -#tally_marks = [] class CaptivePortalAsset(Asset): """ @@ -56,11 +52,11 @@ class CaptivePortal(Test): Compares content and status codes of HTTP responses, and attempts to determine if content has been altered. - TODO: compare headers, random URL requests with control obtained - through Tor. + TODO: compare headers, compare 0x20 dns requests with authoritative + server answers. """ - def __init__(self, ooni): - Test.__init__(self, ooni, name='test') + def __init__(self, ooni, name=__plugoo__): + Test.__init__(self, ooni, name) self.default_ua = ooni.config.tests.default_ua def http_fetch(self, url, headers={}): @@ -71,7 +67,8 @@ class CaptivePortal(Test): url = urlparse(url).geturl() request = urllib2.Request(url, None, headers) response = urllib2.urlopen(request) - return response + response_headers = dict(response.headers) + return response, response_headers def http_content_match_fuzzy_opt(self, experimental_url, control_result, headers=None, fuzzy=False): @@ -82,7 +79,8 @@ class CaptivePortal(Test): True, the response_content is compared with a regex of the control_result. If the response_content from the experimental_url and the control_result match, returns True - with the HTTP status code, False and status code if otherwise. + with the HTTP status code and headers; False, status code, and + headers if otherwise. """ log = self.logger @@ -90,7 +88,7 @@ class CaptivePortal(Test): default_ua = self.default_ua headers = {'User-Agent': default_ua} - response = self.http_fetch(experimental_url, headers) + response, response_headers = self.http_fetch(experimental_url, headers) response_content = response.read() response_code = response.code if response_content is not None: @@ -101,23 +99,23 @@ class CaptivePortal(Test): log.info("Fuzzy HTTP content comparison for experiment URL") log.info("'%s'" % experimental_url) log.info("does not match!") - return False, response_code + return False, response_code, response_headers else: log.info("Fuzzy HTTP content comparison of experiment URL") log.info("'%s'" % experimental_url) log.info("and the expected control result yielded a match.") - return True, response_code + return True, response_code, response_headers else: if str(response_content) != str(control_result): log.info("HTTP content comparison of experiment URL") log.info("'%s'" % experimental_url) log.info("and the expected control result do not match.") - return False, response_code + return False, response_code, response_headers else: - return True, response_code + return True, response_code, response_headers else: log.warn("HTTP connection appears to have failed.") - return False, False + return False, False, False def http_status_code_match(self, experiment_code, control_code): """ @@ -135,13 +133,6 @@ class CaptivePortal(Test): return False return True - def http_headers(self, experiment_url): - """ - Return only the headers of an HTTP response. - """ - response = self.http_fetch(url) - return response.headers - def dns_resolve(self, hostname, nameserver=None): """ Resolves hostname though nameserver ns to its corresponding @@ -155,13 +146,6 @@ class CaptivePortal(Test): else: res = resolver.Resolver() - def __addresses__(hostname): - answer = res.query(hostname) - response = [] - for addr in answer: - response.append(addr.address) - return response - # This is gross and needs to be cleaned up, but it # was the best way I could find to handle all the # exceptions properly. @@ -170,7 +154,6 @@ class CaptivePortal(Test): response = [] for addr in answer: response.append(addr.address) - #response = __addresses__(hostname) return response except resolver.NoNameservers as nns: res.nameservers = ['8.8.8.8'] @@ -179,7 +162,6 @@ class CaptivePortal(Test): response = [] for addr in answer: response.append(addr.address) - #response = __addresses__(hostname) return response except resolver.NXDOMAIN as nx: log.info("DNS resolution for %s returned NXDOMAIN" % hostname) @@ -332,20 +314,22 @@ class CaptivePortal(Test): Google Chrome resolves three 10-byte random hostnames. """ log = self.logger + subtest = "Google Chrome DNS-based" + log.info("") log.info("Running the Google Chrome DNS-based captive portal test...") - gmatch, g_dns_result = self.compare_random_hostnames(3, 10) + gmatch, google_dns_result = self.compare_random_hostnames(3, 10) if gmatch: log.info("Google Chrome DNS-based captive portal test did not") log.info("detect a captive portal.") - return g_dns_result + return google_dns_result else: log.info("Google Chrome DNS-based captive portal test believes") log.info("you are in a captive portal, or else something very") log.info("odd is happening with your DNS.") - return g_dns_result + return google_dns_result def ms_dns_cp_test(self): """ @@ -353,9 +337,11 @@ class CaptivePortal(Test): to the same address. """ log = self.logger + subtest = "Microsoft NCSI DNS-based" + log.info("") - log.info("Running the Microsoft NCSI DNS-based captive") - log.info("portal test...") + log.info("Running the Microsoft NCSI DNS-based captive portal") + log.info("test...") msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com", "131.107.255.255") @@ -409,42 +395,40 @@ class CaptivePortal(Test): snm = self.http_status_code_no_match log = self.logger - def compare_content(status_func, exp_url, ctrl_result, ctrl_code, headers, - test_name, fuzzy): + def compare_content(status_func, fuzzy, experiment_url, control_result, + control_code, headers, test_name): log.info("") log.info("Running the %s test..." % test_name) - content_match, exp_code = cm(exp_url, ctrl_result, headers, fuzzy) - status_match = status_func(exp_code, ctrl_code) + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + headers, fuzzy) + status_match = status_func(experiment_code, control_code) if status_match and content_match: - log.info("The %s test was unable to detect " % test_name) + log.info("The %s test was unable to detect" % test_name) log.info("a captive portal.") else: log.info("The %s test shows that your network" % test_name) log.info("is filtered.") for vt in vendor_tests: - exp_url = vt[0] - ctrl_result = vt[1] - ctrl_code = vt[2] + experiment_url = vt[0] + control_result = vt[1] + control_code = vt[2] headers = {'User-Agent': vt[3]} test_name = vt[4] + args = (experiment_url, control_result, control_code, headers, test_name) + if test_name == "MS HTTP Captive Portal": - fuzzy = False - compare_content(sm, exp_url, ctrl_result, ctrl_code, headers, - test_name, fuzzy) + compare_content(sm, False, *args) elif test_name == "Apple HTTP Captive Portal": - fuzzy = True - compare_content(sm, exp_url, ctrl_result, ctrl_code, headers, - test_name, fuzzy) + compare_content(sm, True, *args) elif test_name == "W3 Captive Portal": - fuzzy = True - compare_content(snm, exp_url, ctrl_result, ctrl_code, headers, - test_name, fuzzy) + compare_content(snm, True, *args) else: log.warn("Ooni is trying to run an undefined CP vendor test.") @@ -475,13 +459,11 @@ class CaptivePortal(Test): snm = self.http_status_code_no_match log = self.logger - - tally = kw['tally'] - tally_marks = kw['tally_marks'] if test_name == "user-defined": log.info("Running %s test for '%s'..." % (test_name, experiment_url)) - content_match, experiment_code = cm(experiment_url, control_result) + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result) status_match = sm(experiment_code, control_code) if status_match and content_match: log.info("The %s test for '%s'" % (test_name, experiment_url)) @@ -490,50 +472,26 @@ class CaptivePortal(Test): elif status_match and not content_match: log.info("Retrying '%s' with fuzzy match enabled." % experiment_url) - content_fuzzy_match, experiment_code = cm(experiment_url, - control_result, - fuzzy=True) - if content_fuzzy_match: + fuzzy_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + fuzzy=True) + if fuzzy_match: return True, test_name else: log.info("Found modified content on '%s'," % experiment_url) log.info("which could indicate a captive portal.") - ## TODO return exp_content and compare HTTP headers - tally = tally + 1 - tally_marks.append([experiment_url, experiment_code, - control_result, control_code]) return False, test_name else: log.info("The content comparison test for ") log.info("'%s'" % experiment_url) log.info("shows that your HTTP traffic is filtered.") - tally = tally + 1 - tally_marks.append([experiment_url, experiment_code, - control_result, control_code]) return False, test_name else: log.warn("Ooni is trying to run an undefined captive portal test.") return False, test_name - def confirmed_kill_count(self, *a, **kw): - """ - Yeah, sounds scary. And it is. - - This returns a tally count for detected censorship events and the - experiment results which upped the count. - """ - log = self.logger - - tally = kw['tally'] - tally_marks = kw['tally_marks'] - - log.info("") - log.info("OONI-probe captive portal test detected %d potential " % tally) - log.info("censorship events.") - log.info("Events which were flagged as potential censorship:") - log.info("%s" % tally_marks) def run(ooni): """ @@ -547,11 +505,9 @@ def run(ooni): Either vendor tests or user-defined tests can be run, or both. """ - #tally = Storage(tally=0) - #tally_marks = Storage(tally_marks=[]) - config = ooni.config log = ooni.logger + tally = ooni.tally assets = [] if (os.path.isfile(os.path.join(config.main.assetdir, @@ -561,9 +517,8 @@ def run(ooni): captiveportal = CaptivePortal(ooni) log.info("Starting captive portal test...") - #captiveportal.run(assets, {'index': 1}) - captiveportal.run(assets, {'index': 1, 'tally': 0, - 'tally_marks': []}) + captiveportal.run(assets, {'index': 1, 'tally': tally.count, + 'tally_marks': tally.marks}) if config.tests.do_captive_portal_vendor_tests: log.info("") @@ -575,6 +530,4 @@ def run(ooni): log.info("Running vendor DNS-based tests...") captiveportal.run_vendor_dns_tests() - #captiveportal.confirmed_kill_count() - log.info("Captive portal test finished!")

1 0