tor-commits July 2012

tor-commits@lists.torproject.org

14 participants
949 discussions

[ooni-probe/master] Merge branch 'captiveportal' of https://github.com/isislovecruft/ooni-probe into captiveportal
by art＠torproject.org 09 Jul '12

09 Jul '12

commit e3011887ec87dfe5bca3667bc6cde0f7096a6172 Merge: bcf5e79 9313c75 Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 16:29:36 2012 +0200 Merge branch 'captiveportal' of https://github.com/isislovecruft/ooni-probe into captiveportal Conflicts: ooniprobe.py plugoo/tests.py TODO | 2 + assets/captive_portal_tests.txt | 4 + ooni/ooni-probe.conf | 23 ++ tests/captiveportal.py | 651 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 680 insertions(+), 0 deletions(-) diff --cc ooni/ooni-probe.conf index d95e410,0000000..0c68fe0 mode 100644,000000..100644 --- a/ooni/ooni-probe.conf +++ b/ooni/ooni-probe.conf @@@ -1,77 -1,0 +1,100 @@@ +# ooni-probe +# +# These are the global configuration parameters necessary to +# make ooni-probe work +[main] +reportdir = reports/ +logfile = ooniprobe.log +assetdir = assets/ +testdir = oonitests/ + +loglevel = DEBUG +consoleloglevel = DEBUG +proxyaddress = 127.0.0.1:9050 + +# The following configurations are for searching for PlanetLab +# nodes, adding them to a slice, and PlanetLab general API +# authentication: +pl_username = yourusername +pl_password = yourpassword + +# These are configurations specific to the tests that should be +# run by ooni-probe +[tests] +run = dnstamper +### DNS testing related config parameters + +# This is the list of hostnames that must be looked up +dns_experiment = top-1m.txt + +# This is the dns servers to be tested +dns_experiment_dns = dns_servers.txt + +# This is the control known good DNS server +dns_control_server = 91.191.136.152 + +# Specify whether the dnstamper test should attempt to remove +# GeoIP-based false positives by doing a reverse DNS resolve +# on positive results. +dns_reverse_lookup = true + ++### captiveportal testing configuration parameters ++ ++# This is an optional list of user defined captive portal tests, ++# one per line, with each line in the format: ++# experiment_url, control_result, control_code ++# where experiment_url is the test page to retrieve, ++# control_result is some unique text found on the test page, ++# and control_code is the expected HTTP status code. ++captive_portal = captive_portal_tests.txt ++ ++# The default User Agent that ooni-probe should send for ++# HTTP requests (pretend we're a Windows box running FF10): ++default_ua = Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.2) Gecko/20100101 Firefox/10.0.2 ++ ++# Enable vendor tests for captive portals: ++do_captive_portal_vendor_tests = true ++ ++# Enable DNS-based vendor tests for captive portals: ++do_captive_portal_vendor_dns_tests = true ++ ++# Enable checking of DNS requests for tampering: ++check_dns_requests = true ++ +### traceroute testing related config parameters + +# This is the list of ips to traceroute to +traceroute = example_exp_list.txt + +# This is the list of ports that should be used +# src_x,src_y,src_z|dst_x,dst_y,dst_z +traceroute_ports = 0,53,80,123,443|0,53,80,123,443 + +# The protocol to be used in the scan +traceroute_proto = UDP, TCP, ICMP + +### keyword injection related tests + +# List of keywords +keywords = keywordlist.txt + +# hosts +keywords_hosts = hostslist.txt + +# Methods to be used for testing +keyword_method = http,telnet + +### Tor bridge testing + +tor_bridges = bridgetests.txt +tor_bridges_timeout = 40 + +[report] +file = report.log +timestamp = true +#ssh = 127.0.0.1:22 +#ssh_user = theusername +#ssh_password = thepassword +#ssh_keyfile = ~/.ssh/mykey_rsa +#ssh_rpath = ~/ooni-probe/ +#tcp = "127.0.0.1:9088"

1 0

[ooni-probe/master] Move captive portal tests to proper subdirectories
by art＠torproject.org 09 Jul '12

09 Jul '12

commit c85c627a7d12ccfeda7010fed6d88a58241e5978 Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 16:35:13 2012 +0200 Move captive portal tests to proper subdirectories --- assets/captive_portal_tests.txt | 4 - ooni/assets/captive_portal_tests.txt | 4 + ooni/plugins/captiveportal.py | 651 ++++++++++++++++++++++++++++++++++ tests/captiveportal.py | 651 ---------------------------------- 4 files changed, 655 insertions(+), 655 deletions(-) diff --git a/assets/captive_portal_tests.txt b/assets/captive_portal_tests.txt deleted file mode 100644 index 7007411..0000000 --- a/assets/captive_portal_tests.txt +++ /dev/null @@ -1,4 +0,0 @@ - -http://ooni.nu, Open Observatory of Network Interference, 200 -https://w2.eff.org/Censorship/Internet_censorship_bills/barlow_0296.declaration, let us now take our leave of them, 200 -https://www.torproject.org, Defend yourself against network surveillance and traffic analysis, 200 diff --git a/ooni/assets/captive_portal_tests.txt b/ooni/assets/captive_portal_tests.txt new file mode 100644 index 0000000..7007411 --- /dev/null +++ b/ooni/assets/captive_portal_tests.txt @@ -0,0 +1,4 @@ + +http://ooni.nu, Open Observatory of Network Interference, 200 +https://w2.eff.org/Censorship/Internet_censorship_bills/barlow_0296.declaration, let us now take our leave of them, 200 +https://www.torproject.org, Defend yourself against network surveillance and traffic analysis, 200 diff --git a/ooni/plugins/captiveportal.py b/ooni/plugins/captiveportal.py new file mode 100644 index 0000000..ffa86c4 --- /dev/null +++ b/ooni/plugins/captiveportal.py @@ -0,0 +1,651 @@ +# -*- coding: utf-8 -*- +""" + captiveportal + ************* + + This test is a collection of tests to detect the presence of a + captive portal. Code is taken, in part, from the old ooni-probe, + which was written by Jacob Appelbaum and Arturo Filastò. + + :copyright: (c) 2012 Isis Lovecruft + :license: see LICENSE for more details +""" +import base64 +import os +import random +import re +import string +import urllib2 +from urlparse import urlparse + +from plugoo.assets import Asset +from plugoo.tests import Test + +try: + from dns import resolver +except ImportError: + print "The dnspython module was not found. https://crate.io/packages/dnspython/" + +try: + from gevent import monkey + monkey.patch_all(socket=True, dns=False, time=True, select=False, thread=True, + os=True, ssl=True, httplib=False, aggressive=True) +except ImportError: + print "The gevent module was not found. https://crate.io/packages/gevent/" + +__plugoo__ = "captiveportal" +__desc__ = "Captive portal detection test" + + +class CaptivePortalAsset(Asset): + """ + Parses captive_portal_tests.txt into an Asset. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + self = Asset.parse_line(self, line) + return line.replace('\n', '').split(', ') + +class CaptivePortal(Test): + """ + Compares content and status codes of HTTP responses, and attempts + to determine if content has been altered. + """ + def __init__(self, ooni, name=__plugoo__): + Test.__init__(self, ooni, name) + self.default_ua = ooni.config.tests.default_ua + + def http_fetch(self, url, headers={}): + """ + Parses an HTTP url, fetches it, and returns a urllib2 response + object. + """ + url = urlparse(url).geturl() + request = urllib2.Request(url, None, headers) + response = urllib2.urlopen(request) + response_headers = dict(response.headers) + return response, response_headers + + def http_content_match_fuzzy_opt(self, experimental_url, control_result, + headers=None, fuzzy=False): + """ + Makes an HTTP request on port 80 for experimental_url, then + compares the response_content of experimental_url with the + control_result. Optionally, if the fuzzy parameter is set to + True, the response_content is compared with a regex of the + control_result. If the response_content from the + experimental_url and the control_result match, returns True + with the HTTP status code and headers; False, status code, and + headers if otherwise. + """ + log = self.logger + + if headers is None: + default_ua = self.default_ua + headers = {'User-Agent': default_ua} + + response, response_headers = self.http_fetch(experimental_url, headers) + response_content = response.read() + response_code = response.code + if response_content is not None: + if fuzzy: + pattern = re.compile(control_result) + match = pattern.search(response_content) + if not match: + log.info("Fuzzy HTTP content comparison for experiment URL") + log.info("'%s'" % experimental_url) + log.info("does not match!") + return False, response_code, response_headers + else: + log.info("Fuzzy HTTP content comparison of experiment URL") + log.info("'%s'" % experimental_url) + log.info("and the expected control result yielded a match.") + return True, response_code, response_headers + else: + if str(response_content) != str(control_result): + log.info("HTTP content comparison of experiment URL") + log.info("'%s'" % experimental_url) + log.info("and the expected control result do not match.") + return False, response_code, response_headers + else: + return True, response_code, response_headers + else: + log.warn("HTTP connection appears to have failed.") + return False, False, False + + def http_status_code_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they match. + """ + if int(experiment_code) != int(control_code): + return False + return True + + def http_status_code_no_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they do not match. + """ + if self.http_status_code_match(experiment_code, control_code): + return False + return True + + def dns_resolve(self, hostname, nameserver=None): + """ + Resolves hostname(s) though nameserver to corresponding + address(es). hostname may be either a single hostname string, + or a list of strings. If nameserver is not given, use local + DNS resolver, and if that fails try using 8.8.8.8. + """ + log = self.logger + + if isinstance(hostname, str): + hostname = [hostname] + + if nameserver is not None: + res = resolver.Resolver(configure=False) + res.nameservers = [nameserver] + else: + res = resolver.Resolver() + + response = [] + answer = None + + for hn in hostname: + try: + answer = res.query(hn) + except resolver.NoNameservers: + res.nameservers = ['8.8.8.8'] + try: + answer = res.query(hn) + except resolver.NXDOMAIN: + log.info("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + except resolver.NXDOMAIN: + log.info("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + finally: + if answer: + for addr in answer: + response.append(addr.address) + return response + + def dns_resolve_match(self, experiment_hostname, control_address): + """ + Resolve experiment_hostname, and check to see that it returns + an experiment_address which matches the control_address. If + they match, returns True and experiment_address; otherwise + returns False and experiment_address. + """ + log = self.logger + + experiment_address = self.dns_resolve(experiment_hostname) + if experiment_address: + if len(set(experiment_address) & set([control_address])) > 0: + return True, experiment_address + else: + log.info("DNS comparison of control '%s' does not" % control_address) + log.info("match experiment response '%s'" % experiment_address) + return False, experiment_address + else: + log.debug("dns_resolve() for %s failed" % experiment_hostname) + return None, experiment_address + + def get_auth_nameservers(self, hostname): + """ + Many CPs set a nameserver to be used. Let's query that + nameserver for the authoritative nameservers of hostname. + + The equivalent of: + $ dig +short NS ooni.nu + """ + res = resolver.Resolver() + answer = res.query(hostname, 'NS') + auth_nameservers = [] + for auth in answer: + auth_nameservers.append(auth.to_text()) + return auth_nameservers + + def hostname_to_0x20(self, hostname): + """ + MaKEs yOur HOsTnaME lOoK LiKE THis. + + For more information, see: + D. Dagon, et. al. "Increased DNS Forgery Resistance + Through 0x20-Bit Encoding". Proc. CSS, 2008. + """ + hostname_0x20 = '' + for char in hostname: + l33t = random.choice(['caps', 'nocaps']) + if l33t == 'caps': + hostname_0x20 = hostname_0x20 + char.capitalize() + else: + hostname_0x20 = hostname_0x20 + char.lower() + return hostname_0x20 + + def check_0x20_to_auth_ns(self, hostname, sample_size=None): + """ + Resolve a 0x20 DNS request for hostname over hostname's + authoritative nameserver(s), and check to make sure that + the capitalization in the 0x20 request matches that of the + response. Also, check the serial numbers of the SOA (Start + of Authority) records on the authoritative nameservers to + make sure that they match. + + If sample_size is given, a random sample equal to that number + of authoritative nameservers will be queried; default is 5. + """ + log = self.logger + log.info("") + log.info("Testing random capitalization of DNS queries...") + log.info("Testing that Start of Authority serial numbers match...") + + auth_nameservers = self.get_auth_nameservers(hostname) + + if sample_size is None: + sample_size = 5 + resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers), + sample_size) + + querynames = [] + answernames = [] + serials = [] + + # Even when gevent monkey patching is on, the requests here + # are sent without being 0x20'd, so we need to 0x20 them. + hostname = self.hostname_to_0x20(hostname) + + for auth_ns in resolved_auth_ns: + res = resolver.Resolver(configure=False) + res.nameservers = [auth_ns] + try: + answer = res.query(hostname, 'SOA') + except resolver.Timeout: + continue + querynames.append(answer.qname.to_text()) + answernames.append(answer.rrset.name.to_text()) + for soa in answer: + serials.append(str(soa.serial)) + + if len(set(querynames).intersection(answernames)) == 1: + log.info("Capitalization in DNS queries and responses match.") + name_match = True + else: + log.info("The random capitalization '%s' used in" % hostname) + log.info("DNS queries to that hostname's authoritative") + log.info("nameservers does not match the capitalization in") + log.info("the response.") + name_match = False + + if len(set(serials)) == 1: + log.info("Start of Authority serial numbers all match.") + serial_match = True + else: + log.info("Some SOA serial numbers did not match the rest!") + serial_match = False + + ret = name_match, serial_match, querynames, answernames, serials + + if name_match and serial_match: + log.info("Your DNS queries do not appear to be tampered.") + return ret + elif name_match or serial_match: + log.info("Something is tampering with your DNS queries.") + return ret + elif not name_match and not serial_match: + log.info("Your DNS queries are definitely being tampered with.") + return ret + + def get_random_url_safe_string(self, length): + """ + Returns a random url-safe string of specified length, where + 0 < length <= 256. The returned string will always start with + an alphabetic character. + """ + if (length <= 0): + length = 1 + elif (length > 256): + length = 256 + + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + while not random_ascii[:1].isalpha(): + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + three_quarters = int((len(random_ascii)) * (3.0/4.0)) + random_string = random_ascii[:three_quarters] + return random_string + + def get_random_hostname(self, length=None): + """ + Returns a random hostname with SLD of specified length. If + length is unspecified, length=32 is used. + + These *should* all resolve to NXDOMAIN. If they actually + resolve to a box that isn't part of a captive portal that + would be rather interesting. + """ + log = self.logger + + if length is None: + length = 32 + + random_sld = self.get_random_url_safe_string(length) + + # if it doesn't start with a letter, chuck it. + while not random_sld[:1].isalpha(): + random_sld = self.get_random_url_safe_string(length) + + tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] + random_tld = urllib2.random.choice(tld_list) + random_hostname = random_sld + random_tld + return random_hostname + + def compare_random_hostnames(self, hostname_count=None, hostname_length=None): + """ + Get hostname_count number of random hostnames with SLD length + of hostname_length, and then attempt DNS resolution. If no + arguments are given, default to three hostnames of 32 bytes + each. These random hostnames *should* resolve to NXDOMAIN, + except in the case where a user is presented with a captive + portal and remains unauthenticated, in which case the captive + portal may return the address of the authentication page. + + If the cardinality of the intersection of the set of resolved + random hostnames and the single element control set + (['NXDOMAIN']) are equal to one, then DNS properly resolved. + + Returns true if only NXDOMAINs were returned, otherwise returns + False with the relative complement of the control set in the + response set. + """ + log = self.logger + + if hostname_count is None: + hostname_count = 3 + + log.info("Generating random hostnames...") + log.info("Resolving DNS for %d random hostnames..." % hostname_count) + + control = ['NXDOMAIN'] + responses = [] + + for x in range(hostname_count): + random_hostname = self.get_random_hostname(hostname_length) + response_match, response_address = self.dns_resolve_match(random_hostname, + control[0]) + for address in response_address: + if response_match is False: + log.info("Strangely, DNS resolution of the random hostname") + log.info("%s actually points to %s" + % (random_hostname, response_address)) + responses = responses + [address] + else: + responses = responses + [address] + + intersection = set(responses) & set(control) + relative_complement = set(responses) - set(control) + r = set(responses) + + if len(intersection) == 1: + log.info("All %d random hostnames properly resolved to NXDOMAIN." + % hostname_count) + return True, relative_complement + elif (len(intersection) == 1) and (len(r) > 1): + log.info("Something odd happened. Some random hostnames correctly") + log.info("resolved to NXDOMAIN, but several others resolved to") + log.info("to the following addresses: %s" % relative_complement) + return False, relative_complement + elif (len(intersection) == 0) and (len(r) == 1): + log.info("All random hostnames resolved to the IP address ") + log.info("'%s', which is indicative of a captive portal." % r) + return False, relative_complement + else: + log.debug("Apparently, pigs are flying on your network, 'cause a") + log.debug("bunch of hostnames made from 32-byte random strings") + log.debug("just magically resolved to a bunch of random addresses.") + log.debug("That is definitely highly improbable. In fact, my napkin") + log.debug("tells me that the probability of just one of those") + log.degug("hostnames resolving to an address is 1.68e-59, making") + log.debug("it nearly twice as unlikely as an MD5 hash collision.") + log.debug("Either someone is seriously messing with your network,") + log.debug("or else you are witnessing the impossible. %s" % r) + return False, relative_complement + + def google_dns_cp_test(self): + """ + Google Chrome resolves three 10-byte random hostnames. + """ + log = self.logger + subtest = "Google Chrome DNS-based" + + log.info("") + log.info("Running the Google Chrome DNS-based captive portal test...") + + gmatch, google_dns_result = self.compare_random_hostnames(3, 10) + + if gmatch: + log.info("Google Chrome DNS-based captive portal test did not") + log.info("detect a captive portal.") + return google_dns_result + else: + log.info("Google Chrome DNS-based captive portal test believes") + log.info("you are in a captive portal, or else something very") + log.info("odd is happening with your DNS.") + return google_dns_result + + def ms_dns_cp_test(self): + """ + Microsoft "phones home" to a server which will always resolve + to the same address. + """ + log = self.logger + subtest = "Microsoft NCSI DNS-based" + + log.info("") + log.info("Running the Microsoft NCSI DNS-based captive portal") + log.info("test...") + + msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com", + "131.107.255.255") + if msmatch: + log.info("Microsoft NCSI DNS-based captive portal test did not") + log.info("detect a captive portal.") + return ms_dns_result + else: + log.info("Microsoft NCSI DNS-based captive portal test ") + log.info("believes you are in a captive portal.") + return ms_dns_result + + def run_vendor_dns_tests(self): + """ + Run the vendor DNS tests. + """ + self.google_dns_cp_test() + self.ms_dns_cp_test() + + return + + def run_vendor_tests(self, *a, **kw): + """ + These are several vendor tests used to detect the presence of + a captive portal. Each test compares HTTP status code and + content to the control results and has its own User-Agent + string, in order to emulate the test as it would occur on the + device it was intended for. Vendor tests are defined in the + format: + [exp_url, ctrl_result, ctrl_code, ua, test_name] + """ + + vendor_tests = [['http://www.apple.com/library/test/success.html', + 'Success', + '200', + 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3', + 'Apple HTTP Captive Portal'], + ['http://tools.ietf.org/html/draft-nottingham-http-portal-02', + '428 Network Authentication Required', + '428', + 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0', + 'W3 Captive Portal'], + ['http://www.msftncsi.com/ncsi.txt', + 'Microsoft NCSI', + '200', + 'Microsoft NCSI', + 'MS HTTP Captive Portal',]] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + log = self.logger + + def compare_content(status_func, fuzzy, experiment_url, control_result, + control_code, headers, test_name): + log.info("") + log.info("Running the %s test..." % test_name) + + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + headers, fuzzy) + status_match = status_func(experiment_code, control_code) + + if status_match and content_match: + log.info("The %s test was unable to detect" % test_name) + log.info("a captive portal.") + else: + log.info("The %s test shows that your network" % test_name) + log.info("is filtered.") + + for vt in vendor_tests: + experiment_url = vt[0] + control_result = vt[1] + control_code = vt[2] + headers = {'User-Agent': vt[3]} + test_name = vt[4] + + args = (experiment_url, control_result, control_code, headers, test_name) + + if test_name == "MS HTTP Captive Portal": + compare_content(sm, False, *args) + + elif test_name == "Apple HTTP Captive Portal": + compare_content(sm, True, *args) + + elif test_name == "W3 Captive Portal": + compare_content(snm, True, *args) + + else: + log.warn("Ooni is trying to run an undefined CP vendor test.") + + def experiment(self, *a, **kw): + """ + Compares the content and status code of the HTTP response for + experiment_url with the control_result and control_code + respectively. If the status codes match, but the experimental + content and control_result do not match, fuzzy matching is enabled + to determine if the control_result is at least included somewhere + in the experimental content. Returns True if matches are found, + and False if otherwise. + """ + if (os.path.isfile(os.path.join(self.config.main.assetdir, + self.config.tests.captive_portal))): + kw['data'].append(None) + kw['data'].append('user-defined') + + experiment_url = kw['data'][0] + control_result = kw['data'][1] + control_code = kw['data'][2] + ua = kw['data'][3] + test_name = kw['data'][4] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + + log = self.logger + + if test_name == "user-defined": + log.info("Running %s test for '%s'..." % (test_name, experiment_url)) + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result) + status_match = sm(experiment_code, control_code) + if status_match and content_match: + log.info("The %s test for '%s'" % (test_name, experiment_url)) + log.info("was unable to detect a captive portal.") + return True, test_name + elif status_match and not content_match: + log.info("Retrying '%s' with fuzzy match enabled." + % experiment_url) + fuzzy_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + fuzzy=True) + if fuzzy_match: + return True, test_name + else: + log.info("Found modified content on '%s'," % experiment_url) + log.info("which could indicate a captive portal.") + + return False, test_name + else: + log.info("The content comparison test for ") + log.info("'%s'" % experiment_url) + log.info("shows that your HTTP traffic is filtered.") + return False, test_name + + else: + log.warn("Ooni is trying to run an undefined captive portal test.") + return False, test_name + + +def run(ooni): + """ + Runs the CaptivePortal(Test). + + CONFIG OPTIONS + -------------- + + If "do_captive_portal_vendor_tests" is set to "true", then vendor + specific captive portal HTTP-based tests will be run. + + If "do_captive_portal_dns_tests" is set to "true", then vendor + specific captive portal DNS-based tests will be run. + + If "check_dns_requests" is set to "true", then Ooni-probe will + attempt to check that your DNS requests are not being tampered with + by a captive portal. + + If "captive_portal" = "yourfilename.txt", then user-specified tests + will be run. + + Any combination of the above tests can be run. + """ + config = ooni.config + log = ooni.logger + + assets = [] + if (os.path.isfile(os.path.join(config.main.assetdir, + config.tests.captive_portal))): + assets.append(CaptivePortalAsset(os.path.join(config.main.assetdir, + config.tests.captive_portal))) + + captiveportal = CaptivePortal(ooni) + log.info("Starting captive portal test...") + captiveportal.run(assets, {'index': 1}) + + if config.tests.do_captive_portal_vendor_tests: + log.info("") + log.info("Running vendor tests...") + captiveportal.run_vendor_tests() + + if config.tests.do_captive_portal_vendor_dns_tests: + log.info("") + log.info("Running vendor DNS-based tests...") + captiveportal.run_vendor_dns_tests() + + if config.tests.check_dns_requests: + log.info("") + log.info("Checking that DNS requests are not being tampered...") + captiveportal.check_0x20_to_auth_ns('ooni.nu') + + log.info("") + log.info("Captive portal test finished!") diff --git a/tests/captiveportal.py b/tests/captiveportal.py deleted file mode 100644 index ffa86c4..0000000 --- a/tests/captiveportal.py +++ /dev/null @@ -1,651 +0,0 @@ -# -*- coding: utf-8 -*- -""" - captiveportal - ************* - - This test is a collection of tests to detect the presence of a - captive portal. Code is taken, in part, from the old ooni-probe, - which was written by Jacob Appelbaum and Arturo Filastò. - - :copyright: (c) 2012 Isis Lovecruft - :license: see LICENSE for more details -""" -import base64 -import os -import random -import re -import string -import urllib2 -from urlparse import urlparse - -from plugoo.assets import Asset -from plugoo.tests import Test - -try: - from dns import resolver -except ImportError: - print "The dnspython module was not found. https://crate.io/packages/dnspython/" - -try: - from gevent import monkey - monkey.patch_all(socket=True, dns=False, time=True, select=False, thread=True, - os=True, ssl=True, httplib=False, aggressive=True) -except ImportError: - print "The gevent module was not found. https://crate.io/packages/gevent/" - -__plugoo__ = "captiveportal" -__desc__ = "Captive portal detection test" - - -class CaptivePortalAsset(Asset): - """ - Parses captive_portal_tests.txt into an Asset. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - def parse_line(self, line): - self = Asset.parse_line(self, line) - return line.replace('\n', '').split(', ') - -class CaptivePortal(Test): - """ - Compares content and status codes of HTTP responses, and attempts - to determine if content has been altered. - """ - def __init__(self, ooni, name=__plugoo__): - Test.__init__(self, ooni, name) - self.default_ua = ooni.config.tests.default_ua - - def http_fetch(self, url, headers={}): - """ - Parses an HTTP url, fetches it, and returns a urllib2 response - object. - """ - url = urlparse(url).geturl() - request = urllib2.Request(url, None, headers) - response = urllib2.urlopen(request) - response_headers = dict(response.headers) - return response, response_headers - - def http_content_match_fuzzy_opt(self, experimental_url, control_result, - headers=None, fuzzy=False): - """ - Makes an HTTP request on port 80 for experimental_url, then - compares the response_content of experimental_url with the - control_result. Optionally, if the fuzzy parameter is set to - True, the response_content is compared with a regex of the - control_result. If the response_content from the - experimental_url and the control_result match, returns True - with the HTTP status code and headers; False, status code, and - headers if otherwise. - """ - log = self.logger - - if headers is None: - default_ua = self.default_ua - headers = {'User-Agent': default_ua} - - response, response_headers = self.http_fetch(experimental_url, headers) - response_content = response.read() - response_code = response.code - if response_content is not None: - if fuzzy: - pattern = re.compile(control_result) - match = pattern.search(response_content) - if not match: - log.info("Fuzzy HTTP content comparison for experiment URL") - log.info("'%s'" % experimental_url) - log.info("does not match!") - return False, response_code, response_headers - else: - log.info("Fuzzy HTTP content comparison of experiment URL") - log.info("'%s'" % experimental_url) - log.info("and the expected control result yielded a match.") - return True, response_code, response_headers - else: - if str(response_content) != str(control_result): - log.info("HTTP content comparison of experiment URL") - log.info("'%s'" % experimental_url) - log.info("and the expected control result do not match.") - return False, response_code, response_headers - else: - return True, response_code, response_headers - else: - log.warn("HTTP connection appears to have failed.") - return False, False, False - - def http_status_code_match(self, experiment_code, control_code): - """ - Compare two HTTP status codes, returns True if they match. - """ - if int(experiment_code) != int(control_code): - return False - return True - - def http_status_code_no_match(self, experiment_code, control_code): - """ - Compare two HTTP status codes, returns True if they do not match. - """ - if self.http_status_code_match(experiment_code, control_code): - return False - return True - - def dns_resolve(self, hostname, nameserver=None): - """ - Resolves hostname(s) though nameserver to corresponding - address(es). hostname may be either a single hostname string, - or a list of strings. If nameserver is not given, use local - DNS resolver, and if that fails try using 8.8.8.8. - """ - log = self.logger - - if isinstance(hostname, str): - hostname = [hostname] - - if nameserver is not None: - res = resolver.Resolver(configure=False) - res.nameservers = [nameserver] - else: - res = resolver.Resolver() - - response = [] - answer = None - - for hn in hostname: - try: - answer = res.query(hn) - except resolver.NoNameservers: - res.nameservers = ['8.8.8.8'] - try: - answer = res.query(hn) - except resolver.NXDOMAIN: - log.info("DNS resolution for %s returned NXDOMAIN" % hn) - response.append('NXDOMAIN') - except resolver.NXDOMAIN: - log.info("DNS resolution for %s returned NXDOMAIN" % hn) - response.append('NXDOMAIN') - finally: - if answer: - for addr in answer: - response.append(addr.address) - return response - - def dns_resolve_match(self, experiment_hostname, control_address): - """ - Resolve experiment_hostname, and check to see that it returns - an experiment_address which matches the control_address. If - they match, returns True and experiment_address; otherwise - returns False and experiment_address. - """ - log = self.logger - - experiment_address = self.dns_resolve(experiment_hostname) - if experiment_address: - if len(set(experiment_address) & set([control_address])) > 0: - return True, experiment_address - else: - log.info("DNS comparison of control '%s' does not" % control_address) - log.info("match experiment response '%s'" % experiment_address) - return False, experiment_address - else: - log.debug("dns_resolve() for %s failed" % experiment_hostname) - return None, experiment_address - - def get_auth_nameservers(self, hostname): - """ - Many CPs set a nameserver to be used. Let's query that - nameserver for the authoritative nameservers of hostname. - - The equivalent of: - $ dig +short NS ooni.nu - """ - res = resolver.Resolver() - answer = res.query(hostname, 'NS') - auth_nameservers = [] - for auth in answer: - auth_nameservers.append(auth.to_text()) - return auth_nameservers - - def hostname_to_0x20(self, hostname): - """ - MaKEs yOur HOsTnaME lOoK LiKE THis. - - For more information, see: - D. Dagon, et. al. "Increased DNS Forgery Resistance - Through 0x20-Bit Encoding". Proc. CSS, 2008. - """ - hostname_0x20 = '' - for char in hostname: - l33t = random.choice(['caps', 'nocaps']) - if l33t == 'caps': - hostname_0x20 = hostname_0x20 + char.capitalize() - else: - hostname_0x20 = hostname_0x20 + char.lower() - return hostname_0x20 - - def check_0x20_to_auth_ns(self, hostname, sample_size=None): - """ - Resolve a 0x20 DNS request for hostname over hostname's - authoritative nameserver(s), and check to make sure that - the capitalization in the 0x20 request matches that of the - response. Also, check the serial numbers of the SOA (Start - of Authority) records on the authoritative nameservers to - make sure that they match. - - If sample_size is given, a random sample equal to that number - of authoritative nameservers will be queried; default is 5. - """ - log = self.logger - log.info("") - log.info("Testing random capitalization of DNS queries...") - log.info("Testing that Start of Authority serial numbers match...") - - auth_nameservers = self.get_auth_nameservers(hostname) - - if sample_size is None: - sample_size = 5 - resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers), - sample_size) - - querynames = [] - answernames = [] - serials = [] - - # Even when gevent monkey patching is on, the requests here - # are sent without being 0x20'd, so we need to 0x20 them. - hostname = self.hostname_to_0x20(hostname) - - for auth_ns in resolved_auth_ns: - res = resolver.Resolver(configure=False) - res.nameservers = [auth_ns] - try: - answer = res.query(hostname, 'SOA') - except resolver.Timeout: - continue - querynames.append(answer.qname.to_text()) - answernames.append(answer.rrset.name.to_text()) - for soa in answer: - serials.append(str(soa.serial)) - - if len(set(querynames).intersection(answernames)) == 1: - log.info("Capitalization in DNS queries and responses match.") - name_match = True - else: - log.info("The random capitalization '%s' used in" % hostname) - log.info("DNS queries to that hostname's authoritative") - log.info("nameservers does not match the capitalization in") - log.info("the response.") - name_match = False - - if len(set(serials)) == 1: - log.info("Start of Authority serial numbers all match.") - serial_match = True - else: - log.info("Some SOA serial numbers did not match the rest!") - serial_match = False - - ret = name_match, serial_match, querynames, answernames, serials - - if name_match and serial_match: - log.info("Your DNS queries do not appear to be tampered.") - return ret - elif name_match or serial_match: - log.info("Something is tampering with your DNS queries.") - return ret - elif not name_match and not serial_match: - log.info("Your DNS queries are definitely being tampered with.") - return ret - - def get_random_url_safe_string(self, length): - """ - Returns a random url-safe string of specified length, where - 0 < length <= 256. The returned string will always start with - an alphabetic character. - """ - if (length <= 0): - length = 1 - elif (length > 256): - length = 256 - - random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) - - while not random_ascii[:1].isalpha(): - random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) - - three_quarters = int((len(random_ascii)) * (3.0/4.0)) - random_string = random_ascii[:three_quarters] - return random_string - - def get_random_hostname(self, length=None): - """ - Returns a random hostname with SLD of specified length. If - length is unspecified, length=32 is used. - - These *should* all resolve to NXDOMAIN. If they actually - resolve to a box that isn't part of a captive portal that - would be rather interesting. - """ - log = self.logger - - if length is None: - length = 32 - - random_sld = self.get_random_url_safe_string(length) - - # if it doesn't start with a letter, chuck it. - while not random_sld[:1].isalpha(): - random_sld = self.get_random_url_safe_string(length) - - tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] - random_tld = urllib2.random.choice(tld_list) - random_hostname = random_sld + random_tld - return random_hostname - - def compare_random_hostnames(self, hostname_count=None, hostname_length=None): - """ - Get hostname_count number of random hostnames with SLD length - of hostname_length, and then attempt DNS resolution. If no - arguments are given, default to three hostnames of 32 bytes - each. These random hostnames *should* resolve to NXDOMAIN, - except in the case where a user is presented with a captive - portal and remains unauthenticated, in which case the captive - portal may return the address of the authentication page. - - If the cardinality of the intersection of the set of resolved - random hostnames and the single element control set - (['NXDOMAIN']) are equal to one, then DNS properly resolved. - - Returns true if only NXDOMAINs were returned, otherwise returns - False with the relative complement of the control set in the - response set. - """ - log = self.logger - - if hostname_count is None: - hostname_count = 3 - - log.info("Generating random hostnames...") - log.info("Resolving DNS for %d random hostnames..." % hostname_count) - - control = ['NXDOMAIN'] - responses = [] - - for x in range(hostname_count): - random_hostname = self.get_random_hostname(hostname_length) - response_match, response_address = self.dns_resolve_match(random_hostname, - control[0]) - for address in response_address: - if response_match is False: - log.info("Strangely, DNS resolution of the random hostname") - log.info("%s actually points to %s" - % (random_hostname, response_address)) - responses = responses + [address] - else: - responses = responses + [address] - - intersection = set(responses) & set(control) - relative_complement = set(responses) - set(control) - r = set(responses) - - if len(intersection) == 1: - log.info("All %d random hostnames properly resolved to NXDOMAIN." - % hostname_count) - return True, relative_complement - elif (len(intersection) == 1) and (len(r) > 1): - log.info("Something odd happened. Some random hostnames correctly") - log.info("resolved to NXDOMAIN, but several others resolved to") - log.info("to the following addresses: %s" % relative_complement) - return False, relative_complement - elif (len(intersection) == 0) and (len(r) == 1): - log.info("All random hostnames resolved to the IP address ") - log.info("'%s', which is indicative of a captive portal." % r) - return False, relative_complement - else: - log.debug("Apparently, pigs are flying on your network, 'cause a") - log.debug("bunch of hostnames made from 32-byte random strings") - log.debug("just magically resolved to a bunch of random addresses.") - log.debug("That is definitely highly improbable. In fact, my napkin") - log.debug("tells me that the probability of just one of those") - log.degug("hostnames resolving to an address is 1.68e-59, making") - log.debug("it nearly twice as unlikely as an MD5 hash collision.") - log.debug("Either someone is seriously messing with your network,") - log.debug("or else you are witnessing the impossible. %s" % r) - return False, relative_complement - - def google_dns_cp_test(self): - """ - Google Chrome resolves three 10-byte random hostnames. - """ - log = self.logger - subtest = "Google Chrome DNS-based" - - log.info("") - log.info("Running the Google Chrome DNS-based captive portal test...") - - gmatch, google_dns_result = self.compare_random_hostnames(3, 10) - - if gmatch: - log.info("Google Chrome DNS-based captive portal test did not") - log.info("detect a captive portal.") - return google_dns_result - else: - log.info("Google Chrome DNS-based captive portal test believes") - log.info("you are in a captive portal, or else something very") - log.info("odd is happening with your DNS.") - return google_dns_result - - def ms_dns_cp_test(self): - """ - Microsoft "phones home" to a server which will always resolve - to the same address. - """ - log = self.logger - subtest = "Microsoft NCSI DNS-based" - - log.info("") - log.info("Running the Microsoft NCSI DNS-based captive portal") - log.info("test...") - - msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com", - "131.107.255.255") - if msmatch: - log.info("Microsoft NCSI DNS-based captive portal test did not") - log.info("detect a captive portal.") - return ms_dns_result - else: - log.info("Microsoft NCSI DNS-based captive portal test ") - log.info("believes you are in a captive portal.") - return ms_dns_result - - def run_vendor_dns_tests(self): - """ - Run the vendor DNS tests. - """ - self.google_dns_cp_test() - self.ms_dns_cp_test() - - return - - def run_vendor_tests(self, *a, **kw): - """ - These are several vendor tests used to detect the presence of - a captive portal. Each test compares HTTP status code and - content to the control results and has its own User-Agent - string, in order to emulate the test as it would occur on the - device it was intended for. Vendor tests are defined in the - format: - [exp_url, ctrl_result, ctrl_code, ua, test_name] - """ - - vendor_tests = [['http://www.apple.com/library/test/success.html', - 'Success', - '200', - 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3', - 'Apple HTTP Captive Portal'], - ['http://tools.ietf.org/html/draft-nottingham-http-portal-02', - '428 Network Authentication Required', - '428', - 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0', - 'W3 Captive Portal'], - ['http://www.msftncsi.com/ncsi.txt', - 'Microsoft NCSI', - '200', - 'Microsoft NCSI', - 'MS HTTP Captive Portal',]] - - cm = self.http_content_match_fuzzy_opt - sm = self.http_status_code_match - snm = self.http_status_code_no_match - log = self.logger - - def compare_content(status_func, fuzzy, experiment_url, control_result, - control_code, headers, test_name): - log.info("") - log.info("Running the %s test..." % test_name) - - content_match, experiment_code, experiment_headers = cm(experiment_url, - control_result, - headers, fuzzy) - status_match = status_func(experiment_code, control_code) - - if status_match and content_match: - log.info("The %s test was unable to detect" % test_name) - log.info("a captive portal.") - else: - log.info("The %s test shows that your network" % test_name) - log.info("is filtered.") - - for vt in vendor_tests: - experiment_url = vt[0] - control_result = vt[1] - control_code = vt[2] - headers = {'User-Agent': vt[3]} - test_name = vt[4] - - args = (experiment_url, control_result, control_code, headers, test_name) - - if test_name == "MS HTTP Captive Portal": - compare_content(sm, False, *args) - - elif test_name == "Apple HTTP Captive Portal": - compare_content(sm, True, *args) - - elif test_name == "W3 Captive Portal": - compare_content(snm, True, *args) - - else: - log.warn("Ooni is trying to run an undefined CP vendor test.") - - def experiment(self, *a, **kw): - """ - Compares the content and status code of the HTTP response for - experiment_url with the control_result and control_code - respectively. If the status codes match, but the experimental - content and control_result do not match, fuzzy matching is enabled - to determine if the control_result is at least included somewhere - in the experimental content. Returns True if matches are found, - and False if otherwise. - """ - if (os.path.isfile(os.path.join(self.config.main.assetdir, - self.config.tests.captive_portal))): - kw['data'].append(None) - kw['data'].append('user-defined') - - experiment_url = kw['data'][0] - control_result = kw['data'][1] - control_code = kw['data'][2] - ua = kw['data'][3] - test_name = kw['data'][4] - - cm = self.http_content_match_fuzzy_opt - sm = self.http_status_code_match - snm = self.http_status_code_no_match - - log = self.logger - - if test_name == "user-defined": - log.info("Running %s test for '%s'..." % (test_name, experiment_url)) - content_match, experiment_code, experiment_headers = cm(experiment_url, - control_result) - status_match = sm(experiment_code, control_code) - if status_match and content_match: - log.info("The %s test for '%s'" % (test_name, experiment_url)) - log.info("was unable to detect a captive portal.") - return True, test_name - elif status_match and not content_match: - log.info("Retrying '%s' with fuzzy match enabled." - % experiment_url) - fuzzy_match, experiment_code, experiment_headers = cm(experiment_url, - control_result, - fuzzy=True) - if fuzzy_match: - return True, test_name - else: - log.info("Found modified content on '%s'," % experiment_url) - log.info("which could indicate a captive portal.") - - return False, test_name - else: - log.info("The content comparison test for ") - log.info("'%s'" % experiment_url) - log.info("shows that your HTTP traffic is filtered.") - return False, test_name - - else: - log.warn("Ooni is trying to run an undefined captive portal test.") - return False, test_name - - -def run(ooni): - """ - Runs the CaptivePortal(Test). - - CONFIG OPTIONS - -------------- - - If "do_captive_portal_vendor_tests" is set to "true", then vendor - specific captive portal HTTP-based tests will be run. - - If "do_captive_portal_dns_tests" is set to "true", then vendor - specific captive portal DNS-based tests will be run. - - If "check_dns_requests" is set to "true", then Ooni-probe will - attempt to check that your DNS requests are not being tampered with - by a captive portal. - - If "captive_portal" = "yourfilename.txt", then user-specified tests - will be run. - - Any combination of the above tests can be run. - """ - config = ooni.config - log = ooni.logger - - assets = [] - if (os.path.isfile(os.path.join(config.main.assetdir, - config.tests.captive_portal))): - assets.append(CaptivePortalAsset(os.path.join(config.main.assetdir, - config.tests.captive_portal))) - - captiveportal = CaptivePortal(ooni) - log.info("Starting captive portal test...") - captiveportal.run(assets, {'index': 1}) - - if config.tests.do_captive_portal_vendor_tests: - log.info("") - log.info("Running vendor tests...") - captiveportal.run_vendor_tests() - - if config.tests.do_captive_portal_vendor_dns_tests: - log.info("") - log.info("Running vendor DNS-based tests...") - captiveportal.run_vendor_dns_tests() - - if config.tests.check_dns_requests: - log.info("") - log.info("Checking that DNS requests are not being tampered...") - captiveportal.check_0x20_to_auth_ns('ooni.nu') - - log.info("") - log.info("Captive portal test finished!")

1 0

[ooni-probe/master] Merge branch 'master' into captiveportal
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 45cc771b7e240fc8ba411670d3533593e3165195 Merge: c85c627 3c632b4 Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 16:40:19 2012 +0200 Merge branch 'master' into captiveportal Conflicts: ooni/plugins/captiveportal.py ooni/plugins/captiveportal.py | 413 +++++++++++++++++++---------------------- 1 files changed, 190 insertions(+), 223 deletions(-) diff --cc ooni/plugins/captiveportal.py index ffa86c4,9b7bee3..f463fe4 --- a/ooni/plugins/captiveportal.py +++ b/ooni/plugins/captiveportal.py @@@ -138,17 -138,16 +138,14 @@@ class CaptivePortal(OONITest) or a list of strings. If nameserver is not given, use local DNS resolver, and if that fails try using 8.8.8.8. """ - log = self.logger -- if isinstance(hostname, str): hostname = [hostname] - + if nameserver is not None: res = resolver.Resolver(configure=False) res.nameservers = [nameserver] else: res = resolver.Resolver() - - response = [] answer = None @@@ -191,7 -188,7 +186,6 @@@ else: log.debug("dns_resolve() for %s failed" % experiment_hostname) return None, experiment_address - - def get_auth_nameservers(self, hostname): """ Many CPs set a nameserver to be used. Let's query that @@@ -210,9 -207,9 +204,8 @@@ def hostname_to_0x20(self, hostname): """ MaKEs yOur HOsTnaME lOoK LiKE THis. -- - For more information, see: - D. Dagon, et. al. "Increased DNS Forgery Resistance + For more information, see: + D. Dagon, et. al. "Increased DNS Forgery Resistance Through 0x20-Bit Encoding". Proc. CSS, 2008. """ hostname_0x20 = '' @@@ -255,7 -251,7 +247,6 @@@ # Even when gevent monkey patching is on, the requests here # are sent without being 0x20'd, so we need to 0x20 them. hostname = self.hostname_to_0x20(hostname) - - for auth_ns in resolved_auth_ns: res = resolver.Resolver(configure=False) res.nameservers = [auth_ns] @@@ -309,7 -305,7 +300,6 @@@ length = 256 random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) - - while not random_ascii[:1].isalpha(): random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) @@@ -336,7 -330,7 +324,6 @@@ # if it doesn't start with a letter, chuck it. while not random_sld[:1].isalpha(): random_sld = self.get_random_url_safe_string(length) - - tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] random_tld = urllib2.random.choice(tld_list) random_hostname = random_sld + random_tld

1 0

[ooni-probe/master] Port Isis's captive portal test to the new framework.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 3c632b495d15cfeb77577e6f9153f4915453cbfc Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 12:41:49 2012 +0200 Port Isis's captive portal test to the new framework. --- ooni/plugins/captiveportal.py | 625 +++++++++++++++++++++++++++++++++++++++++ 1 files changed, 625 insertions(+), 0 deletions(-) diff --git a/ooni/plugins/captiveportal.py b/ooni/plugins/captiveportal.py new file mode 100644 index 0000000..9b7bee3 --- /dev/null +++ b/ooni/plugins/captiveportal.py @@ -0,0 +1,625 @@ +# -*- coding: utf-8 -*- +""" + captiveportal + ************* + + This test is a collection of tests to detect the presence of a + captive portal. Code is taken, in part, from the old ooni-probe, + which was written by Jacob Appelbaum and Arturo Filastò. + + :copyright: (c) 2012 Isis Lovecruft + :license: see LICENSE for more details +""" +import base64 +import os +import random +import re +import string +import urllib2 +from urlparse import urlparse + +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, OONITest +from ooni.protocols import http +from ooni import log + +try: + from dns import resolver +except ImportError: + print "The dnspython module was not found. https://crate.io/packages/dnspython/" + +__plugoo__ = "captiveportal" +__desc__ = "Captive portal detection test" + +class CaptivePortalArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index'], + ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'], + ['user-agent', 'u', random.choice(http.useragents), + 'User agent for HTTP requests'] + ] + +class CaptivePortal(OONITest): + """ + Compares content and status codes of HTTP responses, and attempts + to determine if content has been altered. + """ + + implements(IPlugin, ITest) + + shortName = "captivep" + description = "Captive Portal Test" + requirements = None + options = CaptivePortalArgs + # Tells this to be blocking. + blocking = True + + def http_fetch(self, url, headers={}): + """ + Parses an HTTP url, fetches it, and returns a urllib2 response + object. + """ + url = urlparse(url).geturl() + request = urllib2.Request(url, None, headers) + response = urllib2.urlopen(request) + response_headers = dict(response.headers) + return response, response_headers + + def http_content_match_fuzzy_opt(self, experimental_url, control_result, + headers=None, fuzzy=False): + """ + Makes an HTTP request on port 80 for experimental_url, then + compares the response_content of experimental_url with the + control_result. Optionally, if the fuzzy parameter is set to + True, the response_content is compared with a regex of the + control_result. If the response_content from the + experimental_url and the control_result match, returns True + with the HTTP status code and headers; False, status code, and + headers if otherwise. + """ + + if headers is None: + default_ua = self.local_options['user-agent'] + headers = {'User-Agent': default_ua} + + response, response_headers = self.http_fetch(experimental_url, headers) + response_content = response.read() + response_code = response.code + if response_content is not None: + if fuzzy: + pattern = re.compile(control_result) + match = pattern.search(response_content) + if not match: + log.msg("Fuzzy HTTP content comparison for experiment URL") + log.msg("'%s'" % experimental_url) + log.msg("does not match!") + return False, response_code, response_headers + else: + log.msg("Fuzzy HTTP content comparison of experiment URL") + log.msg("'%s'" % experimental_url) + log.msg("and the expected control result yielded a match.") + return True, response_code, response_headers + else: + if str(response_content) != str(control_result): + log.msg("HTTP content comparison of experiment URL") + log.msg("'%s'" % experimental_url) + log.msg("and the expected control result do not match.") + return False, response_code, response_headers + else: + return True, response_code, response_headers + else: + log.warn("HTTP connection appears to have failed.") + return False, False, False + + def http_status_code_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they match. + """ + if int(experiment_code) != int(control_code): + return False + return True + + def http_status_code_no_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they do not match. + """ + if self.http_status_code_match(experiment_code, control_code): + return False + return True + + def dns_resolve(self, hostname, nameserver=None): + """ + Resolves hostname(s) though nameserver to corresponding + address(es). hostname may be either a single hostname string, + or a list of strings. If nameserver is not given, use local + DNS resolver, and if that fails try using 8.8.8.8. + """ + + if isinstance(hostname, str): + hostname = [hostname] + + if nameserver is not None: + res = resolver.Resolver(configure=False) + res.nameservers = [nameserver] + else: + res = resolver.Resolver() + + response = [] + answer = None + + for hn in hostname: + try: + answer = res.query(hn) + except resolver.NoNameservers: + res.nameservers = ['8.8.8.8'] + try: + answer = res.query(hn) + except resolver.NXDOMAIN: + log.msg("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + except resolver.NXDOMAIN: + log.msg("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + finally: + if answer: + for addr in answer: + response.append(addr.address) + return response + + def dns_resolve_match(self, experiment_hostname, control_address): + """ + Resolve experiment_hostname, and check to see that it returns + an experiment_address which matches the control_address. If + they match, returns True and experiment_address; otherwise + returns False and experiment_address. + """ + experiment_address = self.dns_resolve(experiment_hostname) + if experiment_address: + if len(set(experiment_address) & set([control_address])) > 0: + return True, experiment_address + else: + log.msg("DNS comparison of control '%s' does not" % control_address) + log.msg("match experiment response '%s'" % experiment_address) + return False, experiment_address + else: + log.debug("dns_resolve() for %s failed" % experiment_hostname) + return None, experiment_address + + def get_auth_nameservers(self, hostname): + """ + Many CPs set a nameserver to be used. Let's query that + nameserver for the authoritative nameservers of hostname. + + The equivalent of: + $ dig +short NS ooni.nu + """ + res = resolver.Resolver() + answer = res.query(hostname, 'NS') + auth_nameservers = [] + for auth in answer: + auth_nameservers.append(auth.to_text()) + return auth_nameservers + + def hostname_to_0x20(self, hostname): + """ + MaKEs yOur HOsTnaME lOoK LiKE THis. + + For more information, see: + D. Dagon, et. al. "Increased DNS Forgery Resistance + Through 0x20-Bit Encoding". Proc. CSS, 2008. + """ + hostname_0x20 = '' + for char in hostname: + l33t = random.choice(['caps', 'nocaps']) + if l33t == 'caps': + hostname_0x20 = hostname_0x20 + char.capitalize() + else: + hostname_0x20 = hostname_0x20 + char.lower() + return hostname_0x20 + + def check_0x20_to_auth_ns(self, hostname, sample_size=None): + """ + Resolve a 0x20 DNS request for hostname over hostname's + authoritative nameserver(s), and check to make sure that + the capitalization in the 0x20 request matches that of the + response. Also, check the serial numbers of the SOA (Start + of Authority) records on the authoritative nameservers to + make sure that they match. + + If sample_size is given, a random sample equal to that number + of authoritative nameservers will be queried; default is 5. + """ + log.msg("") + log.msg("Testing random capitalization of DNS queries...") + log.msg("Testing that Start of Authority serial numbers match...") + + auth_nameservers = self.get_auth_nameservers(hostname) + + if sample_size is None: + sample_size = 5 + resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers), + sample_size) + + querynames = [] + answernames = [] + serials = [] + + # Even when gevent monkey patching is on, the requests here + # are sent without being 0x20'd, so we need to 0x20 them. + hostname = self.hostname_to_0x20(hostname) + + for auth_ns in resolved_auth_ns: + res = resolver.Resolver(configure=False) + res.nameservers = [auth_ns] + try: + answer = res.query(hostname, 'SOA') + except resolver.Timeout: + continue + querynames.append(answer.qname.to_text()) + answernames.append(answer.rrset.name.to_text()) + for soa in answer: + serials.append(str(soa.serial)) + + if len(set(querynames).intersection(answernames)) == 1: + log.msg("Capitalization in DNS queries and responses match.") + name_match = True + else: + log.msg("The random capitalization '%s' used in" % hostname) + log.msg("DNS queries to that hostname's authoritative") + log.msg("nameservers does not match the capitalization in") + log.msg("the response.") + name_match = False + + if len(set(serials)) == 1: + log.msg("Start of Authority serial numbers all match.") + serial_match = True + else: + log.msg("Some SOA serial numbers did not match the rest!") + serial_match = False + + ret = name_match, serial_match, querynames, answernames, serials + + if name_match and serial_match: + log.msg("Your DNS queries do not appear to be tampered.") + return ret + elif name_match or serial_match: + log.msg("Something is tampering with your DNS queries.") + return ret + elif not name_match and not serial_match: + log.msg("Your DNS queries are definitely being tampered with.") + return ret + + def get_random_url_safe_string(self, length): + """ + Returns a random url-safe string of specified length, where + 0 < length <= 256. The returned string will always start with + an alphabetic character. + """ + if (length <= 0): + length = 1 + elif (length > 256): + length = 256 + + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + while not random_ascii[:1].isalpha(): + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + three_quarters = int((len(random_ascii)) * (3.0/4.0)) + random_string = random_ascii[:three_quarters] + return random_string + + def get_random_hostname(self, length=None): + """ + Returns a random hostname with SLD of specified length. If + length is unspecified, length=32 is used. + + These *should* all resolve to NXDOMAIN. If they actually + resolve to a box that isn't part of a captive portal that + would be rather interesting. + """ + if length is None: + length = 32 + + random_sld = self.get_random_url_safe_string(length) + + # if it doesn't start with a letter, chuck it. + while not random_sld[:1].isalpha(): + random_sld = self.get_random_url_safe_string(length) + + tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] + random_tld = urllib2.random.choice(tld_list) + random_hostname = random_sld + random_tld + return random_hostname + + def compare_random_hostnames(self, hostname_count=None, hostname_length=None): + """ + Get hostname_count number of random hostnames with SLD length + of hostname_length, and then attempt DNS resolution. If no + arguments are given, default to three hostnames of 32 bytes + each. These random hostnames *should* resolve to NXDOMAIN, + except in the case where a user is presented with a captive + portal and remains unauthenticated, in which case the captive + portal may return the address of the authentication page. + + If the cardinality of the intersection of the set of resolved + random hostnames and the single element control set + (['NXDOMAIN']) are equal to one, then DNS properly resolved. + + Returns true if only NXDOMAINs were returned, otherwise returns + False with the relative complement of the control set in the + response set. + """ + if hostname_count is None: + hostname_count = 3 + + log.msg("Generating random hostnames...") + log.msg("Resolving DNS for %d random hostnames..." % hostname_count) + + control = ['NXDOMAIN'] + responses = [] + + for x in range(hostname_count): + random_hostname = self.get_random_hostname(hostname_length) + response_match, response_address = self.dns_resolve_match(random_hostname, + control[0]) + for address in response_address: + if response_match is False: + log.msg("Strangely, DNS resolution of the random hostname") + log.msg("%s actually points to %s" + % (random_hostname, response_address)) + responses = responses + [address] + else: + responses = responses + [address] + + intersection = set(responses) & set(control) + relative_complement = set(responses) - set(control) + r = set(responses) + + if len(intersection) == 1: + log.msg("All %d random hostnames properly resolved to NXDOMAIN." + % hostname_count) + return True, relative_complement + elif (len(intersection) == 1) and (len(r) > 1): + log.msg("Something odd happened. Some random hostnames correctly") + log.msg("resolved to NXDOMAIN, but several others resolved to") + log.msg("to the following addresses: %s" % relative_complement) + return False, relative_complement + elif (len(intersection) == 0) and (len(r) == 1): + log.msg("All random hostnames resolved to the IP address ") + log.msg("'%s', which is indicative of a captive portal." % r) + return False, relative_complement + else: + log.debug("Apparently, pigs are flying on your network, 'cause a") + log.debug("bunch of hostnames made from 32-byte random strings") + log.debug("just magically resolved to a bunch of random addresses.") + log.debug("That is definitely highly improbable. In fact, my napkin") + log.debug("tells me that the probability of just one of those") + log.degug("hostnames resolving to an address is 1.68e-59, making") + log.debug("it nearly twice as unlikely as an MD5 hash collision.") + log.debug("Either someone is seriously messing with your network,") + log.debug("or else you are witnessing the impossible. %s" % r) + return False, relative_complement + + def google_dns_cp_test(self): + """ + Google Chrome resolves three 10-byte random hostnames. + """ + subtest = "Google Chrome DNS-based" + + log.msg("") + log.msg("Running the Google Chrome DNS-based captive portal test...") + + gmatch, google_dns_result = self.compare_random_hostnames(3, 10) + + if gmatch: + log.msg("Google Chrome DNS-based captive portal test did not") + log.msg("detect a captive portal.") + return google_dns_result + else: + log.msg("Google Chrome DNS-based captive portal test believes") + log.msg("you are in a captive portal, or else something very") + log.msg("odd is happening with your DNS.") + return google_dns_result + + def ms_dns_cp_test(self): + """ + Microsoft "phones home" to a server which will always resolve + to the same address. + """ + subtest = "Microsoft NCSI DNS-based" + + log.msg("") + log.msg("Running the Microsoft NCSI DNS-based captive portal") + log.msg("test...") + + msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com", + "131.107.255.255") + if msmatch: + log.msg("Microsoft NCSI DNS-based captive portal test did not") + log.msg("detect a captive portal.") + return ms_dns_result + else: + log.msg("Microsoft NCSI DNS-based captive portal test ") + log.msg("believes you are in a captive portal.") + return ms_dns_result + + def run_vendor_dns_tests(self): + """ + Run the vendor DNS tests. + """ + report = {} + report['google_dns_cp'] = self.google_dns_cp_test() + report['ms_dns_cp'] = self.ms_dns_cp_test() + + return report + + def run_vendor_tests(self, *a, **kw): + """ + These are several vendor tests used to detect the presence of + a captive portal. Each test compares HTTP status code and + content to the control results and has its own User-Agent + string, in order to emulate the test as it would occur on the + device it was intended for. Vendor tests are defined in the + format: + [exp_url, ctrl_result, ctrl_code, ua, test_name] + """ + + vendor_tests = [['http://www.apple.com/library/test/success.html', + 'Success', + '200', + 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3', + 'Apple HTTP Captive Portal'], + ['http://tools.ietf.org/html/draft-nottingham-http-portal-02', + '428 Network Authentication Required', + '428', + 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0', + 'W3 Captive Portal'], + ['http://www.msftncsi.com/ncsi.txt', + 'Microsoft NCSI', + '200', + 'Microsoft NCSI', + 'MS HTTP Captive Portal',]] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + + def compare_content(status_func, fuzzy, experiment_url, control_result, + control_code, headers, test_name): + log.msg("") + log.msg("Running the %s test..." % test_name) + + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + headers, fuzzy) + status_match = status_func(experiment_code, control_code) + + if status_match and content_match: + log.msg("The %s test was unable to detect" % test_name) + log.msg("a captive portal.") + return True + else: + log.msg("The %s test shows that your network" % test_name) + log.msg("is filtered.") + return False + + result = [] + for vt in vendor_tests: + report = {} + report['vt'] = vt + + experiment_url = vt[0] + control_result = vt[1] + control_code = vt[2] + headers = {'User-Agent': vt[3]} + test_name = vt[4] + + args = (experiment_url, control_result, control_code, headers, test_name) + + if test_name == "MS HTTP Captive Portal": + report['result'] = compare_content(sm, False, *args) + + elif test_name == "Apple HTTP Captive Portal": + report['result'] = compare_content(sm, True, *args) + + elif test_name == "W3 Captive Portal": + report['result'] = compare_content(snm, True, *args) + + else: + log.warn("Ooni is trying to run an undefined CP vendor test.") + result.append(report) + return result + + def control(self, experiment_result, args): + """ + Compares the content and status code of the HTTP response for + experiment_url with the control_result and control_code + respectively. If the status codes match, but the experimental + content and control_result do not match, fuzzy matching is enabled + to determine if the control_result is at least included somewhere + in the experimental content. Returns True if matches are found, + and False if otherwise. + """ + experiment_url = self.local_options['experiment-url'] + control_result = 'XX' + control_code = 200 + ua = self.local_options['user-agent'] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + + log.msg("Running test for '%s'..." % experiment_url) + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result) + status_match = sm(experiment_code, control_code) + if status_match and content_match: + log.msg("The test for '%s'" % experiment_url) + log.msg("was unable to detect a captive portal.") + return experiment_result, True + + elif status_match and not content_match: + log.msg("Retrying '%s' with fuzzy match enabled." + % experiment_url) + fuzzy_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + fuzzy=True) + if fuzzy_match: + return experiment_result, True + else: + log.msg("Found modified content on '%s'," % experiment_url) + log.msg("which could indicate a captive portal.") + + return experiment_result, False + else: + log.msg("The content comparison test for ") + log.msg("'%s'" % experiment_url) + log.msg("shows that your HTTP traffic is filtered.") + return experiment_result, False + + def experiment(self, args): + """ + Runs the CaptivePortal(Test). + + CONFIG OPTIONS + -------------- + + If "do_captive_portal_vendor_tests" is set to "true", then vendor + specific captive portal HTTP-based tests will be run. + + If "do_captive_portal_dns_tests" is set to "true", then vendor + specific captive portal DNS-based tests will be run. + + If "check_dns_requests" is set to "true", then Ooni-probe will + attempt to check that your DNS requests are not being tampered with + by a captive portal. + + If "captive_portal" = "yourfilename.txt", then user-specified tests + will be run. + + Any combination of the above tests can be run. + """ + report = {} + + log.msg("") + log.msg("Running vendor tests...") + report['vendor_tests'] = self.run_vendor_tests() + + log.msg("") + log.msg("Running vendor DNS-based tests...") + report['vendor_dns_tests'] = self.run_vendor_dns_tests() + + log.msg("") + log.msg("Checking that DNS requests are not being tampered...") + report['check0x20'] = self.check_0x20_to_auth_ns('ooni.nu') + + log.msg("") + log.msg("Captive portal test finished!") + return report + +cp = CaptivePortal(None, None, None)

1 0

[ooni-probe/master] Fix the last conflicts, hopefully... argh...
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 793f013cf2ca50f415398677dd12ab59ae9c57b4 Merge: 45cc771 428e8e8 Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 16:41:48 2012 +0200 Fix the last conflicts, hopefully... argh... Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe Conflicts: ooni/plugins/captiveportal.py ooni/plugins/captiveportal.py | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-)

1 0

[metrics-tasks/master] Fail gracefully for old consensus
by karsten＠torproject.org 09 Jul '12

09 Jul '12

commit cc86e841928a7a85168143ff524b0779c0e5059a Author: Sathyanarayanan <gsathya.ceg(a)gmail.com> Date: Mon Jul 9 16:57:08 2012 +0300 Fail gracefully for old consensus --- task-6232/pyentropy.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 0ae0f25..23850d1 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -61,8 +61,8 @@ def run(file_name): totalExitBW += router.bandwidth if len(routers) <= 0: - print "Error: amount of routers must be > 0." - return; + print "Error: Old consensus file. Not able to parse." + return entropy, entropy_exit, entropy_guard = 0.0, 0.0, 0.0 for router in routers: @@ -92,4 +92,5 @@ if __name__ == "__main__": with open(sys.argv[2], 'w') as f: for file_name in os.listdir(sys.argv[1]): string = run(os.path.join(sys.argv[1], file_name)) - f.write("%s\n" % (string)) + if string: + f.write("%s\n" % (string))

1 0

[metrics-tasks/master] Merge remote-tracking branch 'gsathya/master'
by karsten＠torproject.org 09 Jul '12

09 Jul '12

commit b2d9e509b4c513aa2e253a550fd832fb9c86a94e Merge: b939e76 cc86e84 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 9 16:16:24 2012 +0200 Merge remote-tracking branch 'gsathya/master' task-6232/pyentropy.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-)

1 0

[metrics-tasks/master] Take out noisy warning.
by karsten＠torproject.org 09 Jul '12

09 Jul '12

commit 3d4e484df398a5fd395149f1fb87b3b702dd4a2c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 9 16:16:44 2012 +0200 Take out noisy warning. --- task-6232/pyentropy.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py index 23850d1..650423b 100644 --- a/task-6232/pyentropy.py +++ b/task-6232/pyentropy.py @@ -61,7 +61,6 @@ def run(file_name): totalExitBW += router.bandwidth if len(routers) <= 0: - print "Error: Old consensus file. Not able to parse." return entropy, entropy_exit, entropy_guard = 0.0, 0.0, 0.0

1 0

[ooni-probe/master] Remove dead code.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit bcf5e79efcc8273a424a6bc668e0b5d6b21f0b0a Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 12:01:12 2012 +0200 Remove dead code. --- ooni/plugoo/work.py | 81 --------------------------------------------------- 1 files changed, 0 insertions(+), 81 deletions(-) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index a3fb168..fc338d7 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -107,84 +107,3 @@ class WorkGenerator(object): self.idx += 1 return (ret, self.Test, self.idx) - def p_next(self): - """ - XXX This is not used for KISS sake. - """ - if self.end: - raise StopIteration - - if not self.assetGenerator: - self.end = True - return WorkUnit(None, self.assetNames, self.Test, self.idx) - - # Plank asset - p_asset = [] - for i in xrange(0, self.size): - try: - asset = self.assetGenerator.next() - p_asset.append(asset) - print p_asset - except StopIteration: - if self.asset_num > 1: - pass - self.end = True - break - - self.idx += 1 - return WorkUnit(p_asset, self.assetNames, self.Test, self.idx) - -class WorkUnit(object): - """ - XXX This is currently not implemented for KISS sake. - - This is an object responsible for completing WorkUnits it will - return its result in a deferred. - - The execution of a unit of work should be Atomic. - - Reporting to the OONI-net happens on completion of a Unit of Work. - - @Node node: This represents the node associated with the Work Unit - @Asset asset: This is the asset associated with the Work Unit - @Test test: This represents the Test to be with the specified assets - @ivar arguments: These are the extra attributes to be passsed to the Test - """ - - node = None - asset = None - test = None - arguments = None - - def __init__(self, asset, assetNames, test, idx): - self.asset = asset - if not asset: - self.assetGenerator = iter([1]) - else: - self.assetGenerator = iter(asset) - self.Test = test - self.assetNames = assetNames - self.idx = idx - - def __iter__(self): - return self - - def __repr__(self): - return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx) - - def serialize(self): - """ - Serialize this unit of work for RPC activity. - """ - return yaml.dump(self) - - def next(self): - """ - Launches the Unit of Work with the specified assets on the node. - """ - try: - asset = self.assetGenerator.next() - ret = self.Test.set_asset(asset) - return ret - except StopIteration: - raise StopIteration

1 0

[ooni-probe/master] Port Isis's captive portal test to the new framework.
by art＠torproject.org 09 Jul '12

09 Jul '12

commit 428e8e827e6f457aaf987be753b4f99c5127f750 Author: Arturo Filastò <art(a)torproject.org> Date: Mon Jul 9 12:41:49 2012 +0200 Port Isis's captive portal test to the new framework. --- ooni/plugins/captiveportal.py | 625 +++++++++++++++++++++++++++++++++++++++++ 1 files changed, 625 insertions(+), 0 deletions(-) diff --git a/ooni/plugins/captiveportal.py b/ooni/plugins/captiveportal.py new file mode 100644 index 0000000..9b7bee3 --- /dev/null +++ b/ooni/plugins/captiveportal.py @@ -0,0 +1,625 @@ +# -*- coding: utf-8 -*- +""" + captiveportal + ************* + + This test is a collection of tests to detect the presence of a + captive portal. Code is taken, in part, from the old ooni-probe, + which was written by Jacob Appelbaum and Arturo Filastò. + + :copyright: (c) 2012 Isis Lovecruft + :license: see LICENSE for more details +""" +import base64 +import os +import random +import re +import string +import urllib2 +from urlparse import urlparse + +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, OONITest +from ooni.protocols import http +from ooni import log + +try: + from dns import resolver +except ImportError: + print "The dnspython module was not found. https://crate.io/packages/dnspython/" + +__plugoo__ = "captiveportal" +__desc__ = "Captive portal detection test" + +class CaptivePortalArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file'], + ['resume', 'r', 0, 'Resume at this index'], + ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'], + ['user-agent', 'u', random.choice(http.useragents), + 'User agent for HTTP requests'] + ] + +class CaptivePortal(OONITest): + """ + Compares content and status codes of HTTP responses, and attempts + to determine if content has been altered. + """ + + implements(IPlugin, ITest) + + shortName = "captivep" + description = "Captive Portal Test" + requirements = None + options = CaptivePortalArgs + # Tells this to be blocking. + blocking = True + + def http_fetch(self, url, headers={}): + """ + Parses an HTTP url, fetches it, and returns a urllib2 response + object. + """ + url = urlparse(url).geturl() + request = urllib2.Request(url, None, headers) + response = urllib2.urlopen(request) + response_headers = dict(response.headers) + return response, response_headers + + def http_content_match_fuzzy_opt(self, experimental_url, control_result, + headers=None, fuzzy=False): + """ + Makes an HTTP request on port 80 for experimental_url, then + compares the response_content of experimental_url with the + control_result. Optionally, if the fuzzy parameter is set to + True, the response_content is compared with a regex of the + control_result. If the response_content from the + experimental_url and the control_result match, returns True + with the HTTP status code and headers; False, status code, and + headers if otherwise. + """ + + if headers is None: + default_ua = self.local_options['user-agent'] + headers = {'User-Agent': default_ua} + + response, response_headers = self.http_fetch(experimental_url, headers) + response_content = response.read() + response_code = response.code + if response_content is not None: + if fuzzy: + pattern = re.compile(control_result) + match = pattern.search(response_content) + if not match: + log.msg("Fuzzy HTTP content comparison for experiment URL") + log.msg("'%s'" % experimental_url) + log.msg("does not match!") + return False, response_code, response_headers + else: + log.msg("Fuzzy HTTP content comparison of experiment URL") + log.msg("'%s'" % experimental_url) + log.msg("and the expected control result yielded a match.") + return True, response_code, response_headers + else: + if str(response_content) != str(control_result): + log.msg("HTTP content comparison of experiment URL") + log.msg("'%s'" % experimental_url) + log.msg("and the expected control result do not match.") + return False, response_code, response_headers + else: + return True, response_code, response_headers + else: + log.warn("HTTP connection appears to have failed.") + return False, False, False + + def http_status_code_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they match. + """ + if int(experiment_code) != int(control_code): + return False + return True + + def http_status_code_no_match(self, experiment_code, control_code): + """ + Compare two HTTP status codes, returns True if they do not match. + """ + if self.http_status_code_match(experiment_code, control_code): + return False + return True + + def dns_resolve(self, hostname, nameserver=None): + """ + Resolves hostname(s) though nameserver to corresponding + address(es). hostname may be either a single hostname string, + or a list of strings. If nameserver is not given, use local + DNS resolver, and if that fails try using 8.8.8.8. + """ + + if isinstance(hostname, str): + hostname = [hostname] + + if nameserver is not None: + res = resolver.Resolver(configure=False) + res.nameservers = [nameserver] + else: + res = resolver.Resolver() + + response = [] + answer = None + + for hn in hostname: + try: + answer = res.query(hn) + except resolver.NoNameservers: + res.nameservers = ['8.8.8.8'] + try: + answer = res.query(hn) + except resolver.NXDOMAIN: + log.msg("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + except resolver.NXDOMAIN: + log.msg("DNS resolution for %s returned NXDOMAIN" % hn) + response.append('NXDOMAIN') + finally: + if answer: + for addr in answer: + response.append(addr.address) + return response + + def dns_resolve_match(self, experiment_hostname, control_address): + """ + Resolve experiment_hostname, and check to see that it returns + an experiment_address which matches the control_address. If + they match, returns True and experiment_address; otherwise + returns False and experiment_address. + """ + experiment_address = self.dns_resolve(experiment_hostname) + if experiment_address: + if len(set(experiment_address) & set([control_address])) > 0: + return True, experiment_address + else: + log.msg("DNS comparison of control '%s' does not" % control_address) + log.msg("match experiment response '%s'" % experiment_address) + return False, experiment_address + else: + log.debug("dns_resolve() for %s failed" % experiment_hostname) + return None, experiment_address + + def get_auth_nameservers(self, hostname): + """ + Many CPs set a nameserver to be used. Let's query that + nameserver for the authoritative nameservers of hostname. + + The equivalent of: + $ dig +short NS ooni.nu + """ + res = resolver.Resolver() + answer = res.query(hostname, 'NS') + auth_nameservers = [] + for auth in answer: + auth_nameservers.append(auth.to_text()) + return auth_nameservers + + def hostname_to_0x20(self, hostname): + """ + MaKEs yOur HOsTnaME lOoK LiKE THis. + + For more information, see: + D. Dagon, et. al. "Increased DNS Forgery Resistance + Through 0x20-Bit Encoding". Proc. CSS, 2008. + """ + hostname_0x20 = '' + for char in hostname: + l33t = random.choice(['caps', 'nocaps']) + if l33t == 'caps': + hostname_0x20 = hostname_0x20 + char.capitalize() + else: + hostname_0x20 = hostname_0x20 + char.lower() + return hostname_0x20 + + def check_0x20_to_auth_ns(self, hostname, sample_size=None): + """ + Resolve a 0x20 DNS request for hostname over hostname's + authoritative nameserver(s), and check to make sure that + the capitalization in the 0x20 request matches that of the + response. Also, check the serial numbers of the SOA (Start + of Authority) records on the authoritative nameservers to + make sure that they match. + + If sample_size is given, a random sample equal to that number + of authoritative nameservers will be queried; default is 5. + """ + log.msg("") + log.msg("Testing random capitalization of DNS queries...") + log.msg("Testing that Start of Authority serial numbers match...") + + auth_nameservers = self.get_auth_nameservers(hostname) + + if sample_size is None: + sample_size = 5 + resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers), + sample_size) + + querynames = [] + answernames = [] + serials = [] + + # Even when gevent monkey patching is on, the requests here + # are sent without being 0x20'd, so we need to 0x20 them. + hostname = self.hostname_to_0x20(hostname) + + for auth_ns in resolved_auth_ns: + res = resolver.Resolver(configure=False) + res.nameservers = [auth_ns] + try: + answer = res.query(hostname, 'SOA') + except resolver.Timeout: + continue + querynames.append(answer.qname.to_text()) + answernames.append(answer.rrset.name.to_text()) + for soa in answer: + serials.append(str(soa.serial)) + + if len(set(querynames).intersection(answernames)) == 1: + log.msg("Capitalization in DNS queries and responses match.") + name_match = True + else: + log.msg("The random capitalization '%s' used in" % hostname) + log.msg("DNS queries to that hostname's authoritative") + log.msg("nameservers does not match the capitalization in") + log.msg("the response.") + name_match = False + + if len(set(serials)) == 1: + log.msg("Start of Authority serial numbers all match.") + serial_match = True + else: + log.msg("Some SOA serial numbers did not match the rest!") + serial_match = False + + ret = name_match, serial_match, querynames, answernames, serials + + if name_match and serial_match: + log.msg("Your DNS queries do not appear to be tampered.") + return ret + elif name_match or serial_match: + log.msg("Something is tampering with your DNS queries.") + return ret + elif not name_match and not serial_match: + log.msg("Your DNS queries are definitely being tampered with.") + return ret + + def get_random_url_safe_string(self, length): + """ + Returns a random url-safe string of specified length, where + 0 < length <= 256. The returned string will always start with + an alphabetic character. + """ + if (length <= 0): + length = 1 + elif (length > 256): + length = 256 + + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + while not random_ascii[:1].isalpha(): + random_ascii = base64.urlsafe_b64encode(os.urandom(int(length))) + + three_quarters = int((len(random_ascii)) * (3.0/4.0)) + random_string = random_ascii[:three_quarters] + return random_string + + def get_random_hostname(self, length=None): + """ + Returns a random hostname with SLD of specified length. If + length is unspecified, length=32 is used. + + These *should* all resolve to NXDOMAIN. If they actually + resolve to a box that isn't part of a captive portal that + would be rather interesting. + """ + if length is None: + length = 32 + + random_sld = self.get_random_url_safe_string(length) + + # if it doesn't start with a letter, chuck it. + while not random_sld[:1].isalpha(): + random_sld = self.get_random_url_safe_string(length) + + tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] + random_tld = urllib2.random.choice(tld_list) + random_hostname = random_sld + random_tld + return random_hostname + + def compare_random_hostnames(self, hostname_count=None, hostname_length=None): + """ + Get hostname_count number of random hostnames with SLD length + of hostname_length, and then attempt DNS resolution. If no + arguments are given, default to three hostnames of 32 bytes + each. These random hostnames *should* resolve to NXDOMAIN, + except in the case where a user is presented with a captive + portal and remains unauthenticated, in which case the captive + portal may return the address of the authentication page. + + If the cardinality of the intersection of the set of resolved + random hostnames and the single element control set + (['NXDOMAIN']) are equal to one, then DNS properly resolved. + + Returns true if only NXDOMAINs were returned, otherwise returns + False with the relative complement of the control set in the + response set. + """ + if hostname_count is None: + hostname_count = 3 + + log.msg("Generating random hostnames...") + log.msg("Resolving DNS for %d random hostnames..." % hostname_count) + + control = ['NXDOMAIN'] + responses = [] + + for x in range(hostname_count): + random_hostname = self.get_random_hostname(hostname_length) + response_match, response_address = self.dns_resolve_match(random_hostname, + control[0]) + for address in response_address: + if response_match is False: + log.msg("Strangely, DNS resolution of the random hostname") + log.msg("%s actually points to %s" + % (random_hostname, response_address)) + responses = responses + [address] + else: + responses = responses + [address] + + intersection = set(responses) & set(control) + relative_complement = set(responses) - set(control) + r = set(responses) + + if len(intersection) == 1: + log.msg("All %d random hostnames properly resolved to NXDOMAIN." + % hostname_count) + return True, relative_complement + elif (len(intersection) == 1) and (len(r) > 1): + log.msg("Something odd happened. Some random hostnames correctly") + log.msg("resolved to NXDOMAIN, but several others resolved to") + log.msg("to the following addresses: %s" % relative_complement) + return False, relative_complement + elif (len(intersection) == 0) and (len(r) == 1): + log.msg("All random hostnames resolved to the IP address ") + log.msg("'%s', which is indicative of a captive portal." % r) + return False, relative_complement + else: + log.debug("Apparently, pigs are flying on your network, 'cause a") + log.debug("bunch of hostnames made from 32-byte random strings") + log.debug("just magically resolved to a bunch of random addresses.") + log.debug("That is definitely highly improbable. In fact, my napkin") + log.debug("tells me that the probability of just one of those") + log.degug("hostnames resolving to an address is 1.68e-59, making") + log.debug("it nearly twice as unlikely as an MD5 hash collision.") + log.debug("Either someone is seriously messing with your network,") + log.debug("or else you are witnessing the impossible. %s" % r) + return False, relative_complement + + def google_dns_cp_test(self): + """ + Google Chrome resolves three 10-byte random hostnames. + """ + subtest = "Google Chrome DNS-based" + + log.msg("") + log.msg("Running the Google Chrome DNS-based captive portal test...") + + gmatch, google_dns_result = self.compare_random_hostnames(3, 10) + + if gmatch: + log.msg("Google Chrome DNS-based captive portal test did not") + log.msg("detect a captive portal.") + return google_dns_result + else: + log.msg("Google Chrome DNS-based captive portal test believes") + log.msg("you are in a captive portal, or else something very") + log.msg("odd is happening with your DNS.") + return google_dns_result + + def ms_dns_cp_test(self): + """ + Microsoft "phones home" to a server which will always resolve + to the same address. + """ + subtest = "Microsoft NCSI DNS-based" + + log.msg("") + log.msg("Running the Microsoft NCSI DNS-based captive portal") + log.msg("test...") + + msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com", + "131.107.255.255") + if msmatch: + log.msg("Microsoft NCSI DNS-based captive portal test did not") + log.msg("detect a captive portal.") + return ms_dns_result + else: + log.msg("Microsoft NCSI DNS-based captive portal test ") + log.msg("believes you are in a captive portal.") + return ms_dns_result + + def run_vendor_dns_tests(self): + """ + Run the vendor DNS tests. + """ + report = {} + report['google_dns_cp'] = self.google_dns_cp_test() + report['ms_dns_cp'] = self.ms_dns_cp_test() + + return report + + def run_vendor_tests(self, *a, **kw): + """ + These are several vendor tests used to detect the presence of + a captive portal. Each test compares HTTP status code and + content to the control results and has its own User-Agent + string, in order to emulate the test as it would occur on the + device it was intended for. Vendor tests are defined in the + format: + [exp_url, ctrl_result, ctrl_code, ua, test_name] + """ + + vendor_tests = [['http://www.apple.com/library/test/success.html', + 'Success', + '200', + 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3', + 'Apple HTTP Captive Portal'], + ['http://tools.ietf.org/html/draft-nottingham-http-portal-02', + '428 Network Authentication Required', + '428', + 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0', + 'W3 Captive Portal'], + ['http://www.msftncsi.com/ncsi.txt', + 'Microsoft NCSI', + '200', + 'Microsoft NCSI', + 'MS HTTP Captive Portal',]] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + + def compare_content(status_func, fuzzy, experiment_url, control_result, + control_code, headers, test_name): + log.msg("") + log.msg("Running the %s test..." % test_name) + + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + headers, fuzzy) + status_match = status_func(experiment_code, control_code) + + if status_match and content_match: + log.msg("The %s test was unable to detect" % test_name) + log.msg("a captive portal.") + return True + else: + log.msg("The %s test shows that your network" % test_name) + log.msg("is filtered.") + return False + + result = [] + for vt in vendor_tests: + report = {} + report['vt'] = vt + + experiment_url = vt[0] + control_result = vt[1] + control_code = vt[2] + headers = {'User-Agent': vt[3]} + test_name = vt[4] + + args = (experiment_url, control_result, control_code, headers, test_name) + + if test_name == "MS HTTP Captive Portal": + report['result'] = compare_content(sm, False, *args) + + elif test_name == "Apple HTTP Captive Portal": + report['result'] = compare_content(sm, True, *args) + + elif test_name == "W3 Captive Portal": + report['result'] = compare_content(snm, True, *args) + + else: + log.warn("Ooni is trying to run an undefined CP vendor test.") + result.append(report) + return result + + def control(self, experiment_result, args): + """ + Compares the content and status code of the HTTP response for + experiment_url with the control_result and control_code + respectively. If the status codes match, but the experimental + content and control_result do not match, fuzzy matching is enabled + to determine if the control_result is at least included somewhere + in the experimental content. Returns True if matches are found, + and False if otherwise. + """ + experiment_url = self.local_options['experiment-url'] + control_result = 'XX' + control_code = 200 + ua = self.local_options['user-agent'] + + cm = self.http_content_match_fuzzy_opt + sm = self.http_status_code_match + snm = self.http_status_code_no_match + + log.msg("Running test for '%s'..." % experiment_url) + content_match, experiment_code, experiment_headers = cm(experiment_url, + control_result) + status_match = sm(experiment_code, control_code) + if status_match and content_match: + log.msg("The test for '%s'" % experiment_url) + log.msg("was unable to detect a captive portal.") + return experiment_result, True + + elif status_match and not content_match: + log.msg("Retrying '%s' with fuzzy match enabled." + % experiment_url) + fuzzy_match, experiment_code, experiment_headers = cm(experiment_url, + control_result, + fuzzy=True) + if fuzzy_match: + return experiment_result, True + else: + log.msg("Found modified content on '%s'," % experiment_url) + log.msg("which could indicate a captive portal.") + + return experiment_result, False + else: + log.msg("The content comparison test for ") + log.msg("'%s'" % experiment_url) + log.msg("shows that your HTTP traffic is filtered.") + return experiment_result, False + + def experiment(self, args): + """ + Runs the CaptivePortal(Test). + + CONFIG OPTIONS + -------------- + + If "do_captive_portal_vendor_tests" is set to "true", then vendor + specific captive portal HTTP-based tests will be run. + + If "do_captive_portal_dns_tests" is set to "true", then vendor + specific captive portal DNS-based tests will be run. + + If "check_dns_requests" is set to "true", then Ooni-probe will + attempt to check that your DNS requests are not being tampered with + by a captive portal. + + If "captive_portal" = "yourfilename.txt", then user-specified tests + will be run. + + Any combination of the above tests can be run. + """ + report = {} + + log.msg("") + log.msg("Running vendor tests...") + report['vendor_tests'] = self.run_vendor_tests() + + log.msg("") + log.msg("Running vendor DNS-based tests...") + report['vendor_dns_tests'] = self.run_vendor_dns_tests() + + log.msg("") + log.msg("Checking that DNS requests are not being tampered...") + report['check0x20'] = self.check_0x20_to_auth_ns('ooni.nu') + + log.msg("") + log.msg("Captive portal test finished!") + return report + +cp = CaptivePortal(None, None, None)

1 0

← Newer
1
...
69
70
71
72
73
74
75
...
95
Older →

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits July 2012