tor-commits
Threads by month
- ----- 2025 -----
- May
- April
- March
- February
- January
- ----- 2024 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2023 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2022 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2021 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2020 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2019 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2018 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2017 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2016 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2015 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2014 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2013 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2012 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2011 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
July 2012
- 14 participants
- 949 discussions

[ooni-probe/master] Merge branch 'captiveportal' of https://github.com/isislovecruft/ooni-probe into captiveportal
by art@torproject.org 09 Jul '12
by art@torproject.org 09 Jul '12
09 Jul '12
commit e3011887ec87dfe5bca3667bc6cde0f7096a6172
Merge: bcf5e79 9313c75
Author: Arturo Filastò <art(a)torproject.org>
Date: Mon Jul 9 16:29:36 2012 +0200
Merge branch 'captiveportal' of https://github.com/isislovecruft/ooni-probe into captiveportal
Conflicts:
ooniprobe.py
plugoo/tests.py
TODO | 2 +
assets/captive_portal_tests.txt | 4 +
ooni/ooni-probe.conf | 23 ++
tests/captiveportal.py | 651 ++++++++++++…
[View More]+++++++++++++++++++++++++++
4 files changed, 680 insertions(+), 0 deletions(-)
diff --cc ooni/ooni-probe.conf
index d95e410,0000000..0c68fe0
mode 100644,000000..100644
--- a/ooni/ooni-probe.conf
+++ b/ooni/ooni-probe.conf
@@@ -1,77 -1,0 +1,100 @@@
+# ooni-probe
+#
+# These are the global configuration parameters necessary to
+# make ooni-probe work
+[main]
+reportdir = reports/
+logfile = ooniprobe.log
+assetdir = assets/
+testdir = oonitests/
+
+loglevel = DEBUG
+consoleloglevel = DEBUG
+proxyaddress = 127.0.0.1:9050
+
+# The following configurations are for searching for PlanetLab
+# nodes, adding them to a slice, and PlanetLab general API
+# authentication:
+pl_username = yourusername
+pl_password = yourpassword
+
+# These are configurations specific to the tests that should be
+# run by ooni-probe
+[tests]
+run = dnstamper
+### DNS testing related config parameters
+
+# This is the list of hostnames that must be looked up
+dns_experiment = top-1m.txt
+
+# This is the dns servers to be tested
+dns_experiment_dns = dns_servers.txt
+
+# This is the control known good DNS server
+dns_control_server = 91.191.136.152
+
+# Specify whether the dnstamper test should attempt to remove
+# GeoIP-based false positives by doing a reverse DNS resolve
+# on positive results.
+dns_reverse_lookup = true
+
++### captiveportal testing configuration parameters
++
++# This is an optional list of user defined captive portal tests,
++# one per line, with each line in the format:
++# experiment_url, control_result, control_code
++# where experiment_url is the test page to retrieve,
++# control_result is some unique text found on the test page,
++# and control_code is the expected HTTP status code.
++captive_portal = captive_portal_tests.txt
++
++# The default User Agent that ooni-probe should send for
++# HTTP requests (pretend we're a Windows box running FF10):
++default_ua = Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.2) Gecko/20100101 Firefox/10.0.2
++
++# Enable vendor tests for captive portals:
++do_captive_portal_vendor_tests = true
++
++# Enable DNS-based vendor tests for captive portals:
++do_captive_portal_vendor_dns_tests = true
++
++# Enable checking of DNS requests for tampering:
++check_dns_requests = true
++
+### traceroute testing related config parameters
+
+# This is the list of ips to traceroute to
+traceroute = example_exp_list.txt
+
+# This is the list of ports that should be used
+# src_x,src_y,src_z|dst_x,dst_y,dst_z
+traceroute_ports = 0,53,80,123,443|0,53,80,123,443
+
+# The protocol to be used in the scan
+traceroute_proto = UDP, TCP, ICMP
+
+### keyword injection related tests
+
+# List of keywords
+keywords = keywordlist.txt
+
+# hosts
+keywords_hosts = hostslist.txt
+
+# Methods to be used for testing
+keyword_method = http,telnet
+
+### Tor bridge testing
+
+tor_bridges = bridgetests.txt
+tor_bridges_timeout = 40
+
+[report]
+file = report.log
+timestamp = true
+#ssh = 127.0.0.1:22
+#ssh_user = theusername
+#ssh_password = thepassword
+#ssh_keyfile = ~/.ssh/mykey_rsa
+#ssh_rpath = ~/ooni-probe/
+#tcp = "127.0.0.1:9088"
[View Less]
1
0

[ooni-probe/master] Move captive portal tests to proper subdirectories
by art@torproject.org 09 Jul '12
by art@torproject.org 09 Jul '12
09 Jul '12
commit c85c627a7d12ccfeda7010fed6d88a58241e5978
Author: Arturo Filastò <art(a)torproject.org>
Date: Mon Jul 9 16:35:13 2012 +0200
Move captive portal tests to proper subdirectories
---
assets/captive_portal_tests.txt | 4 -
ooni/assets/captive_portal_tests.txt | 4 +
ooni/plugins/captiveportal.py | 651 ++++++++++++++++++++++++++++++++++
tests/captiveportal.py | 651 ----------------------------------
4 files changed, 655 insertions(+), 655 …
[View More]deletions(-)
diff --git a/assets/captive_portal_tests.txt b/assets/captive_portal_tests.txt
deleted file mode 100644
index 7007411..0000000
--- a/assets/captive_portal_tests.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-
-http://ooni.nu, Open Observatory of Network Interference, 200
-https://w2.eff.org/Censorship/Internet_censorship_bills/barlow_0296.declaration, let us now take our leave of them, 200
-https://www.torproject.org, Defend yourself against network surveillance and traffic analysis, 200
diff --git a/ooni/assets/captive_portal_tests.txt b/ooni/assets/captive_portal_tests.txt
new file mode 100644
index 0000000..7007411
--- /dev/null
+++ b/ooni/assets/captive_portal_tests.txt
@@ -0,0 +1,4 @@
+
+http://ooni.nu, Open Observatory of Network Interference, 200
+https://w2.eff.org/Censorship/Internet_censorship_bills/barlow_0296.declaration, let us now take our leave of them, 200
+https://www.torproject.org, Defend yourself against network surveillance and traffic analysis, 200
diff --git a/ooni/plugins/captiveportal.py b/ooni/plugins/captiveportal.py
new file mode 100644
index 0000000..ffa86c4
--- /dev/null
+++ b/ooni/plugins/captiveportal.py
@@ -0,0 +1,651 @@
+# -*- coding: utf-8 -*-
+"""
+ captiveportal
+ *************
+
+ This test is a collection of tests to detect the presence of a
+ captive portal. Code is taken, in part, from the old ooni-probe,
+ which was written by Jacob Appelbaum and Arturo Filastò.
+
+ :copyright: (c) 2012 Isis Lovecruft
+ :license: see LICENSE for more details
+"""
+import base64
+import os
+import random
+import re
+import string
+import urllib2
+from urlparse import urlparse
+
+from plugoo.assets import Asset
+from plugoo.tests import Test
+
+try:
+ from dns import resolver
+except ImportError:
+ print "The dnspython module was not found. https://crate.io/packages/dnspython/"
+
+try:
+ from gevent import monkey
+ monkey.patch_all(socket=True, dns=False, time=True, select=False, thread=True,
+ os=True, ssl=True, httplib=False, aggressive=True)
+except ImportError:
+ print "The gevent module was not found. https://crate.io/packages/gevent/"
+
+__plugoo__ = "captiveportal"
+__desc__ = "Captive portal detection test"
+
+
+class CaptivePortalAsset(Asset):
+ """
+ Parses captive_portal_tests.txt into an Asset.
+ """
+ def __init__(self, file=None):
+ self = Asset.__init__(self, file)
+
+ def parse_line(self, line):
+ self = Asset.parse_line(self, line)
+ return line.replace('\n', '').split(', ')
+
+class CaptivePortal(Test):
+ """
+ Compares content and status codes of HTTP responses, and attempts
+ to determine if content has been altered.
+ """
+ def __init__(self, ooni, name=__plugoo__):
+ Test.__init__(self, ooni, name)
+ self.default_ua = ooni.config.tests.default_ua
+
+ def http_fetch(self, url, headers={}):
+ """
+ Parses an HTTP url, fetches it, and returns a urllib2 response
+ object.
+ """
+ url = urlparse(url).geturl()
+ request = urllib2.Request(url, None, headers)
+ response = urllib2.urlopen(request)
+ response_headers = dict(response.headers)
+ return response, response_headers
+
+ def http_content_match_fuzzy_opt(self, experimental_url, control_result,
+ headers=None, fuzzy=False):
+ """
+ Makes an HTTP request on port 80 for experimental_url, then
+ compares the response_content of experimental_url with the
+ control_result. Optionally, if the fuzzy parameter is set to
+ True, the response_content is compared with a regex of the
+ control_result. If the response_content from the
+ experimental_url and the control_result match, returns True
+ with the HTTP status code and headers; False, status code, and
+ headers if otherwise.
+ """
+ log = self.logger
+
+ if headers is None:
+ default_ua = self.default_ua
+ headers = {'User-Agent': default_ua}
+
+ response, response_headers = self.http_fetch(experimental_url, headers)
+ response_content = response.read()
+ response_code = response.code
+ if response_content is not None:
+ if fuzzy:
+ pattern = re.compile(control_result)
+ match = pattern.search(response_content)
+ if not match:
+ log.info("Fuzzy HTTP content comparison for experiment URL")
+ log.info("'%s'" % experimental_url)
+ log.info("does not match!")
+ return False, response_code, response_headers
+ else:
+ log.info("Fuzzy HTTP content comparison of experiment URL")
+ log.info("'%s'" % experimental_url)
+ log.info("and the expected control result yielded a match.")
+ return True, response_code, response_headers
+ else:
+ if str(response_content) != str(control_result):
+ log.info("HTTP content comparison of experiment URL")
+ log.info("'%s'" % experimental_url)
+ log.info("and the expected control result do not match.")
+ return False, response_code, response_headers
+ else:
+ return True, response_code, response_headers
+ else:
+ log.warn("HTTP connection appears to have failed.")
+ return False, False, False
+
+ def http_status_code_match(self, experiment_code, control_code):
+ """
+ Compare two HTTP status codes, returns True if they match.
+ """
+ if int(experiment_code) != int(control_code):
+ return False
+ return True
+
+ def http_status_code_no_match(self, experiment_code, control_code):
+ """
+ Compare two HTTP status codes, returns True if they do not match.
+ """
+ if self.http_status_code_match(experiment_code, control_code):
+ return False
+ return True
+
+ def dns_resolve(self, hostname, nameserver=None):
+ """
+ Resolves hostname(s) though nameserver to corresponding
+ address(es). hostname may be either a single hostname string,
+ or a list of strings. If nameserver is not given, use local
+ DNS resolver, and if that fails try using 8.8.8.8.
+ """
+ log = self.logger
+
+ if isinstance(hostname, str):
+ hostname = [hostname]
+
+ if nameserver is not None:
+ res = resolver.Resolver(configure=False)
+ res.nameservers = [nameserver]
+ else:
+ res = resolver.Resolver()
+
+ response = []
+ answer = None
+
+ for hn in hostname:
+ try:
+ answer = res.query(hn)
+ except resolver.NoNameservers:
+ res.nameservers = ['8.8.8.8']
+ try:
+ answer = res.query(hn)
+ except resolver.NXDOMAIN:
+ log.info("DNS resolution for %s returned NXDOMAIN" % hn)
+ response.append('NXDOMAIN')
+ except resolver.NXDOMAIN:
+ log.info("DNS resolution for %s returned NXDOMAIN" % hn)
+ response.append('NXDOMAIN')
+ finally:
+ if answer:
+ for addr in answer:
+ response.append(addr.address)
+ return response
+
+ def dns_resolve_match(self, experiment_hostname, control_address):
+ """
+ Resolve experiment_hostname, and check to see that it returns
+ an experiment_address which matches the control_address. If
+ they match, returns True and experiment_address; otherwise
+ returns False and experiment_address.
+ """
+ log = self.logger
+
+ experiment_address = self.dns_resolve(experiment_hostname)
+ if experiment_address:
+ if len(set(experiment_address) & set([control_address])) > 0:
+ return True, experiment_address
+ else:
+ log.info("DNS comparison of control '%s' does not" % control_address)
+ log.info("match experiment response '%s'" % experiment_address)
+ return False, experiment_address
+ else:
+ log.debug("dns_resolve() for %s failed" % experiment_hostname)
+ return None, experiment_address
+
+ def get_auth_nameservers(self, hostname):
+ """
+ Many CPs set a nameserver to be used. Let's query that
+ nameserver for the authoritative nameservers of hostname.
+
+ The equivalent of:
+ $ dig +short NS ooni.nu
+ """
+ res = resolver.Resolver()
+ answer = res.query(hostname, 'NS')
+ auth_nameservers = []
+ for auth in answer:
+ auth_nameservers.append(auth.to_text())
+ return auth_nameservers
+
+ def hostname_to_0x20(self, hostname):
+ """
+ MaKEs yOur HOsTnaME lOoK LiKE THis.
+
+ For more information, see:
+ D. Dagon, et. al. "Increased DNS Forgery Resistance
+ Through 0x20-Bit Encoding". Proc. CSS, 2008.
+ """
+ hostname_0x20 = ''
+ for char in hostname:
+ l33t = random.choice(['caps', 'nocaps'])
+ if l33t == 'caps':
+ hostname_0x20 = hostname_0x20 + char.capitalize()
+ else:
+ hostname_0x20 = hostname_0x20 + char.lower()
+ return hostname_0x20
+
+ def check_0x20_to_auth_ns(self, hostname, sample_size=None):
+ """
+ Resolve a 0x20 DNS request for hostname over hostname's
+ authoritative nameserver(s), and check to make sure that
+ the capitalization in the 0x20 request matches that of the
+ response. Also, check the serial numbers of the SOA (Start
+ of Authority) records on the authoritative nameservers to
+ make sure that they match.
+
+ If sample_size is given, a random sample equal to that number
+ of authoritative nameservers will be queried; default is 5.
+ """
+ log = self.logger
+ log.info("")
+ log.info("Testing random capitalization of DNS queries...")
+ log.info("Testing that Start of Authority serial numbers match...")
+
+ auth_nameservers = self.get_auth_nameservers(hostname)
+
+ if sample_size is None:
+ sample_size = 5
+ resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers),
+ sample_size)
+
+ querynames = []
+ answernames = []
+ serials = []
+
+ # Even when gevent monkey patching is on, the requests here
+ # are sent without being 0x20'd, so we need to 0x20 them.
+ hostname = self.hostname_to_0x20(hostname)
+
+ for auth_ns in resolved_auth_ns:
+ res = resolver.Resolver(configure=False)
+ res.nameservers = [auth_ns]
+ try:
+ answer = res.query(hostname, 'SOA')
+ except resolver.Timeout:
+ continue
+ querynames.append(answer.qname.to_text())
+ answernames.append(answer.rrset.name.to_text())
+ for soa in answer:
+ serials.append(str(soa.serial))
+
+ if len(set(querynames).intersection(answernames)) == 1:
+ log.info("Capitalization in DNS queries and responses match.")
+ name_match = True
+ else:
+ log.info("The random capitalization '%s' used in" % hostname)
+ log.info("DNS queries to that hostname's authoritative")
+ log.info("nameservers does not match the capitalization in")
+ log.info("the response.")
+ name_match = False
+
+ if len(set(serials)) == 1:
+ log.info("Start of Authority serial numbers all match.")
+ serial_match = True
+ else:
+ log.info("Some SOA serial numbers did not match the rest!")
+ serial_match = False
+
+ ret = name_match, serial_match, querynames, answernames, serials
+
+ if name_match and serial_match:
+ log.info("Your DNS queries do not appear to be tampered.")
+ return ret
+ elif name_match or serial_match:
+ log.info("Something is tampering with your DNS queries.")
+ return ret
+ elif not name_match and not serial_match:
+ log.info("Your DNS queries are definitely being tampered with.")
+ return ret
+
+ def get_random_url_safe_string(self, length):
+ """
+ Returns a random url-safe string of specified length, where
+ 0 < length <= 256. The returned string will always start with
+ an alphabetic character.
+ """
+ if (length <= 0):
+ length = 1
+ elif (length > 256):
+ length = 256
+
+ random_ascii = base64.urlsafe_b64encode(os.urandom(int(length)))
+
+ while not random_ascii[:1].isalpha():
+ random_ascii = base64.urlsafe_b64encode(os.urandom(int(length)))
+
+ three_quarters = int((len(random_ascii)) * (3.0/4.0))
+ random_string = random_ascii[:three_quarters]
+ return random_string
+
+ def get_random_hostname(self, length=None):
+ """
+ Returns a random hostname with SLD of specified length. If
+ length is unspecified, length=32 is used.
+
+ These *should* all resolve to NXDOMAIN. If they actually
+ resolve to a box that isn't part of a captive portal that
+ would be rather interesting.
+ """
+ log = self.logger
+
+ if length is None:
+ length = 32
+
+ random_sld = self.get_random_url_safe_string(length)
+
+ # if it doesn't start with a letter, chuck it.
+ while not random_sld[:1].isalpha():
+ random_sld = self.get_random_url_safe_string(length)
+
+ tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid']
+ random_tld = urllib2.random.choice(tld_list)
+ random_hostname = random_sld + random_tld
+ return random_hostname
+
+ def compare_random_hostnames(self, hostname_count=None, hostname_length=None):
+ """
+ Get hostname_count number of random hostnames with SLD length
+ of hostname_length, and then attempt DNS resolution. If no
+ arguments are given, default to three hostnames of 32 bytes
+ each. These random hostnames *should* resolve to NXDOMAIN,
+ except in the case where a user is presented with a captive
+ portal and remains unauthenticated, in which case the captive
+ portal may return the address of the authentication page.
+
+ If the cardinality of the intersection of the set of resolved
+ random hostnames and the single element control set
+ (['NXDOMAIN']) are equal to one, then DNS properly resolved.
+
+ Returns true if only NXDOMAINs were returned, otherwise returns
+ False with the relative complement of the control set in the
+ response set.
+ """
+ log = self.logger
+
+ if hostname_count is None:
+ hostname_count = 3
+
+ log.info("Generating random hostnames...")
+ log.info("Resolving DNS for %d random hostnames..." % hostname_count)
+
+ control = ['NXDOMAIN']
+ responses = []
+
+ for x in range(hostname_count):
+ random_hostname = self.get_random_hostname(hostname_length)
+ response_match, response_address = self.dns_resolve_match(random_hostname,
+ control[0])
+ for address in response_address:
+ if response_match is False:
+ log.info("Strangely, DNS resolution of the random hostname")
+ log.info("%s actually points to %s"
+ % (random_hostname, response_address))
+ responses = responses + [address]
+ else:
+ responses = responses + [address]
+
+ intersection = set(responses) & set(control)
+ relative_complement = set(responses) - set(control)
+ r = set(responses)
+
+ if len(intersection) == 1:
+ log.info("All %d random hostnames properly resolved to NXDOMAIN."
+ % hostname_count)
+ return True, relative_complement
+ elif (len(intersection) == 1) and (len(r) > 1):
+ log.info("Something odd happened. Some random hostnames correctly")
+ log.info("resolved to NXDOMAIN, but several others resolved to")
+ log.info("to the following addresses: %s" % relative_complement)
+ return False, relative_complement
+ elif (len(intersection) == 0) and (len(r) == 1):
+ log.info("All random hostnames resolved to the IP address ")
+ log.info("'%s', which is indicative of a captive portal." % r)
+ return False, relative_complement
+ else:
+ log.debug("Apparently, pigs are flying on your network, 'cause a")
+ log.debug("bunch of hostnames made from 32-byte random strings")
+ log.debug("just magically resolved to a bunch of random addresses.")
+ log.debug("That is definitely highly improbable. In fact, my napkin")
+ log.debug("tells me that the probability of just one of those")
+ log.degug("hostnames resolving to an address is 1.68e-59, making")
+ log.debug("it nearly twice as unlikely as an MD5 hash collision.")
+ log.debug("Either someone is seriously messing with your network,")
+ log.debug("or else you are witnessing the impossible. %s" % r)
+ return False, relative_complement
+
+ def google_dns_cp_test(self):
+ """
+ Google Chrome resolves three 10-byte random hostnames.
+ """
+ log = self.logger
+ subtest = "Google Chrome DNS-based"
+
+ log.info("")
+ log.info("Running the Google Chrome DNS-based captive portal test...")
+
+ gmatch, google_dns_result = self.compare_random_hostnames(3, 10)
+
+ if gmatch:
+ log.info("Google Chrome DNS-based captive portal test did not")
+ log.info("detect a captive portal.")
+ return google_dns_result
+ else:
+ log.info("Google Chrome DNS-based captive portal test believes")
+ log.info("you are in a captive portal, or else something very")
+ log.info("odd is happening with your DNS.")
+ return google_dns_result
+
+ def ms_dns_cp_test(self):
+ """
+ Microsoft "phones home" to a server which will always resolve
+ to the same address.
+ """
+ log = self.logger
+ subtest = "Microsoft NCSI DNS-based"
+
+ log.info("")
+ log.info("Running the Microsoft NCSI DNS-based captive portal")
+ log.info("test...")
+
+ msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com",
+ "131.107.255.255")
+ if msmatch:
+ log.info("Microsoft NCSI DNS-based captive portal test did not")
+ log.info("detect a captive portal.")
+ return ms_dns_result
+ else:
+ log.info("Microsoft NCSI DNS-based captive portal test ")
+ log.info("believes you are in a captive portal.")
+ return ms_dns_result
+
+ def run_vendor_dns_tests(self):
+ """
+ Run the vendor DNS tests.
+ """
+ self.google_dns_cp_test()
+ self.ms_dns_cp_test()
+
+ return
+
+ def run_vendor_tests(self, *a, **kw):
+ """
+ These are several vendor tests used to detect the presence of
+ a captive portal. Each test compares HTTP status code and
+ content to the control results and has its own User-Agent
+ string, in order to emulate the test as it would occur on the
+ device it was intended for. Vendor tests are defined in the
+ format:
+ [exp_url, ctrl_result, ctrl_code, ua, test_name]
+ """
+
+ vendor_tests = [['http://www.apple.com/library/test/success.html',
+ 'Success',
+ '200',
+ 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3',
+ 'Apple HTTP Captive Portal'],
+ ['http://tools.ietf.org/html/draft-nottingham-http-portal-02',
+ '428 Network Authentication Required',
+ '428',
+ 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0',
+ 'W3 Captive Portal'],
+ ['http://www.msftncsi.com/ncsi.txt',
+ 'Microsoft NCSI',
+ '200',
+ 'Microsoft NCSI',
+ 'MS HTTP Captive Portal',]]
+
+ cm = self.http_content_match_fuzzy_opt
+ sm = self.http_status_code_match
+ snm = self.http_status_code_no_match
+ log = self.logger
+
+ def compare_content(status_func, fuzzy, experiment_url, control_result,
+ control_code, headers, test_name):
+ log.info("")
+ log.info("Running the %s test..." % test_name)
+
+ content_match, experiment_code, experiment_headers = cm(experiment_url,
+ control_result,
+ headers, fuzzy)
+ status_match = status_func(experiment_code, control_code)
+
+ if status_match and content_match:
+ log.info("The %s test was unable to detect" % test_name)
+ log.info("a captive portal.")
+ else:
+ log.info("The %s test shows that your network" % test_name)
+ log.info("is filtered.")
+
+ for vt in vendor_tests:
+ experiment_url = vt[0]
+ control_result = vt[1]
+ control_code = vt[2]
+ headers = {'User-Agent': vt[3]}
+ test_name = vt[4]
+
+ args = (experiment_url, control_result, control_code, headers, test_name)
+
+ if test_name == "MS HTTP Captive Portal":
+ compare_content(sm, False, *args)
+
+ elif test_name == "Apple HTTP Captive Portal":
+ compare_content(sm, True, *args)
+
+ elif test_name == "W3 Captive Portal":
+ compare_content(snm, True, *args)
+
+ else:
+ log.warn("Ooni is trying to run an undefined CP vendor test.")
+
+ def experiment(self, *a, **kw):
+ """
+ Compares the content and status code of the HTTP response for
+ experiment_url with the control_result and control_code
+ respectively. If the status codes match, but the experimental
+ content and control_result do not match, fuzzy matching is enabled
+ to determine if the control_result is at least included somewhere
+ in the experimental content. Returns True if matches are found,
+ and False if otherwise.
+ """
+ if (os.path.isfile(os.path.join(self.config.main.assetdir,
+ self.config.tests.captive_portal))):
+ kw['data'].append(None)
+ kw['data'].append('user-defined')
+
+ experiment_url = kw['data'][0]
+ control_result = kw['data'][1]
+ control_code = kw['data'][2]
+ ua = kw['data'][3]
+ test_name = kw['data'][4]
+
+ cm = self.http_content_match_fuzzy_opt
+ sm = self.http_status_code_match
+ snm = self.http_status_code_no_match
+
+ log = self.logger
+
+ if test_name == "user-defined":
+ log.info("Running %s test for '%s'..." % (test_name, experiment_url))
+ content_match, experiment_code, experiment_headers = cm(experiment_url,
+ control_result)
+ status_match = sm(experiment_code, control_code)
+ if status_match and content_match:
+ log.info("The %s test for '%s'" % (test_name, experiment_url))
+ log.info("was unable to detect a captive portal.")
+ return True, test_name
+ elif status_match and not content_match:
+ log.info("Retrying '%s' with fuzzy match enabled."
+ % experiment_url)
+ fuzzy_match, experiment_code, experiment_headers = cm(experiment_url,
+ control_result,
+ fuzzy=True)
+ if fuzzy_match:
+ return True, test_name
+ else:
+ log.info("Found modified content on '%s'," % experiment_url)
+ log.info("which could indicate a captive portal.")
+
+ return False, test_name
+ else:
+ log.info("The content comparison test for ")
+ log.info("'%s'" % experiment_url)
+ log.info("shows that your HTTP traffic is filtered.")
+ return False, test_name
+
+ else:
+ log.warn("Ooni is trying to run an undefined captive portal test.")
+ return False, test_name
+
+
+def run(ooni):
+ """
+ Runs the CaptivePortal(Test).
+
+ CONFIG OPTIONS
+ --------------
+
+ If "do_captive_portal_vendor_tests" is set to "true", then vendor
+ specific captive portal HTTP-based tests will be run.
+
+ If "do_captive_portal_dns_tests" is set to "true", then vendor
+ specific captive portal DNS-based tests will be run.
+
+ If "check_dns_requests" is set to "true", then Ooni-probe will
+ attempt to check that your DNS requests are not being tampered with
+ by a captive portal.
+
+ If "captive_portal" = "yourfilename.txt", then user-specified tests
+ will be run.
+
+ Any combination of the above tests can be run.
+ """
+ config = ooni.config
+ log = ooni.logger
+
+ assets = []
+ if (os.path.isfile(os.path.join(config.main.assetdir,
+ config.tests.captive_portal))):
+ assets.append(CaptivePortalAsset(os.path.join(config.main.assetdir,
+ config.tests.captive_portal)))
+
+ captiveportal = CaptivePortal(ooni)
+ log.info("Starting captive portal test...")
+ captiveportal.run(assets, {'index': 1})
+
+ if config.tests.do_captive_portal_vendor_tests:
+ log.info("")
+ log.info("Running vendor tests...")
+ captiveportal.run_vendor_tests()
+
+ if config.tests.do_captive_portal_vendor_dns_tests:
+ log.info("")
+ log.info("Running vendor DNS-based tests...")
+ captiveportal.run_vendor_dns_tests()
+
+ if config.tests.check_dns_requests:
+ log.info("")
+ log.info("Checking that DNS requests are not being tampered...")
+ captiveportal.check_0x20_to_auth_ns('ooni.nu')
+
+ log.info("")
+ log.info("Captive portal test finished!")
diff --git a/tests/captiveportal.py b/tests/captiveportal.py
deleted file mode 100644
index ffa86c4..0000000
--- a/tests/captiveportal.py
+++ /dev/null
@@ -1,651 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
- captiveportal
- *************
-
- This test is a collection of tests to detect the presence of a
- captive portal. Code is taken, in part, from the old ooni-probe,
- which was written by Jacob Appelbaum and Arturo Filastò.
-
- :copyright: (c) 2012 Isis Lovecruft
- :license: see LICENSE for more details
-"""
-import base64
-import os
-import random
-import re
-import string
-import urllib2
-from urlparse import urlparse
-
-from plugoo.assets import Asset
-from plugoo.tests import Test
-
-try:
- from dns import resolver
-except ImportError:
- print "The dnspython module was not found. https://crate.io/packages/dnspython/"
-
-try:
- from gevent import monkey
- monkey.patch_all(socket=True, dns=False, time=True, select=False, thread=True,
- os=True, ssl=True, httplib=False, aggressive=True)
-except ImportError:
- print "The gevent module was not found. https://crate.io/packages/gevent/"
-
-__plugoo__ = "captiveportal"
-__desc__ = "Captive portal detection test"
-
-
-class CaptivePortalAsset(Asset):
- """
- Parses captive_portal_tests.txt into an Asset.
- """
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
- def parse_line(self, line):
- self = Asset.parse_line(self, line)
- return line.replace('\n', '').split(', ')
-
-class CaptivePortal(Test):
- """
- Compares content and status codes of HTTP responses, and attempts
- to determine if content has been altered.
- """
- def __init__(self, ooni, name=__plugoo__):
- Test.__init__(self, ooni, name)
- self.default_ua = ooni.config.tests.default_ua
-
- def http_fetch(self, url, headers={}):
- """
- Parses an HTTP url, fetches it, and returns a urllib2 response
- object.
- """
- url = urlparse(url).geturl()
- request = urllib2.Request(url, None, headers)
- response = urllib2.urlopen(request)
- response_headers = dict(response.headers)
- return response, response_headers
-
- def http_content_match_fuzzy_opt(self, experimental_url, control_result,
- headers=None, fuzzy=False):
- """
- Makes an HTTP request on port 80 for experimental_url, then
- compares the response_content of experimental_url with the
- control_result. Optionally, if the fuzzy parameter is set to
- True, the response_content is compared with a regex of the
- control_result. If the response_content from the
- experimental_url and the control_result match, returns True
- with the HTTP status code and headers; False, status code, and
- headers if otherwise.
- """
- log = self.logger
-
- if headers is None:
- default_ua = self.default_ua
- headers = {'User-Agent': default_ua}
-
- response, response_headers = self.http_fetch(experimental_url, headers)
- response_content = response.read()
- response_code = response.code
- if response_content is not None:
- if fuzzy:
- pattern = re.compile(control_result)
- match = pattern.search(response_content)
- if not match:
- log.info("Fuzzy HTTP content comparison for experiment URL")
- log.info("'%s'" % experimental_url)
- log.info("does not match!")
- return False, response_code, response_headers
- else:
- log.info("Fuzzy HTTP content comparison of experiment URL")
- log.info("'%s'" % experimental_url)
- log.info("and the expected control result yielded a match.")
- return True, response_code, response_headers
- else:
- if str(response_content) != str(control_result):
- log.info("HTTP content comparison of experiment URL")
- log.info("'%s'" % experimental_url)
- log.info("and the expected control result do not match.")
- return False, response_code, response_headers
- else:
- return True, response_code, response_headers
- else:
- log.warn("HTTP connection appears to have failed.")
- return False, False, False
-
- def http_status_code_match(self, experiment_code, control_code):
- """
- Compare two HTTP status codes, returns True if they match.
- """
- if int(experiment_code) != int(control_code):
- return False
- return True
-
- def http_status_code_no_match(self, experiment_code, control_code):
- """
- Compare two HTTP status codes, returns True if they do not match.
- """
- if self.http_status_code_match(experiment_code, control_code):
- return False
- return True
-
- def dns_resolve(self, hostname, nameserver=None):
- """
- Resolves hostname(s) though nameserver to corresponding
- address(es). hostname may be either a single hostname string,
- or a list of strings. If nameserver is not given, use local
- DNS resolver, and if that fails try using 8.8.8.8.
- """
- log = self.logger
-
- if isinstance(hostname, str):
- hostname = [hostname]
-
- if nameserver is not None:
- res = resolver.Resolver(configure=False)
- res.nameservers = [nameserver]
- else:
- res = resolver.Resolver()
-
- response = []
- answer = None
-
- for hn in hostname:
- try:
- answer = res.query(hn)
- except resolver.NoNameservers:
- res.nameservers = ['8.8.8.8']
- try:
- answer = res.query(hn)
- except resolver.NXDOMAIN:
- log.info("DNS resolution for %s returned NXDOMAIN" % hn)
- response.append('NXDOMAIN')
- except resolver.NXDOMAIN:
- log.info("DNS resolution for %s returned NXDOMAIN" % hn)
- response.append('NXDOMAIN')
- finally:
- if answer:
- for addr in answer:
- response.append(addr.address)
- return response
-
- def dns_resolve_match(self, experiment_hostname, control_address):
- """
- Resolve experiment_hostname, and check to see that it returns
- an experiment_address which matches the control_address. If
- they match, returns True and experiment_address; otherwise
- returns False and experiment_address.
- """
- log = self.logger
-
- experiment_address = self.dns_resolve(experiment_hostname)
- if experiment_address:
- if len(set(experiment_address) & set([control_address])) > 0:
- return True, experiment_address
- else:
- log.info("DNS comparison of control '%s' does not" % control_address)
- log.info("match experiment response '%s'" % experiment_address)
- return False, experiment_address
- else:
- log.debug("dns_resolve() for %s failed" % experiment_hostname)
- return None, experiment_address
-
- def get_auth_nameservers(self, hostname):
- """
- Many CPs set a nameserver to be used. Let's query that
- nameserver for the authoritative nameservers of hostname.
-
- The equivalent of:
- $ dig +short NS ooni.nu
- """
- res = resolver.Resolver()
- answer = res.query(hostname, 'NS')
- auth_nameservers = []
- for auth in answer:
- auth_nameservers.append(auth.to_text())
- return auth_nameservers
-
- def hostname_to_0x20(self, hostname):
- """
- MaKEs yOur HOsTnaME lOoK LiKE THis.
-
- For more information, see:
- D. Dagon, et. al. "Increased DNS Forgery Resistance
- Through 0x20-Bit Encoding". Proc. CSS, 2008.
- """
- hostname_0x20 = ''
- for char in hostname:
- l33t = random.choice(['caps', 'nocaps'])
- if l33t == 'caps':
- hostname_0x20 = hostname_0x20 + char.capitalize()
- else:
- hostname_0x20 = hostname_0x20 + char.lower()
- return hostname_0x20
-
- def check_0x20_to_auth_ns(self, hostname, sample_size=None):
- """
- Resolve a 0x20 DNS request for hostname over hostname's
- authoritative nameserver(s), and check to make sure that
- the capitalization in the 0x20 request matches that of the
- response. Also, check the serial numbers of the SOA (Start
- of Authority) records on the authoritative nameservers to
- make sure that they match.
-
- If sample_size is given, a random sample equal to that number
- of authoritative nameservers will be queried; default is 5.
- """
- log = self.logger
- log.info("")
- log.info("Testing random capitalization of DNS queries...")
- log.info("Testing that Start of Authority serial numbers match...")
-
- auth_nameservers = self.get_auth_nameservers(hostname)
-
- if sample_size is None:
- sample_size = 5
- resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers),
- sample_size)
-
- querynames = []
- answernames = []
- serials = []
-
- # Even when gevent monkey patching is on, the requests here
- # are sent without being 0x20'd, so we need to 0x20 them.
- hostname = self.hostname_to_0x20(hostname)
-
- for auth_ns in resolved_auth_ns:
- res = resolver.Resolver(configure=False)
- res.nameservers = [auth_ns]
- try:
- answer = res.query(hostname, 'SOA')
- except resolver.Timeout:
- continue
- querynames.append(answer.qname.to_text())
- answernames.append(answer.rrset.name.to_text())
- for soa in answer:
- serials.append(str(soa.serial))
-
- if len(set(querynames).intersection(answernames)) == 1:
- log.info("Capitalization in DNS queries and responses match.")
- name_match = True
- else:
- log.info("The random capitalization '%s' used in" % hostname)
- log.info("DNS queries to that hostname's authoritative")
- log.info("nameservers does not match the capitalization in")
- log.info("the response.")
- name_match = False
-
- if len(set(serials)) == 1:
- log.info("Start of Authority serial numbers all match.")
- serial_match = True
- else:
- log.info("Some SOA serial numbers did not match the rest!")
- serial_match = False
-
- ret = name_match, serial_match, querynames, answernames, serials
-
- if name_match and serial_match:
- log.info("Your DNS queries do not appear to be tampered.")
- return ret
- elif name_match or serial_match:
- log.info("Something is tampering with your DNS queries.")
- return ret
- elif not name_match and not serial_match:
- log.info("Your DNS queries are definitely being tampered with.")
- return ret
-
- def get_random_url_safe_string(self, length):
- """
- Returns a random url-safe string of specified length, where
- 0 < length <= 256. The returned string will always start with
- an alphabetic character.
- """
- if (length <= 0):
- length = 1
- elif (length > 256):
- length = 256
-
- random_ascii = base64.urlsafe_b64encode(os.urandom(int(length)))
-
- while not random_ascii[:1].isalpha():
- random_ascii = base64.urlsafe_b64encode(os.urandom(int(length)))
-
- three_quarters = int((len(random_ascii)) * (3.0/4.0))
- random_string = random_ascii[:three_quarters]
- return random_string
-
- def get_random_hostname(self, length=None):
- """
- Returns a random hostname with SLD of specified length. If
- length is unspecified, length=32 is used.
-
- These *should* all resolve to NXDOMAIN. If they actually
- resolve to a box that isn't part of a captive portal that
- would be rather interesting.
- """
- log = self.logger
-
- if length is None:
- length = 32
-
- random_sld = self.get_random_url_safe_string(length)
-
- # if it doesn't start with a letter, chuck it.
- while not random_sld[:1].isalpha():
- random_sld = self.get_random_url_safe_string(length)
-
- tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid']
- random_tld = urllib2.random.choice(tld_list)
- random_hostname = random_sld + random_tld
- return random_hostname
-
- def compare_random_hostnames(self, hostname_count=None, hostname_length=None):
- """
- Get hostname_count number of random hostnames with SLD length
- of hostname_length, and then attempt DNS resolution. If no
- arguments are given, default to three hostnames of 32 bytes
- each. These random hostnames *should* resolve to NXDOMAIN,
- except in the case where a user is presented with a captive
- portal and remains unauthenticated, in which case the captive
- portal may return the address of the authentication page.
-
- If the cardinality of the intersection of the set of resolved
- random hostnames and the single element control set
- (['NXDOMAIN']) are equal to one, then DNS properly resolved.
-
- Returns true if only NXDOMAINs were returned, otherwise returns
- False with the relative complement of the control set in the
- response set.
- """
- log = self.logger
-
- if hostname_count is None:
- hostname_count = 3
-
- log.info("Generating random hostnames...")
- log.info("Resolving DNS for %d random hostnames..." % hostname_count)
-
- control = ['NXDOMAIN']
- responses = []
-
- for x in range(hostname_count):
- random_hostname = self.get_random_hostname(hostname_length)
- response_match, response_address = self.dns_resolve_match(random_hostname,
- control[0])
- for address in response_address:
- if response_match is False:
- log.info("Strangely, DNS resolution of the random hostname")
- log.info("%s actually points to %s"
- % (random_hostname, response_address))
- responses = responses + [address]
- else:
- responses = responses + [address]
-
- intersection = set(responses) & set(control)
- relative_complement = set(responses) - set(control)
- r = set(responses)
-
- if len(intersection) == 1:
- log.info("All %d random hostnames properly resolved to NXDOMAIN."
- % hostname_count)
- return True, relative_complement
- elif (len(intersection) == 1) and (len(r) > 1):
- log.info("Something odd happened. Some random hostnames correctly")
- log.info("resolved to NXDOMAIN, but several others resolved to")
- log.info("to the following addresses: %s" % relative_complement)
- return False, relative_complement
- elif (len(intersection) == 0) and (len(r) == 1):
- log.info("All random hostnames resolved to the IP address ")
- log.info("'%s', which is indicative of a captive portal." % r)
- return False, relative_complement
- else:
- log.debug("Apparently, pigs are flying on your network, 'cause a")
- log.debug("bunch of hostnames made from 32-byte random strings")
- log.debug("just magically resolved to a bunch of random addresses.")
- log.debug("That is definitely highly improbable. In fact, my napkin")
- log.debug("tells me that the probability of just one of those")
- log.degug("hostnames resolving to an address is 1.68e-59, making")
- log.debug("it nearly twice as unlikely as an MD5 hash collision.")
- log.debug("Either someone is seriously messing with your network,")
- log.debug("or else you are witnessing the impossible. %s" % r)
- return False, relative_complement
-
- def google_dns_cp_test(self):
- """
- Google Chrome resolves three 10-byte random hostnames.
- """
- log = self.logger
- subtest = "Google Chrome DNS-based"
-
- log.info("")
- log.info("Running the Google Chrome DNS-based captive portal test...")
-
- gmatch, google_dns_result = self.compare_random_hostnames(3, 10)
-
- if gmatch:
- log.info("Google Chrome DNS-based captive portal test did not")
- log.info("detect a captive portal.")
- return google_dns_result
- else:
- log.info("Google Chrome DNS-based captive portal test believes")
- log.info("you are in a captive portal, or else something very")
- log.info("odd is happening with your DNS.")
- return google_dns_result
-
- def ms_dns_cp_test(self):
- """
- Microsoft "phones home" to a server which will always resolve
- to the same address.
- """
- log = self.logger
- subtest = "Microsoft NCSI DNS-based"
-
- log.info("")
- log.info("Running the Microsoft NCSI DNS-based captive portal")
- log.info("test...")
-
- msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com",
- "131.107.255.255")
- if msmatch:
- log.info("Microsoft NCSI DNS-based captive portal test did not")
- log.info("detect a captive portal.")
- return ms_dns_result
- else:
- log.info("Microsoft NCSI DNS-based captive portal test ")
- log.info("believes you are in a captive portal.")
- return ms_dns_result
-
- def run_vendor_dns_tests(self):
- """
- Run the vendor DNS tests.
- """
- self.google_dns_cp_test()
- self.ms_dns_cp_test()
-
- return
-
- def run_vendor_tests(self, *a, **kw):
- """
- These are several vendor tests used to detect the presence of
- a captive portal. Each test compares HTTP status code and
- content to the control results and has its own User-Agent
- string, in order to emulate the test as it would occur on the
- device it was intended for. Vendor tests are defined in the
- format:
- [exp_url, ctrl_result, ctrl_code, ua, test_name]
- """
-
- vendor_tests = [['http://www.apple.com/library/test/success.html',
- 'Success',
- '200',
- 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3',
- 'Apple HTTP Captive Portal'],
- ['http://tools.ietf.org/html/draft-nottingham-http-portal-02',
- '428 Network Authentication Required',
- '428',
- 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0',
- 'W3 Captive Portal'],
- ['http://www.msftncsi.com/ncsi.txt',
- 'Microsoft NCSI',
- '200',
- 'Microsoft NCSI',
- 'MS HTTP Captive Portal',]]
-
- cm = self.http_content_match_fuzzy_opt
- sm = self.http_status_code_match
- snm = self.http_status_code_no_match
- log = self.logger
-
- def compare_content(status_func, fuzzy, experiment_url, control_result,
- control_code, headers, test_name):
- log.info("")
- log.info("Running the %s test..." % test_name)
-
- content_match, experiment_code, experiment_headers = cm(experiment_url,
- control_result,
- headers, fuzzy)
- status_match = status_func(experiment_code, control_code)
-
- if status_match and content_match:
- log.info("The %s test was unable to detect" % test_name)
- log.info("a captive portal.")
- else:
- log.info("The %s test shows that your network" % test_name)
- log.info("is filtered.")
-
- for vt in vendor_tests:
- experiment_url = vt[0]
- control_result = vt[1]
- control_code = vt[2]
- headers = {'User-Agent': vt[3]}
- test_name = vt[4]
-
- args = (experiment_url, control_result, control_code, headers, test_name)
-
- if test_name == "MS HTTP Captive Portal":
- compare_content(sm, False, *args)
-
- elif test_name == "Apple HTTP Captive Portal":
- compare_content(sm, True, *args)
-
- elif test_name == "W3 Captive Portal":
- compare_content(snm, True, *args)
-
- else:
- log.warn("Ooni is trying to run an undefined CP vendor test.")
-
- def experiment(self, *a, **kw):
- """
- Compares the content and status code of the HTTP response for
- experiment_url with the control_result and control_code
- respectively. If the status codes match, but the experimental
- content and control_result do not match, fuzzy matching is enabled
- to determine if the control_result is at least included somewhere
- in the experimental content. Returns True if matches are found,
- and False if otherwise.
- """
- if (os.path.isfile(os.path.join(self.config.main.assetdir,
- self.config.tests.captive_portal))):
- kw['data'].append(None)
- kw['data'].append('user-defined')
-
- experiment_url = kw['data'][0]
- control_result = kw['data'][1]
- control_code = kw['data'][2]
- ua = kw['data'][3]
- test_name = kw['data'][4]
-
- cm = self.http_content_match_fuzzy_opt
- sm = self.http_status_code_match
- snm = self.http_status_code_no_match
-
- log = self.logger
-
- if test_name == "user-defined":
- log.info("Running %s test for '%s'..." % (test_name, experiment_url))
- content_match, experiment_code, experiment_headers = cm(experiment_url,
- control_result)
- status_match = sm(experiment_code, control_code)
- if status_match and content_match:
- log.info("The %s test for '%s'" % (test_name, experiment_url))
- log.info("was unable to detect a captive portal.")
- return True, test_name
- elif status_match and not content_match:
- log.info("Retrying '%s' with fuzzy match enabled."
- % experiment_url)
- fuzzy_match, experiment_code, experiment_headers = cm(experiment_url,
- control_result,
- fuzzy=True)
- if fuzzy_match:
- return True, test_name
- else:
- log.info("Found modified content on '%s'," % experiment_url)
- log.info("which could indicate a captive portal.")
-
- return False, test_name
- else:
- log.info("The content comparison test for ")
- log.info("'%s'" % experiment_url)
- log.info("shows that your HTTP traffic is filtered.")
- return False, test_name
-
- else:
- log.warn("Ooni is trying to run an undefined captive portal test.")
- return False, test_name
-
-
-def run(ooni):
- """
- Runs the CaptivePortal(Test).
-
- CONFIG OPTIONS
- --------------
-
- If "do_captive_portal_vendor_tests" is set to "true", then vendor
- specific captive portal HTTP-based tests will be run.
-
- If "do_captive_portal_dns_tests" is set to "true", then vendor
- specific captive portal DNS-based tests will be run.
-
- If "check_dns_requests" is set to "true", then Ooni-probe will
- attempt to check that your DNS requests are not being tampered with
- by a captive portal.
-
- If "captive_portal" = "yourfilename.txt", then user-specified tests
- will be run.
-
- Any combination of the above tests can be run.
- """
- config = ooni.config
- log = ooni.logger
-
- assets = []
- if (os.path.isfile(os.path.join(config.main.assetdir,
- config.tests.captive_portal))):
- assets.append(CaptivePortalAsset(os.path.join(config.main.assetdir,
- config.tests.captive_portal)))
-
- captiveportal = CaptivePortal(ooni)
- log.info("Starting captive portal test...")
- captiveportal.run(assets, {'index': 1})
-
- if config.tests.do_captive_portal_vendor_tests:
- log.info("")
- log.info("Running vendor tests...")
- captiveportal.run_vendor_tests()
-
- if config.tests.do_captive_portal_vendor_dns_tests:
- log.info("")
- log.info("Running vendor DNS-based tests...")
- captiveportal.run_vendor_dns_tests()
-
- if config.tests.check_dns_requests:
- log.info("")
- log.info("Checking that DNS requests are not being tampered...")
- captiveportal.check_0x20_to_auth_ns('ooni.nu')
-
- log.info("")
- log.info("Captive portal test finished!")
[View Less]
1
0

09 Jul '12
commit 45cc771b7e240fc8ba411670d3533593e3165195
Merge: c85c627 3c632b4
Author: Arturo Filastò <art(a)torproject.org>
Date: Mon Jul 9 16:40:19 2012 +0200
Merge branch 'master' into captiveportal
Conflicts:
ooni/plugins/captiveportal.py
ooni/plugins/captiveportal.py | 413 +++++++++++++++++++----------------------
1 files changed, 190 insertions(+), 223 deletions(-)
diff --cc ooni/plugins/captiveportal.py
index ffa86c4,9b7bee3..f463fe4
--- a/ooni/plugins/…
[View More]captiveportal.py
+++ b/ooni/plugins/captiveportal.py
@@@ -138,17 -138,16 +138,14 @@@ class CaptivePortal(OONITest)
or a list of strings. If nameserver is not given, use local
DNS resolver, and if that fails try using 8.8.8.8.
"""
- log = self.logger
--
if isinstance(hostname, str):
hostname = [hostname]
-
+
if nameserver is not None:
res = resolver.Resolver(configure=False)
res.nameservers = [nameserver]
else:
res = resolver.Resolver()
-
-
response = []
answer = None
@@@ -191,7 -188,7 +186,6 @@@
else:
log.debug("dns_resolve() for %s failed" % experiment_hostname)
return None, experiment_address
-
-
def get_auth_nameservers(self, hostname):
"""
Many CPs set a nameserver to be used. Let's query that
@@@ -210,9 -207,9 +204,8 @@@
def hostname_to_0x20(self, hostname):
"""
MaKEs yOur HOsTnaME lOoK LiKE THis.
--
- For more information, see:
- D. Dagon, et. al. "Increased DNS Forgery Resistance
+ For more information, see:
+ D. Dagon, et. al. "Increased DNS Forgery Resistance
Through 0x20-Bit Encoding". Proc. CSS, 2008.
"""
hostname_0x20 = ''
@@@ -255,7 -251,7 +247,6 @@@
# Even when gevent monkey patching is on, the requests here
# are sent without being 0x20'd, so we need to 0x20 them.
hostname = self.hostname_to_0x20(hostname)
-
-
for auth_ns in resolved_auth_ns:
res = resolver.Resolver(configure=False)
res.nameservers = [auth_ns]
@@@ -309,7 -305,7 +300,6 @@@
length = 256
random_ascii = base64.urlsafe_b64encode(os.urandom(int(length)))
-
-
while not random_ascii[:1].isalpha():
random_ascii = base64.urlsafe_b64encode(os.urandom(int(length)))
@@@ -336,7 -330,7 +324,6 @@@
# if it doesn't start with a letter, chuck it.
while not random_sld[:1].isalpha():
random_sld = self.get_random_url_safe_string(length)
-
-
tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid']
random_tld = urllib2.random.choice(tld_list)
random_hostname = random_sld + random_tld
[View Less]
1
0

[ooni-probe/master] Port Isis's captive portal test to the new framework.
by art@torproject.org 09 Jul '12
by art@torproject.org 09 Jul '12
09 Jul '12
commit 3c632b495d15cfeb77577e6f9153f4915453cbfc
Author: Arturo Filastò <art(a)torproject.org>
Date: Mon Jul 9 12:41:49 2012 +0200
Port Isis's captive portal test to the new framework.
---
ooni/plugins/captiveportal.py | 625 +++++++++++++++++++++++++++++++++++++++++
1 files changed, 625 insertions(+), 0 deletions(-)
diff --git a/ooni/plugins/captiveportal.py b/ooni/plugins/captiveportal.py
new file mode 100644
index 0000000..9b7bee3
--- /dev/null
+++ b/ooni/plugins/captiveportal.…
[View More]py
@@ -0,0 +1,625 @@
+# -*- coding: utf-8 -*-
+"""
+ captiveportal
+ *************
+
+ This test is a collection of tests to detect the presence of a
+ captive portal. Code is taken, in part, from the old ooni-probe,
+ which was written by Jacob Appelbaum and Arturo Filastò.
+
+ :copyright: (c) 2012 Isis Lovecruft
+ :license: see LICENSE for more details
+"""
+import base64
+import os
+import random
+import re
+import string
+import urllib2
+from urlparse import urlparse
+
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+
+from ooni.plugoo.assets import Asset
+from ooni.plugoo.tests import ITest, OONITest
+from ooni.protocols import http
+from ooni import log
+
+try:
+ from dns import resolver
+except ImportError:
+ print "The dnspython module was not found. https://crate.io/packages/dnspython/"
+
+__plugoo__ = "captiveportal"
+__desc__ = "Captive portal detection test"
+
+class CaptivePortalArgs(usage.Options):
+ optParameters = [['asset', 'a', None, 'Asset file'],
+ ['resume', 'r', 0, 'Resume at this index'],
+ ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'],
+ ['user-agent', 'u', random.choice(http.useragents),
+ 'User agent for HTTP requests']
+ ]
+
+class CaptivePortal(OONITest):
+ """
+ Compares content and status codes of HTTP responses, and attempts
+ to determine if content has been altered.
+ """
+
+ implements(IPlugin, ITest)
+
+ shortName = "captivep"
+ description = "Captive Portal Test"
+ requirements = None
+ options = CaptivePortalArgs
+ # Tells this to be blocking.
+ blocking = True
+
+ def http_fetch(self, url, headers={}):
+ """
+ Parses an HTTP url, fetches it, and returns a urllib2 response
+ object.
+ """
+ url = urlparse(url).geturl()
+ request = urllib2.Request(url, None, headers)
+ response = urllib2.urlopen(request)
+ response_headers = dict(response.headers)
+ return response, response_headers
+
+ def http_content_match_fuzzy_opt(self, experimental_url, control_result,
+ headers=None, fuzzy=False):
+ """
+ Makes an HTTP request on port 80 for experimental_url, then
+ compares the response_content of experimental_url with the
+ control_result. Optionally, if the fuzzy parameter is set to
+ True, the response_content is compared with a regex of the
+ control_result. If the response_content from the
+ experimental_url and the control_result match, returns True
+ with the HTTP status code and headers; False, status code, and
+ headers if otherwise.
+ """
+
+ if headers is None:
+ default_ua = self.local_options['user-agent']
+ headers = {'User-Agent': default_ua}
+
+ response, response_headers = self.http_fetch(experimental_url, headers)
+ response_content = response.read()
+ response_code = response.code
+ if response_content is not None:
+ if fuzzy:
+ pattern = re.compile(control_result)
+ match = pattern.search(response_content)
+ if not match:
+ log.msg("Fuzzy HTTP content comparison for experiment URL")
+ log.msg("'%s'" % experimental_url)
+ log.msg("does not match!")
+ return False, response_code, response_headers
+ else:
+ log.msg("Fuzzy HTTP content comparison of experiment URL")
+ log.msg("'%s'" % experimental_url)
+ log.msg("and the expected control result yielded a match.")
+ return True, response_code, response_headers
+ else:
+ if str(response_content) != str(control_result):
+ log.msg("HTTP content comparison of experiment URL")
+ log.msg("'%s'" % experimental_url)
+ log.msg("and the expected control result do not match.")
+ return False, response_code, response_headers
+ else:
+ return True, response_code, response_headers
+ else:
+ log.warn("HTTP connection appears to have failed.")
+ return False, False, False
+
+ def http_status_code_match(self, experiment_code, control_code):
+ """
+ Compare two HTTP status codes, returns True if they match.
+ """
+ if int(experiment_code) != int(control_code):
+ return False
+ return True
+
+ def http_status_code_no_match(self, experiment_code, control_code):
+ """
+ Compare two HTTP status codes, returns True if they do not match.
+ """
+ if self.http_status_code_match(experiment_code, control_code):
+ return False
+ return True
+
+ def dns_resolve(self, hostname, nameserver=None):
+ """
+ Resolves hostname(s) though nameserver to corresponding
+ address(es). hostname may be either a single hostname string,
+ or a list of strings. If nameserver is not given, use local
+ DNS resolver, and if that fails try using 8.8.8.8.
+ """
+
+ if isinstance(hostname, str):
+ hostname = [hostname]
+
+ if nameserver is not None:
+ res = resolver.Resolver(configure=False)
+ res.nameservers = [nameserver]
+ else:
+ res = resolver.Resolver()
+
+ response = []
+ answer = None
+
+ for hn in hostname:
+ try:
+ answer = res.query(hn)
+ except resolver.NoNameservers:
+ res.nameservers = ['8.8.8.8']
+ try:
+ answer = res.query(hn)
+ except resolver.NXDOMAIN:
+ log.msg("DNS resolution for %s returned NXDOMAIN" % hn)
+ response.append('NXDOMAIN')
+ except resolver.NXDOMAIN:
+ log.msg("DNS resolution for %s returned NXDOMAIN" % hn)
+ response.append('NXDOMAIN')
+ finally:
+ if answer:
+ for addr in answer:
+ response.append(addr.address)
+ return response
+
+ def dns_resolve_match(self, experiment_hostname, control_address):
+ """
+ Resolve experiment_hostname, and check to see that it returns
+ an experiment_address which matches the control_address. If
+ they match, returns True and experiment_address; otherwise
+ returns False and experiment_address.
+ """
+ experiment_address = self.dns_resolve(experiment_hostname)
+ if experiment_address:
+ if len(set(experiment_address) & set([control_address])) > 0:
+ return True, experiment_address
+ else:
+ log.msg("DNS comparison of control '%s' does not" % control_address)
+ log.msg("match experiment response '%s'" % experiment_address)
+ return False, experiment_address
+ else:
+ log.debug("dns_resolve() for %s failed" % experiment_hostname)
+ return None, experiment_address
+
+ def get_auth_nameservers(self, hostname):
+ """
+ Many CPs set a nameserver to be used. Let's query that
+ nameserver for the authoritative nameservers of hostname.
+
+ The equivalent of:
+ $ dig +short NS ooni.nu
+ """
+ res = resolver.Resolver()
+ answer = res.query(hostname, 'NS')
+ auth_nameservers = []
+ for auth in answer:
+ auth_nameservers.append(auth.to_text())
+ return auth_nameservers
+
+ def hostname_to_0x20(self, hostname):
+ """
+ MaKEs yOur HOsTnaME lOoK LiKE THis.
+
+ For more information, see:
+ D. Dagon, et. al. "Increased DNS Forgery Resistance
+ Through 0x20-Bit Encoding". Proc. CSS, 2008.
+ """
+ hostname_0x20 = ''
+ for char in hostname:
+ l33t = random.choice(['caps', 'nocaps'])
+ if l33t == 'caps':
+ hostname_0x20 = hostname_0x20 + char.capitalize()
+ else:
+ hostname_0x20 = hostname_0x20 + char.lower()
+ return hostname_0x20
+
+ def check_0x20_to_auth_ns(self, hostname, sample_size=None):
+ """
+ Resolve a 0x20 DNS request for hostname over hostname's
+ authoritative nameserver(s), and check to make sure that
+ the capitalization in the 0x20 request matches that of the
+ response. Also, check the serial numbers of the SOA (Start
+ of Authority) records on the authoritative nameservers to
+ make sure that they match.
+
+ If sample_size is given, a random sample equal to that number
+ of authoritative nameservers will be queried; default is 5.
+ """
+ log.msg("")
+ log.msg("Testing random capitalization of DNS queries...")
+ log.msg("Testing that Start of Authority serial numbers match...")
+
+ auth_nameservers = self.get_auth_nameservers(hostname)
+
+ if sample_size is None:
+ sample_size = 5
+ resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers),
+ sample_size)
+
+ querynames = []
+ answernames = []
+ serials = []
+
+ # Even when gevent monkey patching is on, the requests here
+ # are sent without being 0x20'd, so we need to 0x20 them.
+ hostname = self.hostname_to_0x20(hostname)
+
+ for auth_ns in resolved_auth_ns:
+ res = resolver.Resolver(configure=False)
+ res.nameservers = [auth_ns]
+ try:
+ answer = res.query(hostname, 'SOA')
+ except resolver.Timeout:
+ continue
+ querynames.append(answer.qname.to_text())
+ answernames.append(answer.rrset.name.to_text())
+ for soa in answer:
+ serials.append(str(soa.serial))
+
+ if len(set(querynames).intersection(answernames)) == 1:
+ log.msg("Capitalization in DNS queries and responses match.")
+ name_match = True
+ else:
+ log.msg("The random capitalization '%s' used in" % hostname)
+ log.msg("DNS queries to that hostname's authoritative")
+ log.msg("nameservers does not match the capitalization in")
+ log.msg("the response.")
+ name_match = False
+
+ if len(set(serials)) == 1:
+ log.msg("Start of Authority serial numbers all match.")
+ serial_match = True
+ else:
+ log.msg("Some SOA serial numbers did not match the rest!")
+ serial_match = False
+
+ ret = name_match, serial_match, querynames, answernames, serials
+
+ if name_match and serial_match:
+ log.msg("Your DNS queries do not appear to be tampered.")
+ return ret
+ elif name_match or serial_match:
+ log.msg("Something is tampering with your DNS queries.")
+ return ret
+ elif not name_match and not serial_match:
+ log.msg("Your DNS queries are definitely being tampered with.")
+ return ret
+
+ def get_random_url_safe_string(self, length):
+ """
+ Returns a random url-safe string of specified length, where
+ 0 < length <= 256. The returned string will always start with
+ an alphabetic character.
+ """
+ if (length <= 0):
+ length = 1
+ elif (length > 256):
+ length = 256
+
+ random_ascii = base64.urlsafe_b64encode(os.urandom(int(length)))
+
+ while not random_ascii[:1].isalpha():
+ random_ascii = base64.urlsafe_b64encode(os.urandom(int(length)))
+
+ three_quarters = int((len(random_ascii)) * (3.0/4.0))
+ random_string = random_ascii[:three_quarters]
+ return random_string
+
+ def get_random_hostname(self, length=None):
+ """
+ Returns a random hostname with SLD of specified length. If
+ length is unspecified, length=32 is used.
+
+ These *should* all resolve to NXDOMAIN. If they actually
+ resolve to a box that isn't part of a captive portal that
+ would be rather interesting.
+ """
+ if length is None:
+ length = 32
+
+ random_sld = self.get_random_url_safe_string(length)
+
+ # if it doesn't start with a letter, chuck it.
+ while not random_sld[:1].isalpha():
+ random_sld = self.get_random_url_safe_string(length)
+
+ tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid']
+ random_tld = urllib2.random.choice(tld_list)
+ random_hostname = random_sld + random_tld
+ return random_hostname
+
+ def compare_random_hostnames(self, hostname_count=None, hostname_length=None):
+ """
+ Get hostname_count number of random hostnames with SLD length
+ of hostname_length, and then attempt DNS resolution. If no
+ arguments are given, default to three hostnames of 32 bytes
+ each. These random hostnames *should* resolve to NXDOMAIN,
+ except in the case where a user is presented with a captive
+ portal and remains unauthenticated, in which case the captive
+ portal may return the address of the authentication page.
+
+ If the cardinality of the intersection of the set of resolved
+ random hostnames and the single element control set
+ (['NXDOMAIN']) are equal to one, then DNS properly resolved.
+
+ Returns true if only NXDOMAINs were returned, otherwise returns
+ False with the relative complement of the control set in the
+ response set.
+ """
+ if hostname_count is None:
+ hostname_count = 3
+
+ log.msg("Generating random hostnames...")
+ log.msg("Resolving DNS for %d random hostnames..." % hostname_count)
+
+ control = ['NXDOMAIN']
+ responses = []
+
+ for x in range(hostname_count):
+ random_hostname = self.get_random_hostname(hostname_length)
+ response_match, response_address = self.dns_resolve_match(random_hostname,
+ control[0])
+ for address in response_address:
+ if response_match is False:
+ log.msg("Strangely, DNS resolution of the random hostname")
+ log.msg("%s actually points to %s"
+ % (random_hostname, response_address))
+ responses = responses + [address]
+ else:
+ responses = responses + [address]
+
+ intersection = set(responses) & set(control)
+ relative_complement = set(responses) - set(control)
+ r = set(responses)
+
+ if len(intersection) == 1:
+ log.msg("All %d random hostnames properly resolved to NXDOMAIN."
+ % hostname_count)
+ return True, relative_complement
+ elif (len(intersection) == 1) and (len(r) > 1):
+ log.msg("Something odd happened. Some random hostnames correctly")
+ log.msg("resolved to NXDOMAIN, but several others resolved to")
+ log.msg("to the following addresses: %s" % relative_complement)
+ return False, relative_complement
+ elif (len(intersection) == 0) and (len(r) == 1):
+ log.msg("All random hostnames resolved to the IP address ")
+ log.msg("'%s', which is indicative of a captive portal." % r)
+ return False, relative_complement
+ else:
+ log.debug("Apparently, pigs are flying on your network, 'cause a")
+ log.debug("bunch of hostnames made from 32-byte random strings")
+ log.debug("just magically resolved to a bunch of random addresses.")
+ log.debug("That is definitely highly improbable. In fact, my napkin")
+ log.debug("tells me that the probability of just one of those")
+ log.degug("hostnames resolving to an address is 1.68e-59, making")
+ log.debug("it nearly twice as unlikely as an MD5 hash collision.")
+ log.debug("Either someone is seriously messing with your network,")
+ log.debug("or else you are witnessing the impossible. %s" % r)
+ return False, relative_complement
+
+ def google_dns_cp_test(self):
+ """
+ Google Chrome resolves three 10-byte random hostnames.
+ """
+ subtest = "Google Chrome DNS-based"
+
+ log.msg("")
+ log.msg("Running the Google Chrome DNS-based captive portal test...")
+
+ gmatch, google_dns_result = self.compare_random_hostnames(3, 10)
+
+ if gmatch:
+ log.msg("Google Chrome DNS-based captive portal test did not")
+ log.msg("detect a captive portal.")
+ return google_dns_result
+ else:
+ log.msg("Google Chrome DNS-based captive portal test believes")
+ log.msg("you are in a captive portal, or else something very")
+ log.msg("odd is happening with your DNS.")
+ return google_dns_result
+
+ def ms_dns_cp_test(self):
+ """
+ Microsoft "phones home" to a server which will always resolve
+ to the same address.
+ """
+ subtest = "Microsoft NCSI DNS-based"
+
+ log.msg("")
+ log.msg("Running the Microsoft NCSI DNS-based captive portal")
+ log.msg("test...")
+
+ msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com",
+ "131.107.255.255")
+ if msmatch:
+ log.msg("Microsoft NCSI DNS-based captive portal test did not")
+ log.msg("detect a captive portal.")
+ return ms_dns_result
+ else:
+ log.msg("Microsoft NCSI DNS-based captive portal test ")
+ log.msg("believes you are in a captive portal.")
+ return ms_dns_result
+
+ def run_vendor_dns_tests(self):
+ """
+ Run the vendor DNS tests.
+ """
+ report = {}
+ report['google_dns_cp'] = self.google_dns_cp_test()
+ report['ms_dns_cp'] = self.ms_dns_cp_test()
+
+ return report
+
+ def run_vendor_tests(self, *a, **kw):
+ """
+ These are several vendor tests used to detect the presence of
+ a captive portal. Each test compares HTTP status code and
+ content to the control results and has its own User-Agent
+ string, in order to emulate the test as it would occur on the
+ device it was intended for. Vendor tests are defined in the
+ format:
+ [exp_url, ctrl_result, ctrl_code, ua, test_name]
+ """
+
+ vendor_tests = [['http://www.apple.com/library/test/success.html',
+ 'Success',
+ '200',
+ 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3',
+ 'Apple HTTP Captive Portal'],
+ ['http://tools.ietf.org/html/draft-nottingham-http-portal-02',
+ '428 Network Authentication Required',
+ '428',
+ 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0',
+ 'W3 Captive Portal'],
+ ['http://www.msftncsi.com/ncsi.txt',
+ 'Microsoft NCSI',
+ '200',
+ 'Microsoft NCSI',
+ 'MS HTTP Captive Portal',]]
+
+ cm = self.http_content_match_fuzzy_opt
+ sm = self.http_status_code_match
+ snm = self.http_status_code_no_match
+
+ def compare_content(status_func, fuzzy, experiment_url, control_result,
+ control_code, headers, test_name):
+ log.msg("")
+ log.msg("Running the %s test..." % test_name)
+
+ content_match, experiment_code, experiment_headers = cm(experiment_url,
+ control_result,
+ headers, fuzzy)
+ status_match = status_func(experiment_code, control_code)
+
+ if status_match and content_match:
+ log.msg("The %s test was unable to detect" % test_name)
+ log.msg("a captive portal.")
+ return True
+ else:
+ log.msg("The %s test shows that your network" % test_name)
+ log.msg("is filtered.")
+ return False
+
+ result = []
+ for vt in vendor_tests:
+ report = {}
+ report['vt'] = vt
+
+ experiment_url = vt[0]
+ control_result = vt[1]
+ control_code = vt[2]
+ headers = {'User-Agent': vt[3]}
+ test_name = vt[4]
+
+ args = (experiment_url, control_result, control_code, headers, test_name)
+
+ if test_name == "MS HTTP Captive Portal":
+ report['result'] = compare_content(sm, False, *args)
+
+ elif test_name == "Apple HTTP Captive Portal":
+ report['result'] = compare_content(sm, True, *args)
+
+ elif test_name == "W3 Captive Portal":
+ report['result'] = compare_content(snm, True, *args)
+
+ else:
+ log.warn("Ooni is trying to run an undefined CP vendor test.")
+ result.append(report)
+ return result
+
+ def control(self, experiment_result, args):
+ """
+ Compares the content and status code of the HTTP response for
+ experiment_url with the control_result and control_code
+ respectively. If the status codes match, but the experimental
+ content and control_result do not match, fuzzy matching is enabled
+ to determine if the control_result is at least included somewhere
+ in the experimental content. Returns True if matches are found,
+ and False if otherwise.
+ """
+ experiment_url = self.local_options['experiment-url']
+ control_result = 'XX'
+ control_code = 200
+ ua = self.local_options['user-agent']
+
+ cm = self.http_content_match_fuzzy_opt
+ sm = self.http_status_code_match
+ snm = self.http_status_code_no_match
+
+ log.msg("Running test for '%s'..." % experiment_url)
+ content_match, experiment_code, experiment_headers = cm(experiment_url,
+ control_result)
+ status_match = sm(experiment_code, control_code)
+ if status_match and content_match:
+ log.msg("The test for '%s'" % experiment_url)
+ log.msg("was unable to detect a captive portal.")
+ return experiment_result, True
+
+ elif status_match and not content_match:
+ log.msg("Retrying '%s' with fuzzy match enabled."
+ % experiment_url)
+ fuzzy_match, experiment_code, experiment_headers = cm(experiment_url,
+ control_result,
+ fuzzy=True)
+ if fuzzy_match:
+ return experiment_result, True
+ else:
+ log.msg("Found modified content on '%s'," % experiment_url)
+ log.msg("which could indicate a captive portal.")
+
+ return experiment_result, False
+ else:
+ log.msg("The content comparison test for ")
+ log.msg("'%s'" % experiment_url)
+ log.msg("shows that your HTTP traffic is filtered.")
+ return experiment_result, False
+
+ def experiment(self, args):
+ """
+ Runs the CaptivePortal(Test).
+
+ CONFIG OPTIONS
+ --------------
+
+ If "do_captive_portal_vendor_tests" is set to "true", then vendor
+ specific captive portal HTTP-based tests will be run.
+
+ If "do_captive_portal_dns_tests" is set to "true", then vendor
+ specific captive portal DNS-based tests will be run.
+
+ If "check_dns_requests" is set to "true", then Ooni-probe will
+ attempt to check that your DNS requests are not being tampered with
+ by a captive portal.
+
+ If "captive_portal" = "yourfilename.txt", then user-specified tests
+ will be run.
+
+ Any combination of the above tests can be run.
+ """
+ report = {}
+
+ log.msg("")
+ log.msg("Running vendor tests...")
+ report['vendor_tests'] = self.run_vendor_tests()
+
+ log.msg("")
+ log.msg("Running vendor DNS-based tests...")
+ report['vendor_dns_tests'] = self.run_vendor_dns_tests()
+
+ log.msg("")
+ log.msg("Checking that DNS requests are not being tampered...")
+ report['check0x20'] = self.check_0x20_to_auth_ns('ooni.nu')
+
+ log.msg("")
+ log.msg("Captive portal test finished!")
+ return report
+
+cp = CaptivePortal(None, None, None)
[View Less]
1
0

09 Jul '12
commit 793f013cf2ca50f415398677dd12ab59ae9c57b4
Merge: 45cc771 428e8e8
Author: Arturo Filastò <art(a)torproject.org>
Date: Mon Jul 9 16:41:48 2012 +0200
Fix the last conflicts, hopefully... argh...
Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe
Conflicts:
ooni/plugins/captiveportal.py
ooni/plugins/captiveportal.py | 7 +++++++
1 files changed, 7 insertions(+), 0 deletions(-)
1
0

09 Jul '12
commit cc86e841928a7a85168143ff524b0779c0e5059a
Author: Sathyanarayanan <gsathya.ceg(a)gmail.com>
Date: Mon Jul 9 16:57:08 2012 +0300
Fail gracefully for old consensus
---
task-6232/pyentropy.py | 7 ++++---
1 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py
index 0ae0f25..23850d1 100644
--- a/task-6232/pyentropy.py
+++ b/task-6232/pyentropy.py
@@ -61,8 +61,8 @@ def run(file_name):
totalExitBW += router.…
[View More]bandwidth
if len(routers) <= 0:
- print "Error: amount of routers must be > 0."
- return;
+ print "Error: Old consensus file. Not able to parse."
+ return
entropy, entropy_exit, entropy_guard = 0.0, 0.0, 0.0
for router in routers:
@@ -92,4 +92,5 @@ if __name__ == "__main__":
with open(sys.argv[2], 'w') as f:
for file_name in os.listdir(sys.argv[1]):
string = run(os.path.join(sys.argv[1], file_name))
- f.write("%s\n" % (string))
+ if string:
+ f.write("%s\n" % (string))
[View Less]
1
0

[metrics-tasks/master] Merge remote-tracking branch 'gsathya/master'
by karsten@torproject.org 09 Jul '12
by karsten@torproject.org 09 Jul '12
09 Jul '12
commit b2d9e509b4c513aa2e253a550fd832fb9c86a94e
Merge: b939e76 cc86e84
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Mon Jul 9 16:16:24 2012 +0200
Merge remote-tracking branch 'gsathya/master'
task-6232/pyentropy.py | 7 ++++---
1 files changed, 4 insertions(+), 3 deletions(-)
1
0
commit 3d4e484df398a5fd395149f1fb87b3b702dd4a2c
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Mon Jul 9 16:16:44 2012 +0200
Take out noisy warning.
---
task-6232/pyentropy.py | 1 -
1 files changed, 0 insertions(+), 1 deletions(-)
diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py
index 23850d1..650423b 100644
--- a/task-6232/pyentropy.py
+++ b/task-6232/pyentropy.py
@@ -61,7 +61,6 @@ def run(file_name):
totalExitBW += router.bandwidth
…
[View More]if len(routers) <= 0:
- print "Error: Old consensus file. Not able to parse."
return
entropy, entropy_exit, entropy_guard = 0.0, 0.0, 0.0
[View Less]
1
0
commit bcf5e79efcc8273a424a6bc668e0b5d6b21f0b0a
Author: Arturo Filastò <art(a)torproject.org>
Date: Mon Jul 9 12:01:12 2012 +0200
Remove dead code.
---
ooni/plugoo/work.py | 81 ---------------------------------------------------
1 files changed, 0 insertions(+), 81 deletions(-)
diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py
index a3fb168..fc338d7 100644
--- a/ooni/plugoo/work.py
+++ b/ooni/plugoo/work.py
@@ -107,84 +107,3 @@ class WorkGenerator(object):
self.…
[View More]idx += 1
return (ret, self.Test, self.idx)
- def p_next(self):
- """
- XXX This is not used for KISS sake.
- """
- if self.end:
- raise StopIteration
-
- if not self.assetGenerator:
- self.end = True
- return WorkUnit(None, self.assetNames, self.Test, self.idx)
-
- # Plank asset
- p_asset = []
- for i in xrange(0, self.size):
- try:
- asset = self.assetGenerator.next()
- p_asset.append(asset)
- print p_asset
- except StopIteration:
- if self.asset_num > 1:
- pass
- self.end = True
- break
-
- self.idx += 1
- return WorkUnit(p_asset, self.assetNames, self.Test, self.idx)
-
-class WorkUnit(object):
- """
- XXX This is currently not implemented for KISS sake.
-
- This is an object responsible for completing WorkUnits it will
- return its result in a deferred.
-
- The execution of a unit of work should be Atomic.
-
- Reporting to the OONI-net happens on completion of a Unit of Work.
-
- @Node node: This represents the node associated with the Work Unit
- @Asset asset: This is the asset associated with the Work Unit
- @Test test: This represents the Test to be with the specified assets
- @ivar arguments: These are the extra attributes to be passsed to the Test
- """
-
- node = None
- asset = None
- test = None
- arguments = None
-
- def __init__(self, asset, assetNames, test, idx):
- self.asset = asset
- if not asset:
- self.assetGenerator = iter([1])
- else:
- self.assetGenerator = iter(asset)
- self.Test = test
- self.assetNames = assetNames
- self.idx = idx
-
- def __iter__(self):
- return self
-
- def __repr__(self):
- return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx)
-
- def serialize(self):
- """
- Serialize this unit of work for RPC activity.
- """
- return yaml.dump(self)
-
- def next(self):
- """
- Launches the Unit of Work with the specified assets on the node.
- """
- try:
- asset = self.assetGenerator.next()
- ret = self.Test.set_asset(asset)
- return ret
- except StopIteration:
- raise StopIteration
[View Less]
1
0

[ooni-probe/master] Port Isis's captive portal test to the new framework.
by art@torproject.org 09 Jul '12
by art@torproject.org 09 Jul '12
09 Jul '12
commit 428e8e827e6f457aaf987be753b4f99c5127f750
Author: Arturo Filastò <art(a)torproject.org>
Date: Mon Jul 9 12:41:49 2012 +0200
Port Isis's captive portal test to the new framework.
---
ooni/plugins/captiveportal.py | 625 +++++++++++++++++++++++++++++++++++++++++
1 files changed, 625 insertions(+), 0 deletions(-)
diff --git a/ooni/plugins/captiveportal.py b/ooni/plugins/captiveportal.py
new file mode 100644
index 0000000..9b7bee3
--- /dev/null
+++ b/ooni/plugins/captiveportal.…
[View More]py
@@ -0,0 +1,625 @@
+# -*- coding: utf-8 -*-
+"""
+ captiveportal
+ *************
+
+ This test is a collection of tests to detect the presence of a
+ captive portal. Code is taken, in part, from the old ooni-probe,
+ which was written by Jacob Appelbaum and Arturo Filastò.
+
+ :copyright: (c) 2012 Isis Lovecruft
+ :license: see LICENSE for more details
+"""
+import base64
+import os
+import random
+import re
+import string
+import urllib2
+from urlparse import urlparse
+
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+
+from ooni.plugoo.assets import Asset
+from ooni.plugoo.tests import ITest, OONITest
+from ooni.protocols import http
+from ooni import log
+
+try:
+ from dns import resolver
+except ImportError:
+ print "The dnspython module was not found. https://crate.io/packages/dnspython/"
+
+__plugoo__ = "captiveportal"
+__desc__ = "Captive portal detection test"
+
+class CaptivePortalArgs(usage.Options):
+ optParameters = [['asset', 'a', None, 'Asset file'],
+ ['resume', 'r', 0, 'Resume at this index'],
+ ['experiment-url', 'e', 'http://google.com/', 'Experiment URL'],
+ ['user-agent', 'u', random.choice(http.useragents),
+ 'User agent for HTTP requests']
+ ]
+
+class CaptivePortal(OONITest):
+ """
+ Compares content and status codes of HTTP responses, and attempts
+ to determine if content has been altered.
+ """
+
+ implements(IPlugin, ITest)
+
+ shortName = "captivep"
+ description = "Captive Portal Test"
+ requirements = None
+ options = CaptivePortalArgs
+ # Tells this to be blocking.
+ blocking = True
+
+ def http_fetch(self, url, headers={}):
+ """
+ Parses an HTTP url, fetches it, and returns a urllib2 response
+ object.
+ """
+ url = urlparse(url).geturl()
+ request = urllib2.Request(url, None, headers)
+ response = urllib2.urlopen(request)
+ response_headers = dict(response.headers)
+ return response, response_headers
+
+ def http_content_match_fuzzy_opt(self, experimental_url, control_result,
+ headers=None, fuzzy=False):
+ """
+ Makes an HTTP request on port 80 for experimental_url, then
+ compares the response_content of experimental_url with the
+ control_result. Optionally, if the fuzzy parameter is set to
+ True, the response_content is compared with a regex of the
+ control_result. If the response_content from the
+ experimental_url and the control_result match, returns True
+ with the HTTP status code and headers; False, status code, and
+ headers if otherwise.
+ """
+
+ if headers is None:
+ default_ua = self.local_options['user-agent']
+ headers = {'User-Agent': default_ua}
+
+ response, response_headers = self.http_fetch(experimental_url, headers)
+ response_content = response.read()
+ response_code = response.code
+ if response_content is not None:
+ if fuzzy:
+ pattern = re.compile(control_result)
+ match = pattern.search(response_content)
+ if not match:
+ log.msg("Fuzzy HTTP content comparison for experiment URL")
+ log.msg("'%s'" % experimental_url)
+ log.msg("does not match!")
+ return False, response_code, response_headers
+ else:
+ log.msg("Fuzzy HTTP content comparison of experiment URL")
+ log.msg("'%s'" % experimental_url)
+ log.msg("and the expected control result yielded a match.")
+ return True, response_code, response_headers
+ else:
+ if str(response_content) != str(control_result):
+ log.msg("HTTP content comparison of experiment URL")
+ log.msg("'%s'" % experimental_url)
+ log.msg("and the expected control result do not match.")
+ return False, response_code, response_headers
+ else:
+ return True, response_code, response_headers
+ else:
+ log.warn("HTTP connection appears to have failed.")
+ return False, False, False
+
+ def http_status_code_match(self, experiment_code, control_code):
+ """
+ Compare two HTTP status codes, returns True if they match.
+ """
+ if int(experiment_code) != int(control_code):
+ return False
+ return True
+
+ def http_status_code_no_match(self, experiment_code, control_code):
+ """
+ Compare two HTTP status codes, returns True if they do not match.
+ """
+ if self.http_status_code_match(experiment_code, control_code):
+ return False
+ return True
+
+ def dns_resolve(self, hostname, nameserver=None):
+ """
+ Resolves hostname(s) though nameserver to corresponding
+ address(es). hostname may be either a single hostname string,
+ or a list of strings. If nameserver is not given, use local
+ DNS resolver, and if that fails try using 8.8.8.8.
+ """
+
+ if isinstance(hostname, str):
+ hostname = [hostname]
+
+ if nameserver is not None:
+ res = resolver.Resolver(configure=False)
+ res.nameservers = [nameserver]
+ else:
+ res = resolver.Resolver()
+
+ response = []
+ answer = None
+
+ for hn in hostname:
+ try:
+ answer = res.query(hn)
+ except resolver.NoNameservers:
+ res.nameservers = ['8.8.8.8']
+ try:
+ answer = res.query(hn)
+ except resolver.NXDOMAIN:
+ log.msg("DNS resolution for %s returned NXDOMAIN" % hn)
+ response.append('NXDOMAIN')
+ except resolver.NXDOMAIN:
+ log.msg("DNS resolution for %s returned NXDOMAIN" % hn)
+ response.append('NXDOMAIN')
+ finally:
+ if answer:
+ for addr in answer:
+ response.append(addr.address)
+ return response
+
+ def dns_resolve_match(self, experiment_hostname, control_address):
+ """
+ Resolve experiment_hostname, and check to see that it returns
+ an experiment_address which matches the control_address. If
+ they match, returns True and experiment_address; otherwise
+ returns False and experiment_address.
+ """
+ experiment_address = self.dns_resolve(experiment_hostname)
+ if experiment_address:
+ if len(set(experiment_address) & set([control_address])) > 0:
+ return True, experiment_address
+ else:
+ log.msg("DNS comparison of control '%s' does not" % control_address)
+ log.msg("match experiment response '%s'" % experiment_address)
+ return False, experiment_address
+ else:
+ log.debug("dns_resolve() for %s failed" % experiment_hostname)
+ return None, experiment_address
+
+ def get_auth_nameservers(self, hostname):
+ """
+ Many CPs set a nameserver to be used. Let's query that
+ nameserver for the authoritative nameservers of hostname.
+
+ The equivalent of:
+ $ dig +short NS ooni.nu
+ """
+ res = resolver.Resolver()
+ answer = res.query(hostname, 'NS')
+ auth_nameservers = []
+ for auth in answer:
+ auth_nameservers.append(auth.to_text())
+ return auth_nameservers
+
+ def hostname_to_0x20(self, hostname):
+ """
+ MaKEs yOur HOsTnaME lOoK LiKE THis.
+
+ For more information, see:
+ D. Dagon, et. al. "Increased DNS Forgery Resistance
+ Through 0x20-Bit Encoding". Proc. CSS, 2008.
+ """
+ hostname_0x20 = ''
+ for char in hostname:
+ l33t = random.choice(['caps', 'nocaps'])
+ if l33t == 'caps':
+ hostname_0x20 = hostname_0x20 + char.capitalize()
+ else:
+ hostname_0x20 = hostname_0x20 + char.lower()
+ return hostname_0x20
+
+ def check_0x20_to_auth_ns(self, hostname, sample_size=None):
+ """
+ Resolve a 0x20 DNS request for hostname over hostname's
+ authoritative nameserver(s), and check to make sure that
+ the capitalization in the 0x20 request matches that of the
+ response. Also, check the serial numbers of the SOA (Start
+ of Authority) records on the authoritative nameservers to
+ make sure that they match.
+
+ If sample_size is given, a random sample equal to that number
+ of authoritative nameservers will be queried; default is 5.
+ """
+ log.msg("")
+ log.msg("Testing random capitalization of DNS queries...")
+ log.msg("Testing that Start of Authority serial numbers match...")
+
+ auth_nameservers = self.get_auth_nameservers(hostname)
+
+ if sample_size is None:
+ sample_size = 5
+ resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers),
+ sample_size)
+
+ querynames = []
+ answernames = []
+ serials = []
+
+ # Even when gevent monkey patching is on, the requests here
+ # are sent without being 0x20'd, so we need to 0x20 them.
+ hostname = self.hostname_to_0x20(hostname)
+
+ for auth_ns in resolved_auth_ns:
+ res = resolver.Resolver(configure=False)
+ res.nameservers = [auth_ns]
+ try:
+ answer = res.query(hostname, 'SOA')
+ except resolver.Timeout:
+ continue
+ querynames.append(answer.qname.to_text())
+ answernames.append(answer.rrset.name.to_text())
+ for soa in answer:
+ serials.append(str(soa.serial))
+
+ if len(set(querynames).intersection(answernames)) == 1:
+ log.msg("Capitalization in DNS queries and responses match.")
+ name_match = True
+ else:
+ log.msg("The random capitalization '%s' used in" % hostname)
+ log.msg("DNS queries to that hostname's authoritative")
+ log.msg("nameservers does not match the capitalization in")
+ log.msg("the response.")
+ name_match = False
+
+ if len(set(serials)) == 1:
+ log.msg("Start of Authority serial numbers all match.")
+ serial_match = True
+ else:
+ log.msg("Some SOA serial numbers did not match the rest!")
+ serial_match = False
+
+ ret = name_match, serial_match, querynames, answernames, serials
+
+ if name_match and serial_match:
+ log.msg("Your DNS queries do not appear to be tampered.")
+ return ret
+ elif name_match or serial_match:
+ log.msg("Something is tampering with your DNS queries.")
+ return ret
+ elif not name_match and not serial_match:
+ log.msg("Your DNS queries are definitely being tampered with.")
+ return ret
+
+ def get_random_url_safe_string(self, length):
+ """
+ Returns a random url-safe string of specified length, where
+ 0 < length <= 256. The returned string will always start with
+ an alphabetic character.
+ """
+ if (length <= 0):
+ length = 1
+ elif (length > 256):
+ length = 256
+
+ random_ascii = base64.urlsafe_b64encode(os.urandom(int(length)))
+
+ while not random_ascii[:1].isalpha():
+ random_ascii = base64.urlsafe_b64encode(os.urandom(int(length)))
+
+ three_quarters = int((len(random_ascii)) * (3.0/4.0))
+ random_string = random_ascii[:three_quarters]
+ return random_string
+
+ def get_random_hostname(self, length=None):
+ """
+ Returns a random hostname with SLD of specified length. If
+ length is unspecified, length=32 is used.
+
+ These *should* all resolve to NXDOMAIN. If they actually
+ resolve to a box that isn't part of a captive portal that
+ would be rather interesting.
+ """
+ if length is None:
+ length = 32
+
+ random_sld = self.get_random_url_safe_string(length)
+
+ # if it doesn't start with a letter, chuck it.
+ while not random_sld[:1].isalpha():
+ random_sld = self.get_random_url_safe_string(length)
+
+ tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid']
+ random_tld = urllib2.random.choice(tld_list)
+ random_hostname = random_sld + random_tld
+ return random_hostname
+
+ def compare_random_hostnames(self, hostname_count=None, hostname_length=None):
+ """
+ Get hostname_count number of random hostnames with SLD length
+ of hostname_length, and then attempt DNS resolution. If no
+ arguments are given, default to three hostnames of 32 bytes
+ each. These random hostnames *should* resolve to NXDOMAIN,
+ except in the case where a user is presented with a captive
+ portal and remains unauthenticated, in which case the captive
+ portal may return the address of the authentication page.
+
+ If the cardinality of the intersection of the set of resolved
+ random hostnames and the single element control set
+ (['NXDOMAIN']) are equal to one, then DNS properly resolved.
+
+ Returns true if only NXDOMAINs were returned, otherwise returns
+ False with the relative complement of the control set in the
+ response set.
+ """
+ if hostname_count is None:
+ hostname_count = 3
+
+ log.msg("Generating random hostnames...")
+ log.msg("Resolving DNS for %d random hostnames..." % hostname_count)
+
+ control = ['NXDOMAIN']
+ responses = []
+
+ for x in range(hostname_count):
+ random_hostname = self.get_random_hostname(hostname_length)
+ response_match, response_address = self.dns_resolve_match(random_hostname,
+ control[0])
+ for address in response_address:
+ if response_match is False:
+ log.msg("Strangely, DNS resolution of the random hostname")
+ log.msg("%s actually points to %s"
+ % (random_hostname, response_address))
+ responses = responses + [address]
+ else:
+ responses = responses + [address]
+
+ intersection = set(responses) & set(control)
+ relative_complement = set(responses) - set(control)
+ r = set(responses)
+
+ if len(intersection) == 1:
+ log.msg("All %d random hostnames properly resolved to NXDOMAIN."
+ % hostname_count)
+ return True, relative_complement
+ elif (len(intersection) == 1) and (len(r) > 1):
+ log.msg("Something odd happened. Some random hostnames correctly")
+ log.msg("resolved to NXDOMAIN, but several others resolved to")
+ log.msg("to the following addresses: %s" % relative_complement)
+ return False, relative_complement
+ elif (len(intersection) == 0) and (len(r) == 1):
+ log.msg("All random hostnames resolved to the IP address ")
+ log.msg("'%s', which is indicative of a captive portal." % r)
+ return False, relative_complement
+ else:
+ log.debug("Apparently, pigs are flying on your network, 'cause a")
+ log.debug("bunch of hostnames made from 32-byte random strings")
+ log.debug("just magically resolved to a bunch of random addresses.")
+ log.debug("That is definitely highly improbable. In fact, my napkin")
+ log.debug("tells me that the probability of just one of those")
+ log.degug("hostnames resolving to an address is 1.68e-59, making")
+ log.debug("it nearly twice as unlikely as an MD5 hash collision.")
+ log.debug("Either someone is seriously messing with your network,")
+ log.debug("or else you are witnessing the impossible. %s" % r)
+ return False, relative_complement
+
+ def google_dns_cp_test(self):
+ """
+ Google Chrome resolves three 10-byte random hostnames.
+ """
+ subtest = "Google Chrome DNS-based"
+
+ log.msg("")
+ log.msg("Running the Google Chrome DNS-based captive portal test...")
+
+ gmatch, google_dns_result = self.compare_random_hostnames(3, 10)
+
+ if gmatch:
+ log.msg("Google Chrome DNS-based captive portal test did not")
+ log.msg("detect a captive portal.")
+ return google_dns_result
+ else:
+ log.msg("Google Chrome DNS-based captive portal test believes")
+ log.msg("you are in a captive portal, or else something very")
+ log.msg("odd is happening with your DNS.")
+ return google_dns_result
+
+ def ms_dns_cp_test(self):
+ """
+ Microsoft "phones home" to a server which will always resolve
+ to the same address.
+ """
+ subtest = "Microsoft NCSI DNS-based"
+
+ log.msg("")
+ log.msg("Running the Microsoft NCSI DNS-based captive portal")
+ log.msg("test...")
+
+ msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com",
+ "131.107.255.255")
+ if msmatch:
+ log.msg("Microsoft NCSI DNS-based captive portal test did not")
+ log.msg("detect a captive portal.")
+ return ms_dns_result
+ else:
+ log.msg("Microsoft NCSI DNS-based captive portal test ")
+ log.msg("believes you are in a captive portal.")
+ return ms_dns_result
+
+ def run_vendor_dns_tests(self):
+ """
+ Run the vendor DNS tests.
+ """
+ report = {}
+ report['google_dns_cp'] = self.google_dns_cp_test()
+ report['ms_dns_cp'] = self.ms_dns_cp_test()
+
+ return report
+
+ def run_vendor_tests(self, *a, **kw):
+ """
+ These are several vendor tests used to detect the presence of
+ a captive portal. Each test compares HTTP status code and
+ content to the control results and has its own User-Agent
+ string, in order to emulate the test as it would occur on the
+ device it was intended for. Vendor tests are defined in the
+ format:
+ [exp_url, ctrl_result, ctrl_code, ua, test_name]
+ """
+
+ vendor_tests = [['http://www.apple.com/library/test/success.html',
+ 'Success',
+ '200',
+ 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3',
+ 'Apple HTTP Captive Portal'],
+ ['http://tools.ietf.org/html/draft-nottingham-http-portal-02',
+ '428 Network Authentication Required',
+ '428',
+ 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0',
+ 'W3 Captive Portal'],
+ ['http://www.msftncsi.com/ncsi.txt',
+ 'Microsoft NCSI',
+ '200',
+ 'Microsoft NCSI',
+ 'MS HTTP Captive Portal',]]
+
+ cm = self.http_content_match_fuzzy_opt
+ sm = self.http_status_code_match
+ snm = self.http_status_code_no_match
+
+ def compare_content(status_func, fuzzy, experiment_url, control_result,
+ control_code, headers, test_name):
+ log.msg("")
+ log.msg("Running the %s test..." % test_name)
+
+ content_match, experiment_code, experiment_headers = cm(experiment_url,
+ control_result,
+ headers, fuzzy)
+ status_match = status_func(experiment_code, control_code)
+
+ if status_match and content_match:
+ log.msg("The %s test was unable to detect" % test_name)
+ log.msg("a captive portal.")
+ return True
+ else:
+ log.msg("The %s test shows that your network" % test_name)
+ log.msg("is filtered.")
+ return False
+
+ result = []
+ for vt in vendor_tests:
+ report = {}
+ report['vt'] = vt
+
+ experiment_url = vt[0]
+ control_result = vt[1]
+ control_code = vt[2]
+ headers = {'User-Agent': vt[3]}
+ test_name = vt[4]
+
+ args = (experiment_url, control_result, control_code, headers, test_name)
+
+ if test_name == "MS HTTP Captive Portal":
+ report['result'] = compare_content(sm, False, *args)
+
+ elif test_name == "Apple HTTP Captive Portal":
+ report['result'] = compare_content(sm, True, *args)
+
+ elif test_name == "W3 Captive Portal":
+ report['result'] = compare_content(snm, True, *args)
+
+ else:
+ log.warn("Ooni is trying to run an undefined CP vendor test.")
+ result.append(report)
+ return result
+
+ def control(self, experiment_result, args):
+ """
+ Compares the content and status code of the HTTP response for
+ experiment_url with the control_result and control_code
+ respectively. If the status codes match, but the experimental
+ content and control_result do not match, fuzzy matching is enabled
+ to determine if the control_result is at least included somewhere
+ in the experimental content. Returns True if matches are found,
+ and False if otherwise.
+ """
+ experiment_url = self.local_options['experiment-url']
+ control_result = 'XX'
+ control_code = 200
+ ua = self.local_options['user-agent']
+
+ cm = self.http_content_match_fuzzy_opt
+ sm = self.http_status_code_match
+ snm = self.http_status_code_no_match
+
+ log.msg("Running test for '%s'..." % experiment_url)
+ content_match, experiment_code, experiment_headers = cm(experiment_url,
+ control_result)
+ status_match = sm(experiment_code, control_code)
+ if status_match and content_match:
+ log.msg("The test for '%s'" % experiment_url)
+ log.msg("was unable to detect a captive portal.")
+ return experiment_result, True
+
+ elif status_match and not content_match:
+ log.msg("Retrying '%s' with fuzzy match enabled."
+ % experiment_url)
+ fuzzy_match, experiment_code, experiment_headers = cm(experiment_url,
+ control_result,
+ fuzzy=True)
+ if fuzzy_match:
+ return experiment_result, True
+ else:
+ log.msg("Found modified content on '%s'," % experiment_url)
+ log.msg("which could indicate a captive portal.")
+
+ return experiment_result, False
+ else:
+ log.msg("The content comparison test for ")
+ log.msg("'%s'" % experiment_url)
+ log.msg("shows that your HTTP traffic is filtered.")
+ return experiment_result, False
+
+ def experiment(self, args):
+ """
+ Runs the CaptivePortal(Test).
+
+ CONFIG OPTIONS
+ --------------
+
+ If "do_captive_portal_vendor_tests" is set to "true", then vendor
+ specific captive portal HTTP-based tests will be run.
+
+ If "do_captive_portal_dns_tests" is set to "true", then vendor
+ specific captive portal DNS-based tests will be run.
+
+ If "check_dns_requests" is set to "true", then Ooni-probe will
+ attempt to check that your DNS requests are not being tampered with
+ by a captive portal.
+
+ If "captive_portal" = "yourfilename.txt", then user-specified tests
+ will be run.
+
+ Any combination of the above tests can be run.
+ """
+ report = {}
+
+ log.msg("")
+ log.msg("Running vendor tests...")
+ report['vendor_tests'] = self.run_vendor_tests()
+
+ log.msg("")
+ log.msg("Running vendor DNS-based tests...")
+ report['vendor_dns_tests'] = self.run_vendor_dns_tests()
+
+ log.msg("")
+ log.msg("Checking that DNS requests are not being tampered...")
+ report['check0x20'] = self.check_0x20_to_auth_ns('ooni.nu')
+
+ log.msg("")
+ log.msg("Captive portal test finished!")
+ return report
+
+cp = CaptivePortal(None, None, None)
[View Less]
1
0