[tor-commits] [ooni-probe/master] * Added gevent monkey patching for SSL
art at torproject.org
art at torproject.org
Mon Jul 9 14:39:04 UTC 2012
commit 98d62ca235a260ab0e0ca708098200dec33168a6
Author: Isis Lovecruft <isis at patternsinthevoid.net>
Date: Fri Apr 13 20:31:10 2012 -0700
* Added gevent monkey patching for SSL
* Fixed NoneType headers for SSL fetches
* Disabled asynchronous select function becauses dnspython requires blocking select
* Added horribly ugly multiple exception handling for DNS queries
* Changed the logic of compare_random_hostnames() to handle cases where they don't properly resolve to NXDOMAIN
* tally_mark system is still broken
---
tests/captiveportal.py | 115 ++++++++++++++++++++++++++++++++----------------
1 files changed, 77 insertions(+), 38 deletions(-)
diff --git a/tests/captiveportal.py b/tests/captiveportal.py
index e74490c..2ca7a18 100644
--- a/tests/captiveportal.py
+++ b/tests/captiveportal.py
@@ -27,7 +27,8 @@ except ImportError:
try:
from gevent import monkey
- monkey.patch_socket(dns=False)
+ monkey.patch_all(socket=True, dns=False, time=True, select=False, thread=True,
+ os=True, ssl=True, httplib=False, aggressive=True)
except ImportError:
print "The gevent module was not found. https://crate.io/packages/gevent/"
@@ -62,7 +63,7 @@ class CaptivePortal(Test):
Test.__init__(self, ooni, name='test')
self.default_ua = ooni.config.tests.default_ua
- def http_fetch(self, url, headers=None):
+ def http_fetch(self, url, headers={}):
"""
Parses an HTTP url, fetches it, and returns a urllib2 response
object.
@@ -134,6 +135,13 @@ class CaptivePortal(Test):
return False
return True
+ def http_headers(self, experiment_url):
+ """
+ Return only the headers of an HTTP response.
+ """
+ response = self.http_fetch(url)
+ return response.headers
+
def dns_resolve(self, hostname, nameserver=None):
"""
Resolves hostname though nameserver ns to its corresponding
@@ -147,18 +155,40 @@ class CaptivePortal(Test):
else:
res = resolver.Resolver()
+ def __addresses__(hostname):
+ answer = res.query(hostname)
+ response = []
+ for addr in answer:
+ response.append(addr.address)
+ return response
+
+ # This is gross and needs to be cleaned up, but it
+ # was the best way I could find to handle all the
+ # exceptions properly.
try:
answer = res.query(hostname)
response = []
for addr in answer:
response.append(addr.address)
+ #response = __addresses__(hostname)
return response
- except resolver.NXDOMAIN as e:
+ except resolver.NoNameservers as nns:
+ res.nameservers = ['8.8.8.8']
+ try:
+ answer = res.query(hostname)
+ response = []
+ for addr in answer:
+ response.append(addr.address)
+ #response = __addresses__(hostname)
+ return response
+ except resolver.NXDOMAIN as nx:
+ log.info("DNS resolution for %s returned NXDOMAIN" % hostname)
+ response = ['NXDOMAIN']
+ return response
+ except resolver.NXDOMAIN as nx:
log.info("DNS resolution for %s returned NXDOMAIN" % hostname)
response = ['NXDOMAIN']
return response
- except:
- return False
def dns_resolve_match(self, experiment_hostname, control_address):
"""
@@ -178,7 +208,8 @@ class CaptivePortal(Test):
"experiment response '%s'" % control_address, address)
return False, experiment_address
else:
- return None
+ log.debug("dns_resolve() for %s failed" % experiment_hostname)
+ return None, experiment_address
def get_random_url_safe_string(self, length):
"""
@@ -258,13 +289,14 @@ class CaptivePortal(Test):
random_hostname = self.get_random_hostname(hostname_length)
response_match, response_address = self.dns_resolve_match(random_hostname,
control[0])
- if response_match is False:
- log.info("Strangely, DNS resolution of the random hostname")
- log.ingo("%s actually points to %s"
- % (random_hostname, response_address))
- responses = responses + response_address
- else:
- responses = responses + response_address
+ for address in response_address:
+ if response_match is False:
+ log.info("Strangely, DNS resolution of the random hostname")
+ log.info("%s actually points to %s"
+ % (random_hostname, response_address))
+ responses = responses + [address]
+ else:
+ responses = responses + [address]
intersection = set(responses) & set(control)
relative_complement = set(responses) - set(control)
@@ -275,24 +307,24 @@ class CaptivePortal(Test):
% hostname_count)
return True, relative_complement
elif (len(intersection) == 1) and (len(r) > 1):
- log.info("Something odd happened. Some random hostnames correctly " \
- "resolved to NXDOMAIN, but several others resolved " \
- "to the following addresses: %s" % relative_complement)
+ log.info("Something odd happened. Some random hostnames correctly")
+ log.info("resolved to NXDOMAIN, but several others resolved to")
+ log.info("to the following addresses: %s" % relative_complement)
return False, relative_complement
elif (len(intersection) == 0) and (len(r) == 1):
log.info("All random hostnames resolved to the IP address ")
log.info("'%s', which is indicative of a captive portal." % r)
return False, relative_complement
else:
- log.debug("Apparently, pigs are flying on your network, 'cause a " \
- "bunch of hostnames made from 32-byte random strings " \
- "just magically resolved to a bunch of random addresses. " \
- "That is definitely highly improbable. In fact, my napkin " \
- "tells me that the probability of just one of those " \
- "hostnames resolving to an address is 1.68e-59, making" \
- "it nearly twice as unlikely as an MD5 hash collision. " \
- "Either someone is seriously messing with your network, " \
- "or else you are witnessing the impossible. %s" % r)
+ log.debug("Apparently, pigs are flying on your network, 'cause a")
+ log.debug("bunch of hostnames made from 32-byte random strings")
+ log.debug("just magically resolved to a bunch of random addresses.")
+ log.debug("That is definitely highly improbable. In fact, my napkin")
+ log.debug("tells me that the probability of just one of those")
+ log.degug("hostnames resolving to an address is 1.68e-59, making")
+ log.debug("it nearly twice as unlikely as an MD5 hash collision.")
+ log.debug("Either someone is seriously messing with your network,")
+ log.debug("or else you are witnessing the impossible. %s" % r)
return False, relative_complement
def google_dns_cp_test(self):
@@ -342,6 +374,7 @@ class CaptivePortal(Test):
"""
self.google_dns_cp_test()
self.ms_dns_cp_test()
+
return
def run_vendor_tests(self, *a, **kw):
@@ -380,8 +413,10 @@ class CaptivePortal(Test):
test_name, fuzzy):
log.info("")
log.info("Running the %s test..." % test_name)
+
content_match, exp_code = cm(exp_url, ctrl_result, headers, fuzzy)
status_match = status_func(exp_code, ctrl_code)
+
if status_match and content_match:
log.info("The %s test was unable to detect " % test_name)
log.info("a captive portal.")
@@ -441,8 +476,8 @@ class CaptivePortal(Test):
log = self.logger
- #tally = kw['tally']
- #tally_marks = kw['tally_marks']
+ tally = kw['tally']
+ tally_marks = kw['tally_marks']
if test_name == "user-defined":
log.info("Running %s test for '%s'..." % (test_name, experiment_url))
@@ -465,17 +500,17 @@ class CaptivePortal(Test):
log.info("which could indicate a captive portal.")
## TODO return exp_content and compare HTTP headers
- #tally = tally + 1
- #tally_marks.append([experiment_url, experiment_code,
- # control_result, control_code])
+ tally = tally + 1
+ tally_marks.append([experiment_url, experiment_code,
+ control_result, control_code])
return False, test_name
else:
log.info("The content comparison test for ")
log.info("'%s'" % experiment_url)
log.info("shows that your HTTP traffic is filtered.")
- #tally = tally + 1
- #tally_marks.append([experiment_url, experiment_code,
- # control_result, control_code])
+ tally = tally + 1
+ tally_marks.append([experiment_url, experiment_code,
+ control_result, control_code])
return False, test_name
else:
@@ -512,6 +547,9 @@ def run(ooni):
Either vendor tests or user-defined tests can be run, or both.
"""
+ #tally = Storage(tally=0)
+ #tally_marks = Storage(tally_marks=[])
+
config = ooni.config
log = ooni.logger
@@ -523,19 +561,20 @@ def run(ooni):
captiveportal = CaptivePortal(ooni)
log.info("Starting captive portal test...")
- captiveportal.run(assets, {'index': 1})
- #captiveportal.run(assets, {'index': 1, 'tally': tally,
- # 'tally_marks': tally_marks})
+ #captiveportal.run(assets, {'index': 1})
+ captiveportal.run(assets, {'index': 1, 'tally': 0,
+ 'tally_marks': []})
if config.tests.do_captive_portal_vendor_tests:
+ log.info("")
log.info("Running vendor tests...")
captiveportal.run_vendor_tests()
if config.tests.do_captive_portal_vendor_dns_tests:
+ log.info("")
log.info("Running vendor DNS-based tests...")
captiveportal.run_vendor_dns_tests()
- #captiveportal.confirmed_kill_count({'tally': tally,
- # 'tally_marks': tally_marks})
+ #captiveportal.confirmed_kill_count()
log.info("Captive portal test finished!")
More information about the tor-commits
mailing list