March 2014 - tor-commits - lists.torproject.org

[ooni-probe/master] Removed unnecessary print
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 03a64de8f9aa72215db0c55803e612d0ab030bde Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Sun Mar 23 07:31:53 2014 +0530 Removed unnecessary print --- ooni/nettests/manipulation/captiveportal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index 49d5b8d..443b2ed 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ … [View More]

1 0

[ooni-probe/master] Replaced dnspython with ooni.templates.dnst for performing A Lookups in
by art＠torproject.org 31 Mar '14

31 Mar '14

commit b4643e05efb4c75315b0893f0a7748948caf7c45 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Sun Mar 23 07:29:39 2014 +0530 Replaced dnspython with ooni.templates.dnst for performing A Lookups in Captiveportal test --- ooni/nettests/manipulation/captiveportal.py | 89 ++++++++++++++------------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index 6862177..… [View More]49d5b8d 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -35,6 +35,7 @@ import re import string from urlparse import urlparse +from twisted.names import error from twisted.python import usage from twisted.internet import defer, threads @@ -154,6 +155,7 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): """ return int(experiment_code) != int(control_code) + @defer.inlineCallbacks def dns_resolve(self, hostname, nameserver=None): """ Resolves hostname(s) though nameserver to corresponding @@ -161,42 +163,29 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): or a list of strings. If nameserver is not given, use local DNS resolver, and if that fails try using 8.8.8.8. """ - if not resolver: - log.msg("dnspython is not installed.\ - Cannot perform DNS Resolve test") - return [] if isinstance(hostname, str): hostname = [hostname] - if nameserver is not None: - res = resolver.Resolver(configure=False) - res.nameservers = [nameserver] - else: - res = resolver.Resolver() - response = [] answer = None - for hn in hostname: try: - answer = res.query(hn) - except resolver.NoNameservers: - res.nameservers = ['8.8.8.8'] - try: - answer = res.query(hn) - except resolver.NXDOMAIN: + answer = yield self.performALookup(hn) + if not answer: + answer = yield self.performALookup(hn, ('8.8.8.8',53)) + except error.DNSNameError: log.msg("DNS resolution for %s returned NXDOMAIN" % hn) response.append('NXDOMAIN') - except resolver.NXDOMAIN: - log.msg("DNS resolution for %s returned NXDOMAIN" % hn) - response.append('NXDOMAIN') + except Exception: + log.err("DNS Resolution failed") finally: if not answer: - return response + defer.returnValue(response) for addr in answer: - response.append(addr.address) - return response + response.append(addr) + defer.returnValue(response) + @defer.inlineCallbacks def dns_resolve_match(self, experiment_hostname, control_address): """ Resolve experiment_hostname, and check to see that it returns @@ -204,17 +193,21 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): they match, returns True and experiment_address; otherwise returns False and experiment_address. """ - experiment_address = self.dns_resolve(experiment_hostname) + experiment_address = yield self.dns_resolve(experiment_hostname) if not experiment_address: log.debug("dns_resolve() for %s failed" % experiment_hostname) - return None, experiment_address + ret = None, experiment_address + defer.returnValue(ret) if len(set(experiment_address) & set([control_address])) > 0: - return True, experiment_address + ret = True, experiment_address + defer.returnValue(ret) else: log.msg("DNS comparison of control '%s' does not" % control_address) log.msg("match experiment response '%s'" % experiment_address) - return False, experiment_address + ret = False, experiment_address + defer.returnValue(ret) + @defer.inlineCallbacks def get_auth_nameservers(self, hostname): """ @@ -266,8 +259,8 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): if sample_size is None: sample_size = 5 - resolved_auth_ns = random.sample(self.dns_resolve(auth_nameservers), - sample_size) + res = yield self.dns_resolve(auth_nameservers) + resolved_auth_ns = random.sample(res,sample_size) querynames = [] answernames = [] @@ -361,6 +354,7 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): random_hostname = random_sld + random_tld return random_hostname + @defer.inlineCallbacks def compare_random_hostnames(self, hostname_count=None, hostname_length=None): """ Get hostname_count number of random hostnames with SLD length @@ -390,7 +384,7 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): for x in range(hostname_count): random_hostname = self.get_random_hostname(hostname_length) - response_match, response_address = self.dns_resolve_match(random_hostname, + response_match, response_address = yield self.dns_resolve_match(random_hostname, control[0]) for address in response_address: if response_match is False: @@ -408,16 +402,19 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): if len(intersection) == 1: log.msg("All %d random hostnames properly resolved to NXDOMAIN." % hostname_count) - return True, relative_complement + ret = True, relative_complement + defer.returnValue(ret) elif (len(intersection) == 1) and (len(r) > 1): log.msg("Something odd happened. Some random hostnames correctly") log.msg("resolved to NXDOMAIN, but several others resolved to") log.msg("to the following addresses: %s" % relative_complement) - return False, relative_complement + ret = False, relative_complement + defer.returnValue(ret) elif (len(intersection) == 0) and (len(r) == 1): log.msg("All random hostnames resolved to the IP address ") log.msg("'%s', which is indicative of a captive portal." % r) - return False, relative_complement + ret = False, relative_complement + defer.returnValue(ret) else: log.debug("Apparently, pigs are flying on your network, 'cause a") log.debug("bunch of hostnames made from 32-byte random strings") @@ -428,8 +425,10 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): log.debug("it nearly twice as unlikely as an MD5 hash collision.") log.debug("Either someone is seriously messing with your network,") log.debug("or else you are witnessing the impossible. %s" % r) - return False, relative_complement + ret = False, relative_complement + defer.returnValue(ret) + @defer.inlineCallbacks def google_dns_cp_test(self): """ Google Chrome resolves three 10-byte random hostnames. @@ -437,18 +436,19 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): subtest = "Google Chrome DNS-based" log.msg("Running the Google Chrome DNS-based captive portal test...") - gmatch, google_dns_result = self.compare_random_hostnames(3, 10) + gmatch, google_dns_result = yield self.compare_random_hostnames(3, 10) if gmatch: log.msg("Google Chrome DNS-based captive portal test did not") log.msg("detect a captive portal.") - return google_dns_result + defer.returnValue(google_dns_result) else: log.msg("Google Chrome DNS-based captive portal test believes") log.msg("you are in a captive portal, or else something very") log.msg("odd is happening with your DNS.") - return google_dns_result + defer.returnValue(google_dns_result) + @defer.inlineCallbacks def ms_dns_cp_test(self): """ Microsoft "phones home" to a server which will always resolve @@ -460,26 +460,27 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): log.msg("Running the Microsoft NCSI DNS-based captive portal") log.msg("test...") - msmatch, ms_dns_result = self.dns_resolve_match("dns.msftncsi.com", + msmatch, ms_dns_result = yield self.dns_resolve_match("dns.msftncsi.com", "131.107.255.255") if msmatch: log.msg("Microsoft NCSI DNS-based captive portal test did not") log.msg("detect a captive portal.") - return ms_dns_result + defer.returnValue(ms_dns_result) else: log.msg("Microsoft NCSI DNS-based captive portal test ") log.msg("believes you are in a captive portal.") - return ms_dns_result + defer.returnValue(ms_dns_result) + @defer.inlineCallbacks def run_vendor_dns_tests(self): """ Run the vendor DNS tests. """ report = {} - report['google_dns_cp'] = self.google_dns_cp_test() - report['ms_dns_cp'] = self.ms_dns_cp_test() + report['google_dns_cp'] = yield self.google_dns_cp_test() + report['ms_dns_cp'] = yield self.ms_dns_cp_test() + defer.returnValue(report) - return report @defer.inlineCallbacks def run_vendor_tests(self, *a, **kw): """ @@ -640,7 +641,7 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): log.msg("") log.msg("Running vendor DNS-based tests...") - self.report['vendor_dns_tests'] = yield threads.deferToThread(self.run_vendor_dns_tests) + self.report['vendor_dns_tests'] = yield self.run_vendor_dns_tests() log.msg("") log.msg("Checking that DNS requests are not being tampered...") [View Less]

1 0

[ooni-probe/master] Replaced dnspython with ooni.templates.dnst for performing SOA Lookup in
by art＠torproject.org 31 Mar '14

31 Mar '14

commit e93b1aaf1b13747bfc9b1761febe36552b862196 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Sun Mar 23 09:18:47 2014 +0530 Replaced dnspython with ooni.templates.dnst for performing SOA Lookup in Captiveportal test --- ooni/nettests/manipulation/captiveportal.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index 443b2ed..afaeee8 100644 --- … [View More]

1 0

[ooni-probe/master] Support for performing SOA Lookup in ooni.templates.DNST
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 6083688f28b65f3aeffbb73171ec0cb6790d7c45 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Sun Mar 23 09:01:18 2014 +0530 Support for performing SOA Lookup in ooni.templates.DNST --- ooni/templates/dnst.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index 66a6d17..38da177 100644 --- a/ooni/templates/dnst.py +++ b/ooni/templates/dnst.py @@ -197,6 +197,66 @@ class … [View More]DNSTest(NetTestCase): return d + def performSOALookup(self, hostname, dns_server = None): + """ + Performs a SOA lookup and returns the response (name,serial). + + :hostname: is the hostname to perform the NS lookup on + + :dns_server: is the dns_server that should be used for the lookup as a + tuple of ip port (ex. ("127.0.0.1", 53)) + """ + query = [dns.Query(hostname, dns.SOA, dns.IN)] + if dns_server: + def gotResponse(message): + addrs = [] + answers = [] + for answer in message.answers: + if answer.type is dns.SOA: + addr = (answer.name.name,answer.payload.serial) + addrs.append(addr) + answers.append(representAnswer(answer)) + + DNSTest.addToReport(self, query, resolver=dns_server, query_type='SOA', + answers=answers, addrs=addrs) + return addrs + + def gotError(failure): + failure.trap(gaierror, TimeoutError) + DNSTest.addToReport(self, query, resolver=dns_server, query_type='SOA', + failure=failure) + return failure + + resolver = Resolver(servers=[dns_server]) + d = resolver.queryUDP(query, timeout=self.queryTimeout) + d.addCallback(gotResponse) + d.addErrback(gotError) + else: + def gotResponse(message): + addrs = [] + answers = [] + for answer in message[0]: + if answer.type is dns.SOA: + addr = (answer.name.name,answer.payload.serial) + addrs.append(addr) + answers.append(representAnswer(answer)) + + DNSTest.addToReport(self, query, resolver=dns_server, query_type='SOA', + answers=answers, addrs=addrs) + return addrs + + def gotError(failure): + failure.trap(gaierror, TimeoutError) + DNSTest.addToReport(self, query, resolver=dns_server, query_type='SOA', + failure=failure) + return failure + + d = client.lookupAuthority(hostname) + d.addCallback(gotResponse) + d.addErrback(gotError) + + return d + def addToReport(self, query, resolver=None, query_type=None, answers=None, name=None, addrs=None, failure=None): log.debug("Adding %s to report)" % query) [View Less]

1 0

[ooni-probe/master] Support for using system dns setting for performALookup in
by art＠torproject.org 31 Mar '14

31 Mar '14

commit d0230379077272fa8aacf3bad2bbfe01880aeaeb Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Sat Mar 22 03:52:39 2014 +0530 Support for using system dns setting for performALookup in ooni.templates.DNST --- ooni/templates/dnst.py | 69 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index ff5afc3..66a6d17 100644 --- a/ooni/templates/dnst.py +++ b/ooni/… [View More]templates/dnst.py @@ -73,7 +73,7 @@ class DNSTest(NetTestCase): d.addErrback(gotError) return d - def performALookup(self, hostname, dns_server): + def performALookup(self, hostname, dns_server = None): """ Performs an A lookup and returns an array containg all the dotted quad IP addresses in the response. @@ -82,31 +82,58 @@ class DNSTest(NetTestCase): :dns_server: is the dns_server that should be used for the lookup as a tuple of ip port (ex. ("127.0.0.1", 53)) + + if None, system dns settings will be used """ query = [dns.Query(hostname, dns.A, dns.IN)] - def gotResponse(message): - addrs = [] - answers = [] - for answer in message.answers: - if answer.type is 1: - addr = answer.payload.dottedQuad() - addrs.append(addr) - answers.append(representAnswer(answer)) + if dns_server: + def gotResponse(message): + addrs = [] + answers = [] + for answer in message.answers: + if answer.type is dns.A: + addr = answer.payload.dottedQuad() + addrs.append(addr) + answers.append(representAnswer(answer)) - DNSTest.addToReport(self, query, resolver=dns_server, query_type='A', - answers=answers, addrs=addrs) - return addrs + DNSTest.addToReport(self, query, resolver=dns_server, query_type='A', + answers=answers, addrs=addrs) + return addrs - def gotError(failure): - failure.trap(gaierror, TimeoutError) - DNSTest.addToReport(self, query, resolver=dns_server, query_type='A', - failure=failure) - return failure + def gotError(failure): + failure.trap(gaierror, TimeoutError) + DNSTest.addToReport(self, query, resolver=dns_server, query_type='A', + failure=failure) + return failure + + resolver = Resolver(servers=[dns_server]) + d = resolver.queryUDP(query, timeout=self.queryTimeout) + d.addCallback(gotResponse) + d.addErrback(gotError) + else: + def gotResponse(message): + addrs = [] + answers = [] + for answer in message[0]: + if answer.type is dns.A: + addr = answer.payload.dottedQuad() + addrs.append(addr) + answers.append(representAnswer(answer)) + + DNSTest.addToReport(self, query, resolver=dns_server, query_type='A', + answers=answers, addrs=addrs) + return addrs + + def gotError(failure): + failure.trap(gaierror, TimeoutError) + DNSTest.addToReport(self, query, resolver=dns_server, query_type='A', + failure=failure) + return failure + + d = client.lookupAddress(hostname) + d.addCallback(gotResponse) + d.addErrback(gotError) - resolver = Resolver(servers=[dns_server]) - d = resolver.queryUDP(query, timeout=self.queryTimeout) - d.addCallback(gotResponse) - d.addErrback(gotError) return d def performNSLookup(self, hostname, dns_server = None): [View Less]

1 0

[ooni-probe/master] Make get_auth_nameservers use ooni.templates.DNST
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 26eb9400f4fa762ddfff3c0bcbc9a717dfb20fa6 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Fri Mar 21 04:08:25 2014 +0530 Make get_auth_nameservers use ooni.templates.DNST --- ooni/nettests/manipulation/captiveportal.py | 31 +++++++++++---------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index 90b972c..6862177 100644 --- a/ooni/nettests/manipulation/… [View More]captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -39,7 +39,7 @@ from twisted.python import usage from twisted.internet import defer, threads from ooni import nettest -from ooni.templates import httpt +from ooni.templates import httpt,dnst from ooni.utils import net from ooni.utils import log @@ -60,7 +60,7 @@ class UsageOptions(usage.Options): 'User agent for HTTP requests'] ] -class CaptivePortal(httpt.HTTPTest): +class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): """ Compares content and status codes of HTTP responses, and attempts to determine if content has been altered. @@ -215,7 +215,7 @@ class CaptivePortal(httpt.HTTPTest): log.msg("DNS comparison of control '%s' does not" % control_address) log.msg("match experiment response '%s'" % experiment_address) return False, experiment_address - + @defer.inlineCallbacks def get_auth_nameservers(self, hostname): """ Many CPs set a nameserver to be used. Let's query that @@ -224,17 +224,9 @@ class CaptivePortal(httpt.HTTPTest): The equivalent of: $ dig +short NS ooni.nu """ - if not resolver: - log.msg("dnspython not installed.") - log.msg("Cannot perform test.") - return [] - - res = resolver.Resolver() - answer = res.query(hostname, 'NS') - auth_nameservers = [] - for auth in answer: - auth_nameservers.append(auth.to_text()) - return auth_nameservers + auth_nameservers = yield self.performNSLookup(hostname) + print auth_nameservers + defer.returnValue(auth_nameservers) def hostname_to_0x20(self, hostname): """ @@ -253,6 +245,7 @@ class CaptivePortal(httpt.HTTPTest): hostname_0x20 += char.lower() return hostname_0x20 + @defer.inlineCallbacks def check_0x20_to_auth_ns(self, hostname, sample_size=None): """ Resolve a 0x20 DNS request for hostname over hostname's @@ -269,7 +262,7 @@ class CaptivePortal(httpt.HTTPTest): log.msg("Testing random capitalization of DNS queries...") log.msg("Testing that Start of Authority serial numbers match...") - auth_nameservers = self.get_auth_nameservers(hostname) + auth_nameservers = yield self.get_auth_nameservers(hostname) if sample_size is None: sample_size = 5 @@ -317,13 +310,13 @@ class CaptivePortal(httpt.HTTPTest): if name_match and serial_match: log.msg("Your DNS queries do not appear to be tampered.") - return ret + defer.returnValue(ret) elif name_match or serial_match: log.msg("Something is tampering with your DNS queries.") - return ret + defer.returnValue(ret) elif not name_match and not serial_match: log.msg("Your DNS queries are definitely being tampered with.") - return ret + defer.returnValue(ret) def get_random_url_safe_string(self, length): """ @@ -651,7 +644,7 @@ class CaptivePortal(httpt.HTTPTest): log.msg("") log.msg("Checking that DNS requests are not being tampered...") - self.report['check0x20'] = yield threads.deferToThread(self.check_0x20_to_auth_ns, 'ooni.nu') + self.report['check0x20'] = yield self.check_0x20_to_auth_ns('ooni.nu') log.msg("") log.msg("Captive portal test finished!") [View Less]

1 0

[ooni-probe/master] Support for performing NS lookup in ooni.templates.DNST
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 6dbaeec310142adbb09a8de405e4e2b8982f6155 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Fri Mar 21 02:13:12 2014 +0530 Support for performing NS lookup in ooni.templates.DNST --- ooni/templates/dnst.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index 3a97ed8..ff5afc3 100644 --- a/ooni/templates/dnst.py +++ b/ooni/templates/dnst.py @@ -109,6 +109,67 @@ class … [View More]DNSTest(NetTestCase): d.addErrback(gotError) return d + def performNSLookup(self, hostname, dns_server = None): + """ + Performs a NS lookup and returns an array containg all nameservers in + the response. + + :hostname: is the hostname to perform the NS lookup on + + :dns_server: is the dns_server that should be used for the lookup as a + tuple of ip port (ex. ("127.0.0.1", 53)) + """ + query = [dns.Query(hostname, dns.NS, dns.IN)] + if dns_server: + def gotResponse(message): + addrs = [] + answers = [] + for answer in message.answers: + if answer.type is dns.NS: + addr = answer.payload.name.name + addrs.append(addr) + answers.append(representAnswer(answer)) + + DNSTest.addToReport(self, query, resolver=dns_server, query_type='NS', + answers=answers, addrs=addrs) + return addrs + + def gotError(failure): + failure.trap(gaierror, TimeoutError) + DNSTest.addToReport(self, query, resolver=dns_server, query_type='NS', + failure=failure) + return failure + + resolver = Resolver(servers=[dns_server]) + d = resolver.queryUDP(query, timeout=self.queryTimeout) + d.addCallback(gotResponse) + d.addErrback(gotError) + else: + def gotResponse(message): + addrs = [] + answers = [] + for answer in message[0]: + if answer.type is dns.NS: + addr = answer.payload.name.name + addrs.append(addr) + answers.append(representAnswer(answer)) + + DNSTest.addToReport(self, query, resolver=dns_server, query_type='NS', + answers=answers, addrs=addrs) + return addrs + + def gotError(failure): + failure.trap(gaierror, TimeoutError) + DNSTest.addToReport(self, query, resolver=dns_server, query_type='NS', + failure=failure) + return failure + + d = client.lookupNameservers(hostname) + d.addCallback(gotResponse) + d.addErrback(gotError) + + return d + def addToReport(self, query, resolver=None, query_type=None, answers=None, name=None, addrs=None, failure=None): log.debug("Adding %s to report)" % query) [View Less]

1 0

[ooni-probe/master] removed unnecessary import
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 9d29cb298e0aaefcc908afb7029a8903d24b0b49 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Fri Mar 7 03:22:20 2014 +0530 removed unnecessary import --- ooni/nettests/manipulation/captiveportal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index 2979120..90b972c 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/… [View More]

1 0

[ooni-probe/master] Construt TrueHeaders using string parameter
by art＠torproject.org 31 Mar '14

31 Mar '14

commit efaaaece1bd86dd76897435846b7bd4d7dc93f2a Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Tue Feb 18 21:51:09 2014 +0530 Construt TrueHeaders using string parameter --- ooni/utils/trueheaders.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index 06e5c02..31ee179 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -32,6 +32,8 @@ class TrueHeaders(http_headers.Headers): … [View More]

1 0

[ooni-probe/master] Replacing urllib2 with httpt.HTTPTest for captiveportal test
by art＠torproject.org 31 Mar '14

31 Mar '14

commit ef91259cbc20645067ca0a308a86cec86c516605 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Tue Feb 11 07:49:53 2014 +0530 Replacing urllib2 with httpt.HTTPTest for captiveportal test --- ooni/nettests/manipulation/captiveportal.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index d6d8040..59c6d60 100644 --- a/ooni/nettests/… [View More]manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -33,11 +33,10 @@ import os import random import re import string -import urllib2 from urlparse import urlparse from twisted.python import usage -from twisted.internet import defer, threads +from twisted.internet import defer, threads, reactor from ooni import nettest from ooni.templates import httpt @@ -61,7 +60,7 @@ class UsageOptions(usage.Options): 'User agent for HTTP requests'] ] -class CaptivePortal(nettest.NetTestCase): +class CaptivePortal(httpt.HTTPTest): """ Compares content and status codes of HTTP responses, and attempts to determine if content has been altered. @@ -77,19 +76,18 @@ class CaptivePortal(nettest.NetTestCase): def http_fetch(self, url, headers={}): """ - Parses an HTTP url, fetches it, and returns a urllib2 response + Parses an HTTP url, fetches it, and returns a response object. """ url = urlparse(url).geturl() - request = urllib2.Request(url, None, headers) #XXX: HTTP Error 302: The HTTP server returned a redirect error that #would lead to an infinite loop. The last 30x error message was: Found try: - response = urllib2.urlopen(request) - response_headers = dict(response.headers) + response = threads.blockingCallFromThread(reactor,self.doRequest,url,"GET",headers) + response_headers = response.headers return response, response_headers - except urllib2.HTTPError, e: - log.err("HTTPError: %s" % e) + except: + log.err("HTTPError") return None, None def http_content_match_fuzzy_opt(self, experimental_url, control_result, @@ -111,7 +109,7 @@ class CaptivePortal(nettest.NetTestCase): response, response_headers = self.http_fetch(experimental_url, headers) - response_content = response.read() if response else None + response_content = response.body if response else None response_code = response.code if response else None if response_content is None: log.err("HTTP connection appears to have failed.") @@ -359,7 +357,7 @@ class CaptivePortal(nettest.NetTestCase): random_sld = self.get_random_url_safe_string(length) tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] - random_tld = urllib2.random.choice(tld_list) + random_tld = random.choice(tld_list) random_hostname = random_sld + random_tld return random_hostname @@ -523,7 +521,6 @@ class CaptivePortal(nettest.NetTestCase): control_result, headers, fuzzy) status_match = status_func(experiment_code, control_code) - if status_match and content_match: log.msg("The %s test was unable to detect" % test_name) log.msg("a captive portal.") [View Less]

1 0