tor-commits March 2014

tor-commits@lists.torproject.org

22 participants
2036 discussions

[ooni-probe/master] Removed unnecessary print
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 03a64de8f9aa72215db0c55803e612d0ab030bde Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Sun Mar 23 07:31:53 2014 +0530 Removed unnecessary print --- ooni/nettests/manipulation/captiveportal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index 49d5b8d..443b2ed 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -218,7 +218,6 @@ class CaptivePortal(httpt.HTTPTest,dnst.DNSTest): $ dig +short NS ooni.nu """ auth_nameservers = yield self.performNSLookup(hostname) - print auth_nameservers defer.returnValue(auth_nameservers) def hostname_to_0x20(self, hostname):

1 0

[ooni-probe/master] Refactoring ooni.templates.DNST
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 68e777d4a3c374a842f7cd149129361620c74e5c Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Wed Mar 26 01:30:00 2014 +0530 Refactoring ooni.templates.DNST --- ooni/templates/dnst.py | 190 +++++++++++++----------------------------------- 1 file changed, 50 insertions(+), 140 deletions(-) diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index 38da177..7f076ed 100644 --- a/ooni/templates/dnst.py +++ b/ooni/templates/dnst.py @@ -85,56 +85,7 @@ class DNSTest(NetTestCase): if None, system dns settings will be used """ - query = [dns.Query(hostname, dns.A, dns.IN)] - if dns_server: - def gotResponse(message): - addrs = [] - answers = [] - for answer in message.answers: - if answer.type is dns.A: - addr = answer.payload.dottedQuad() - addrs.append(addr) - answers.append(representAnswer(answer)) - - DNSTest.addToReport(self, query, resolver=dns_server, query_type='A', - answers=answers, addrs=addrs) - return addrs - - def gotError(failure): - failure.trap(gaierror, TimeoutError) - DNSTest.addToReport(self, query, resolver=dns_server, query_type='A', - failure=failure) - return failure - - resolver = Resolver(servers=[dns_server]) - d = resolver.queryUDP(query, timeout=self.queryTimeout) - d.addCallback(gotResponse) - d.addErrback(gotError) - else: - def gotResponse(message): - addrs = [] - answers = [] - for answer in message[0]: - if answer.type is dns.A: - addr = answer.payload.dottedQuad() - addrs.append(addr) - answers.append(representAnswer(answer)) - - DNSTest.addToReport(self, query, resolver=dns_server, query_type='A', - answers=answers, addrs=addrs) - return addrs - - def gotError(failure): - failure.trap(gaierror, TimeoutError) - DNSTest.addToReport(self, query, resolver=dns_server, query_type='A', - failure=failure) - return failure - - d = client.lookupAddress(hostname) - d.addCallback(gotResponse) - d.addErrback(gotError) - - return d + return self.dnsLookup(hostname, 'A', dns_server) def performNSLookup(self, hostname, dns_server = None): """ @@ -145,118 +96,77 @@ class DNSTest(NetTestCase): :dns_server: is the dns_server that should be used for the lookup as a tuple of ip port (ex. ("127.0.0.1", 53)) - """ - query = [dns.Query(hostname, dns.NS, dns.IN)] - if dns_server: - def gotResponse(message): - addrs = [] - answers = [] - for answer in message.answers: - if answer.type is dns.NS: - addr = answer.payload.name.name - addrs.append(addr) - answers.append(representAnswer(answer)) - - DNSTest.addToReport(self, query, resolver=dns_server, query_type='NS', - answers=answers, addrs=addrs) - return addrs - - def gotError(failure): - failure.trap(gaierror, TimeoutError) - DNSTest.addToReport(self, query, resolver=dns_server, query_type='NS', - failure=failure) - return failure - - resolver = Resolver(servers=[dns_server]) - d = resolver.queryUDP(query, timeout=self.queryTimeout) - d.addCallback(gotResponse) - d.addErrback(gotError) - else: - def gotResponse(message): - addrs = [] - answers = [] - for answer in message[0]: - if answer.type is dns.NS: - addr = answer.payload.name.name - addrs.append(addr) - answers.append(representAnswer(answer)) - - DNSTest.addToReport(self, query, resolver=dns_server, query_type='NS', - answers=answers, addrs=addrs) - return addrs - def gotError(failure): - failure.trap(gaierror, TimeoutError) - DNSTest.addToReport(self, query, resolver=dns_server, query_type='NS', - failure=failure) - return failure - - d = client.lookupNameservers(hostname) - d.addCallback(gotResponse) - d.addErrback(gotError) - - return d + if None, system dns settings will be used + """ + return self.dnsLookup(hostname, 'NS', dns_server) def performSOALookup(self, hostname, dns_server = None): """ Performs a SOA lookup and returns the response (name,serial). - :hostname: is the hostname to perform the NS lookup on + :hostname: is the hostname to perform the SOA lookup on + :dns_server: is the dns_server that should be used for the lookup as a + tuple of ip port (ex. ("127.0.0.1", 53)) + + if None, system dns settings will be used + """ + return self.dnsLookup(hostname,'SOA',dns_server) + def dnsLookup(self, hostname, dns_type, dns_server = None): + """ + Performs a DNS lookup and returns the response. + + :hostname: is the hostname to perform the DNS lookup on + :dns_type: type of lookup 'NS'/'A'/'SOA' :dns_server: is the dns_server that should be used for the lookup as a tuple of ip port (ex. ("127.0.0.1", 53)) """ - query = [dns.Query(hostname, dns.SOA, dns.IN)] - if dns_server: - def gotResponse(message): - addrs = [] - answers = [] - for answer in message.answers: - if answer.type is dns.SOA: + types={'NS':dns.NS,'A':dns.A,'SOA':dns.SOA} + dnsType=types[dns_type] + query = [dns.Query(hostname, dnsType, dns.IN)] + def gotResponse(message): + addrs = [] + answers = [] + if dns_server: + msg = message.answers + else: + msg = message[0] + for answer in msg: + if answer.type is dnsType: + if dnsType is dns.SOA: addr = (answer.name.name,answer.payload.serial) - addrs.append(addr) - answers.append(representAnswer(answer)) + elif dnsType is dns.NS: + addr = answer.payload.name.name + elif dnsType is dns.A: + addr = answer.payload.dottedQuad() + else: + addr = None + addrs.append(addr) + answers.append(representAnswer(answer)) - DNSTest.addToReport(self, query, resolver=dns_server, query_type='SOA', + DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type, answers=answers, addrs=addrs) - return addrs + return addrs - def gotError(failure): - failure.trap(gaierror, TimeoutError) - DNSTest.addToReport(self, query, resolver=dns_server, query_type='SOA', + def gotError(failure): + failure.trap(gaierror, TimeoutError) + DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type, failure=failure) - return failure + return failure + if dns_server: resolver = Resolver(servers=[dns_server]) d = resolver.queryUDP(query, timeout=self.queryTimeout) - d.addCallback(gotResponse) - d.addErrback(gotError) else: - def gotResponse(message): - addrs = [] - answers = [] - for answer in message[0]: - if answer.type is dns.SOA: - addr = (answer.name.name,answer.payload.serial) - addrs.append(addr) - answers.append(representAnswer(answer)) - - DNSTest.addToReport(self, query, resolver=dns_server, query_type='SOA', - answers=answers, addrs=addrs) - return addrs - - def gotError(failure): - failure.trap(gaierror, TimeoutError) - DNSTest.addToReport(self, query, resolver=dns_server, query_type='SOA', - failure=failure) - return failure - - d = client.lookupAuthority(hostname) - d.addCallback(gotResponse) - d.addErrback(gotError) + lookupFunction={'NS':client.lookupNameservers, 'SOA':client.lookupAuthority, 'A':client.lookupAddress} + d = lookupFunction[dns_type](hostname) + d.addCallback(gotResponse) + d.addErrback(gotError) return d + def addToReport(self, query, resolver=None, query_type=None, answers=None, name=None, addrs=None, failure=None): log.debug("Adding %s to report)" % query)

1 0

[ooni-probe/master] Removed unnecessary import
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 86318636124cefb9c133018bfc6ed79d4d9ff07d Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Sun Mar 23 09:20:53 2014 +0530 Removed unnecessary import --- ooni/nettests/manipulation/captiveportal.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index afaeee8..717bb10 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -44,13 +44,6 @@ from ooni.templates import httpt,dnst from ooni.utils import net from ooni.utils import log -try: - from dns import resolver -except ImportError: - print "The dnspython module was not found:" - print "See https://crate.io/packages/dnspython/" - resolver = None - __plugoo__ = "captiveportal" __desc__ = "Captive portal detection test"

1 0

[ooni-probe/master] Dependency dnspython removed
by art＠torproject.org 31 Mar '14

31 Mar '14

commit a4b2792f94094e3780a06395a8b789aef149bfcc Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Wed Mar 26 07:16:01 2014 +0530 Dependency dnspython removed --- debian/control | 1 - requirements.txt | 1 - setup-dependencies.sh | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/debian/control b/debian/control index ba7281b..9bdf641 100644 --- a/debian/control +++ b/debian/control @@ -29,7 +29,6 @@ Depends: ${misc:Depends}, ${python:Depends}, tcpdump, tor, python-all (>= 2.6.6-3~), - python-dnspython, python-dumbnet, python-geoip (>= 0.2.5), python-ipaddr (>= 2.1.10), diff --git a/requirements.txt b/requirements.txt index 53709ba..3fb0142 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,5 @@ geoip txtorcon>=0.7 txsocksx>=0.0.2 parsley>=1.1 -dnspython>=1.10.0 scapy-real>=2.2.0-dev pypcap>=1.1 diff --git a/setup-dependencies.sh b/setup-dependencies.sh index a6867c7..2aac715 100755 --- a/setup-dependencies.sh +++ b/setup-dependencies.sh @@ -896,7 +896,7 @@ case $DISTRO_VERSION in echo "[+] Updating OS package list..."; sudo sudo apt-get update 2>&1 > /dev/null; echo "[+] Installing packages for your system..."; - DO "sudo apt-get -y install curl git-core python python-dev python-setuptools build-essential libdumbnet1 python-dumbnet python-libpcap python-dnspython tor tor-geoipdb libgeoip-dev libpcap0.8-dev" "0" + DO "sudo apt-get -y install curl git-core python python-dev python-setuptools build-essential libdumbnet1 python-dumbnet python-libpcap tor tor-geoipdb libgeoip-dev libpcap0.8-dev" "0" if [ "$PRIV_MODE" -eq "0" ]; then echo "[+] Using virtualenvironment..."

1 0

[ooni-probe/master] Override some of the twisted.names methods to disable logging.
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 2e2bd541d76983bf14d22e3da632e5591292de39 Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Mar 31 11:30:59 2014 +0200 Override some of the twisted.names methods to disable logging. --- ooni/templates/dnst.py | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index 4ebc439..70c9c8e 100644 --- a/ooni/templates/dnst.py +++ b/ooni/templates/dnst.py @@ -3,7 +3,7 @@ # :authors: Arturo Filastò # :licence: see LICENSE -from twisted.internet import defer +from twisted.internet import defer, udp, error, base from twisted.internet.defer import TimeoutError from twisted.names import client, dns from twisted.names.client import Resolver @@ -14,8 +14,50 @@ from ooni.utils import log from ooni.nettest import NetTestCase from ooni.errors import failureToString +import socket from socket import gaierror +dns.DNSDatagramProtocol.noisy = False + +def _bindSocket(self): + """ + _bindSocket taken from Twisted 13.1.0 to suppress logging. + """ + try: + skt = self.createInternetSocket() + skt.bind((self.interface, self.port)) + except socket.error as le: + raise error.CannotListenError(self.interface, self.port, le) + + # Make sure that if we listened on port 0, we update that to + # reflect what the OS actually assigned us. + self._realPortNumber = skt.getsockname()[1] + + # Here we remove the logging. + # log.msg("%s starting on %s" % ( + # self._getLogPrefix(self.protocol), self._realPortNumber)) + + self.connected = 1 + self.socket = skt + self.fileno = self.socket.fileno +udp.Port._bindSocket = _bindSocket + +def connectionLost(self, reason=None): + """ + Taken from Twisted 13.1.0 to suppress log.msg printing. + """ + # Here we remove the logging. + # log.msg('(UDP Port %s Closed)' % self._realPortNumber) + self._realPortNumber = None + base.BasePort.connectionLost(self, reason) + self.protocol.doStop() + self.socket.close() + del self.socket + del self.fileno + if hasattr(self, "d"): + self.d.callback(None) + del self.d +udp.Port.connectionLost = connectionLost def representAnswer(answer): # We store the resource record and the answer payload in a @@ -127,8 +169,6 @@ class DNSTest(NetTestCase): return addrs def gotError(failure): - log.err("Failed to perform "+dns_type+" lookup") - log.exception(failure) failure.trap(gaierror, TimeoutError) DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type, failure=failure)

1 0

[ooni-probe/master] Refactored ooni.templates.dnst
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 2177585daf3f930a5b6438ce1f2a46e3ad7c5ddc Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Wed Mar 26 08:00:05 2014 +0530 Refactored ooni.templates.dnst --- ooni/templates/dnst.py | 43 +++++++++++-------------------------------- 1 file changed, 11 insertions(+), 32 deletions(-) diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py index 7f076ed..4ebc439 100644 --- a/ooni/templates/dnst.py +++ b/ooni/templates/dnst.py @@ -34,7 +34,7 @@ class DNSTest(NetTestCase): self.report['queries'] = [] - def performPTRLookup(self, address, dns_server): + def performPTRLookup(self, address, dns_server = None): """ Does a reverse DNS lookup on the input ip address @@ -42,36 +42,11 @@ class DNSTest(NetTestCase): :dns_server: is the dns_server that should be used for the lookup as a tuple of ip port (ex. ("127.0.0.1", 53)) + + if None, system dns settings will be used """ ptr = '.'.join(address.split('.')[::-1]) + '.in-addr.arpa' - query = [dns.Query(ptr, dns.PTR, dns.IN)] - def gotResponse(message): - log.debug("Lookup successful") - log.debug(message) - answers = [] - name = '' - for answer in message.answers: - if answer.type is 12: - name = str(answer.payload.name) - answers.append(representAnswer(answer)) - - DNSTest.addToReport(self, query, resolver=dns_server, - query_type = 'PTR', answers=answers, name=name) - return name - - def gotError(failure): - log.err("Failed to perform lookup") - log.exception(failure) - failure.trap(gaierror, TimeoutError) - DNSTest.addToReport(self, query, resolver=dns_server, - query_type = 'PTR', failure=failure) - return None - - resolver = Resolver(servers=[dns_server]) - d = resolver.queryUDP(query, timeout=self.queryTimeout) - d.addCallback(gotResponse) - d.addErrback(gotError) - return d + return self.dnsLookup(ptr, 'PTR', dns_server) def performALookup(self, hostname, dns_server = None): """ @@ -122,10 +97,12 @@ class DNSTest(NetTestCase): :dns_server: is the dns_server that should be used for the lookup as a tuple of ip port (ex. ("127.0.0.1", 53)) """ - types={'NS':dns.NS,'A':dns.A,'SOA':dns.SOA} + types={'NS':dns.NS,'A':dns.A,'SOA':dns.SOA,'PTR':dns.PTR} dnsType=types[dns_type] query = [dns.Query(hostname, dnsType, dns.IN)] def gotResponse(message): + log.debug(dns_type+" Lookup successful") + log.debug(message) addrs = [] answers = [] if dns_server: @@ -136,7 +113,7 @@ class DNSTest(NetTestCase): if answer.type is dnsType: if dnsType is dns.SOA: addr = (answer.name.name,answer.payload.serial) - elif dnsType is dns.NS: + elif dnsType in [dns.NS,dns.PTR]: addr = answer.payload.name.name elif dnsType is dns.A: addr = answer.payload.dottedQuad() @@ -150,6 +127,8 @@ class DNSTest(NetTestCase): return addrs def gotError(failure): + log.err("Failed to perform "+dns_type+" lookup") + log.exception(failure) failure.trap(gaierror, TimeoutError) DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type, failure=failure) @@ -159,7 +138,7 @@ class DNSTest(NetTestCase): resolver = Resolver(servers=[dns_server]) d = resolver.queryUDP(query, timeout=self.queryTimeout) else: - lookupFunction={'NS':client.lookupNameservers, 'SOA':client.lookupAuthority, 'A':client.lookupAddress} + lookupFunction={'NS':client.lookupNameservers, 'SOA':client.lookupAuthority, 'A':client.lookupAddress, 'PTR':client.lookupPointer} d = lookupFunction[dns_type](hostname) d.addCallback(gotResponse)

1 0

[ooni-probe/master] s/threads/defereds/
by art＠torproject.org 31 Mar '14

31 Mar '14

commit 3cf943a235b1f04491c3933eead95b1e3dc49d8b Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Fri Mar 7 03:03:26 2014 +0530 s/threads/defereds/ --- ooni/nettests/manipulation/captiveportal.py | 53 ++++++++++++++++----------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index 59c6d60..2979120 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -74,6 +74,7 @@ class CaptivePortal(httpt.HTTPTest): requiresRoot = False requiresTor = False + @defer.inlineCallbacks def http_fetch(self, url, headers={}): """ Parses an HTTP url, fetches it, and returns a response @@ -83,13 +84,13 @@ class CaptivePortal(httpt.HTTPTest): #XXX: HTTP Error 302: The HTTP server returned a redirect error that #would lead to an infinite loop. The last 30x error message was: Found try: - response = threads.blockingCallFromThread(reactor,self.doRequest,url,"GET",headers) - response_headers = response.headers - return response, response_headers - except: + response = yield self.doRequest(url,"GET",headers) + defer.returnValue(response) + except Exception: log.err("HTTPError") - return None, None + defer.returnValue(None) + @defer.inlineCallbacks def http_content_match_fuzzy_opt(self, experimental_url, control_result, headers=None, fuzzy=False): """ @@ -107,13 +108,15 @@ class CaptivePortal(httpt.HTTPTest): default_ua = self.local_options['user-agent'] headers = {'User-Agent': default_ua} - response, response_headers = self.http_fetch(experimental_url, headers) + response = yield self.http_fetch(experimental_url, headers) + response_headers = response.headers response_content = response.body if response else None response_code = response.code if response else None if response_content is None: log.err("HTTP connection appears to have failed.") - return False, False, False + r = (False, False, False) + defer.returnValue(r) if fuzzy: pattern = re.compile(control_result) @@ -122,18 +125,22 @@ class CaptivePortal(httpt.HTTPTest): log.msg("'%s'" % experimental_url) if not match: log.msg("does not match!") - return False, response_code, response_headers + r = (False, response_code, response_headers) + defer.returnValue(r) else: log.msg("and the expected control result yielded a match.") - return True, response_code, response_headers + r = (True, response_code, response_headers) + defer.returnValue(r) else: if str(response_content) != str(control_result): log.msg("HTTP content comparison of experiment URL") log.msg("'%s'" % experimental_url) log.msg("and the expected control result do not match.") - return False, response_code, response_headers + r = (False, response_code, response_headers) + defer.returnValue(r) else: - return True, response_code, response_headers + r = (True, response_code, response_headers) + defer.returnValue(r) def http_status_code_match(self, experiment_code, control_code): """ @@ -480,7 +487,7 @@ class CaptivePortal(httpt.HTTPTest): report['ms_dns_cp'] = self.ms_dns_cp_test() return report - + @defer.inlineCallbacks def run_vendor_tests(self, *a, **kw): """ These are several vendor tests used to detect the presence of @@ -512,23 +519,24 @@ class CaptivePortal(httpt.HTTPTest): sm = self.http_status_code_match snm = self.http_status_code_no_match + @defer.inlineCallbacks def compare_content(status_func, fuzzy, experiment_url, control_result, control_code, headers, test_name): log.msg("") log.msg("Running the %s test..." % test_name) - content_match, experiment_code, experiment_headers = cm(experiment_url, + content_match, experiment_code, experiment_headers = yield cm(experiment_url, control_result, headers, fuzzy) status_match = status_func(experiment_code, control_code) if status_match and content_match: log.msg("The %s test was unable to detect" % test_name) log.msg("a captive portal.") - return True + defer.returnValue(True) else: log.msg("The %s test shows that your network" % test_name) log.msg("is filtered.") - return False + defer.returnValue(False) result = [] for vt in vendor_tests: @@ -544,19 +552,20 @@ class CaptivePortal(httpt.HTTPTest): args = (experiment_url, control_result, control_code, headers, test_name) if test_name == "MS HTTP Captive Portal": - report['result'] = compare_content(sm, False, *args) + report['result'] = yield compare_content(sm, False, *args) elif test_name == "Apple HTTP Captive Portal": - report['result'] = compare_content(sm, True, *args) + report['result'] = yield compare_content(sm, True, *args) elif test_name == "W3 Captive Portal": - report['result'] = compare_content(snm, True, *args) + report['result'] = yield compare_content(snm, True, *args) else: log.err("Ooni is trying to run an undefined CP vendor test.") result.append(report) - return result + defer.returnValue(result) + @defer.inlineCallbacks def control(self, experiment_result, args): """ Compares the content and status code of the HTTP response for @@ -579,7 +588,7 @@ class CaptivePortal(httpt.HTTPTest): snm = self.http_status_code_no_match log.msg("Running test for '%s'..." % experiment_url) - content_match, experiment_code, experiment_headers = cm(experiment_url, + content_match, experiment_code, experiment_headers = yield cm(experiment_url, control_result) status_match = sm(experiment_code, control_code) if status_match and content_match: @@ -591,7 +600,7 @@ class CaptivePortal(httpt.HTTPTest): elif status_match and not content_match: log.msg("Retrying '%s' with fuzzy match enabled." % experiment_url) - fuzzy_match, experiment_code, experiment_headers = cm(experiment_url, + fuzzy_match, experiment_code, experiment_headers = yield cm(experiment_url, control_result, fuzzy=True) if fuzzy_match: @@ -634,7 +643,7 @@ class CaptivePortal(httpt.HTTPTest): log.msg("") log.msg("Running vendor tests...") - self.report['vendor_tests'] = yield threads.deferToThread(self.run_vendor_tests) + self.report['vendor_tests'] = yield self.run_vendor_tests() log.msg("") log.msg("Running vendor DNS-based tests...")

1 0

[ooni-probe/master] Fix parsing of collector key in reports.
by art＠torproject.org 31 Mar '14

31 Mar '14

commit d982fedd0c28c75d545b9265c71d3226827bea6c Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Mar 28 02:23:52 2014 +0100 Fix parsing of collector key in reports. --- ooni/oonicli.py | 2 +- ooni/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 4fed4c7..7ac1e76 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -232,7 +232,7 @@ def runWithDirector(logging=True, start_tor=True): if not global_options['no-collector']: if global_options['collector']: collector = global_options['collector'] - elif config.reports['collector']: + elif 'collector' in config.reports and config.reports['collector']: collector = config.report['collector'] elif net_test_loader.collector: collector = net_test_loader.collector diff --git a/ooni/settings.py b/ooni/settings.py index 1c4f42d..97e5297 100644 --- a/ooni/settings.py +++ b/ooni/settings.py @@ -95,7 +95,7 @@ class OConfig(object): config_file_contents = '\n'.join(f.readlines()) configuration = yaml.safe_load(config_file_contents) - for setting in ['basic', 'advanced', 'privacy', 'tor']: + for setting in ['basic', 'reports', 'advanced', 'privacy', 'tor']: try: for k, v in configuration[setting].items(): getattr(self, setting)[k] = v

1 0

[ooni-probe/master] Replacing urllib2 with httpt.HTTPTest for captiveportal test
by art＠torproject.org 31 Mar '14

31 Mar '14

commit ef91259cbc20645067ca0a308a86cec86c516605 Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Tue Feb 11 07:49:53 2014 +0530 Replacing urllib2 with httpt.HTTPTest for captiveportal test --- ooni/nettests/manipulation/captiveportal.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/ooni/nettests/manipulation/captiveportal.py b/ooni/nettests/manipulation/captiveportal.py index d6d8040..59c6d60 100644 --- a/ooni/nettests/manipulation/captiveportal.py +++ b/ooni/nettests/manipulation/captiveportal.py @@ -33,11 +33,10 @@ import os import random import re import string -import urllib2 from urlparse import urlparse from twisted.python import usage -from twisted.internet import defer, threads +from twisted.internet import defer, threads, reactor from ooni import nettest from ooni.templates import httpt @@ -61,7 +60,7 @@ class UsageOptions(usage.Options): 'User agent for HTTP requests'] ] -class CaptivePortal(nettest.NetTestCase): +class CaptivePortal(httpt.HTTPTest): """ Compares content and status codes of HTTP responses, and attempts to determine if content has been altered. @@ -77,19 +76,18 @@ class CaptivePortal(nettest.NetTestCase): def http_fetch(self, url, headers={}): """ - Parses an HTTP url, fetches it, and returns a urllib2 response + Parses an HTTP url, fetches it, and returns a response object. """ url = urlparse(url).geturl() - request = urllib2.Request(url, None, headers) #XXX: HTTP Error 302: The HTTP server returned a redirect error that #would lead to an infinite loop. The last 30x error message was: Found try: - response = urllib2.urlopen(request) - response_headers = dict(response.headers) + response = threads.blockingCallFromThread(reactor,self.doRequest,url,"GET",headers) + response_headers = response.headers return response, response_headers - except urllib2.HTTPError, e: - log.err("HTTPError: %s" % e) + except: + log.err("HTTPError") return None, None def http_content_match_fuzzy_opt(self, experimental_url, control_result, @@ -111,7 +109,7 @@ class CaptivePortal(nettest.NetTestCase): response, response_headers = self.http_fetch(experimental_url, headers) - response_content = response.read() if response else None + response_content = response.body if response else None response_code = response.code if response else None if response_content is None: log.err("HTTP connection appears to have failed.") @@ -359,7 +357,7 @@ class CaptivePortal(nettest.NetTestCase): random_sld = self.get_random_url_safe_string(length) tld_list = ['.com', '.net', '.org', '.info', '.test', '.invalid'] - random_tld = urllib2.random.choice(tld_list) + random_tld = random.choice(tld_list) random_hostname = random_sld + random_tld return random_hostname @@ -523,7 +521,6 @@ class CaptivePortal(nettest.NetTestCase): control_result, headers, fuzzy) status_match = status_func(experiment_code, control_code) - if status_match and content_match: log.msg("The %s test was unable to detect" % test_name) log.msg("a captive portal.")

1 0

[ooni-probe/master] Construt TrueHeaders using string parameter
by art＠torproject.org 31 Mar '14

31 Mar '14

commit efaaaece1bd86dd76897435846b7bd4d7dc93f2a Author: Arun Pandian G <arunpandianp(a)gmail.com> Date: Tue Feb 18 21:51:09 2014 +0530 Construt TrueHeaders using string parameter --- ooni/utils/trueheaders.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index 06e5c02..31ee179 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -32,6 +32,8 @@ class TrueHeaders(http_headers.Headers): self.setRawHeaders(name, values[:]) elif type(values) is dict: self._rawHeaders[name.lower()] = values + elif type(values) is str: + self.setRawHeaders(name, values) def setRawHeaders(self, name, values): if name.lower() not in self._rawHeaders:

1 0

← Newer
1
2
3
4
5
6
7
8
...
204
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits March 2014