May 2016 - tor-commits - lists.torproject.org

[ooni-probe/master] Match status codes only when the control status code is not an error
by art＠torproject.org 30 May '16

30 May '16

commit cd36a0d059b88810695a1b952ab54e853253f17e Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed May 25 12:35:22 2016 +0200 Match status codes only when the control status code is not an error --- ooni/nettests/blocking/web_connectivity.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 8f048e1..7692d70 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -317,10 +317,11 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): self.report['headers_match'] = \ self.compare_headers(experiment_http_response) - self.report['status_code_match'] = ( - experiment_http_response.code == - self.control['http_request']['status_code'] - ) + if str(self.control['http_request']['status_code'])[0] != '5': + self.report['status_code_match'] = ( + self.control['http_request']['status_code'] == + experiment_http_response.code + ) self.report['title_match'] = self.compare_titles(experiment_http_response) @@ -397,8 +398,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): got_expected_web_page = ( (self.report['body_length_match'] is True or self.report['headers_match'] is True or - self.report['title_match']) - and self.report['status_code_match'] is True + self.report['title_match'] is True) + and self.report['status_code_match'] is not False ) if (dns_consistent == True and tcp_connect == False and

1 0

[ooni-probe/master] Add a timeout to the web_connectivity test to avoid the test timing out before it's done
by art＠torproject.org 30 May '16

30 May '16

commit a21ff3c777fb245096bc850532cf08ee0c4893d5 Author: Arturo Filastò <arturo(a)filasto.net> Date: Wed May 25 15:28:20 2016 +0200 Add a timeout to the web_connectivity test to avoid the test timing out before it's done --- ooni/nettests/blocking/web_connectivity.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 7692d70..41f6c2a 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -50,6 +50,7 @@ class UsageOptions(usage.Options): ['dns-discovery', 'd', 'whoami.akamai.net', 'Specify the dns discovery test helper'], ['backend', 'b', None, 'The web_consistency backend test helper'], ['retries', 'r', 1, 'Number of retries for the HTTP request'], + ['timeout', 't', 240, 'Total timeout for this test'], ] @@ -163,6 +164,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): except ValueError: self.localOptions['retries'] = 2 + self.timeout = int(self.localOptions['timeout']) + self.report['retries'] = self.localOptions['retries'] self.report['client_resolver'] = self.resolverIp self.report['dns_consistency'] = None @@ -237,6 +240,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): result['status']['success'] = False result['status']['failure'] = failureToString(failure) self.report['tcp_connect'].append(result) + return d @defer.inlineCallbacks @@ -249,6 +253,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): @defer.inlineCallbacks def experiment_http_get_request(self): + log.msg("* doing HTTP(s) request {}".format(self.input)) retries = 0 while True: try: @@ -256,8 +261,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): headers=REQUEST_HEADERS) break except: - if self.localOptions['retries'] > retries: + if retries > self.localOptions['retries']: + log.debug("Finished all the allowed retries") raise + log.debug("Re-running HTTP request") retries += 1 defer.returnValue(result) @@ -470,15 +477,18 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http = self.experiment_http_get_request() @experiment_http.addErrback def http_experiment_err(failure): - self.report['http_experiment_failure'] = failureToString(failure) + failure_string = failureToString(failure) + log.err("Failed to perform HTTP request %s" % failure_string) + self.report['http_experiment_failure'] = failure_string experiment_http_response = yield experiment_http control_request = self.control_request(sockets) @control_request.addErrback def control_err(failure): - log.err("Failed to perform control lookup") - self.report['control_failure'] = failureToString(failure) + failure_string = failureToString(failure) + log.err("Failed to perform control lookup: %s" % failure_string) + self.report['control_failure'] = failure_string yield control_request

1 0

[ooni-probe/master] Inject the correct url when following redirects
by art＠torproject.org 30 May '16

30 May '16

commit 6380468033cd7cca542184ebe8d2b4148aaec2b7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Tue May 24 18:25:40 2016 +0200 Inject the correct url when following redirects --- ooni/templates/httpt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index ca0306b..6e0154c 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -196,6 +196,9 @@ class HTTPTest(NetTestCase): if (config.privacy.includeip is False and config.probe_ip.address is not None and (isinstance(response_body, str) or isinstance(response_body, unicode))): response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") + if (getattr(response, 'request', None) and + getattr(response.request, 'absoluteURI', None)): + session['request']['url'] = response.request.absoluteURI session['response'] = { 'headers': _representHeaders(response.headers), 'body': response_body,

1 0

[ooni-probe/master] Share code from ooni-backend via the ooni.common module
by art＠torproject.org 30 May '16

30 May '16

commit 3aa17317012aa0c85cff77d6942249f0bd68f8e6 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat May 28 15:45:05 2016 +0200 Share code from ooni-backend via the ooni.common module --- ooni/backend_client.py | 2 +- ooni/common/__init__.py | 10 + ooni/common/http_utils.py | 55 +++++ ooni/common/tcp_utils.py | 10 + ooni/common/txextra.py | 202 +++++++++++++++++ ooni/nettests/blocking/web_connectivity.py | 25 +-- .../manipulation/http_header_field_manipulation.py | 2 +- ooni/templates/httpt.py | 42 +--- ooni/templates/scapyt.py | 8 +- ooni/tests/test_trueheaders.py | 2 +- ooni/utils/__init__.py | 8 - ooni/utils/net.py | 8 - ooni/utils/socks.py | 25 +++ ooni/utils/trueheaders.py | 241 --------------------- 14 files changed, 322 insertions(+), 318 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py index de01ceb..d4c463e 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -16,7 +16,7 @@ from ooni import errors as e from ooni.settings import config from ooni.utils import log, onion from ooni.utils.net import BodyReceiver, StringProducer, Downloader -from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent +from ooni.utils.socks import TrueHeadersSOCKS5Agent class OONIBClient(object): diff --git a/ooni/common/__init__.py b/ooni/common/__init__.py new file mode 100644 index 0000000..7f6cf73 --- /dev/null +++ b/ooni/common/__init__.py @@ -0,0 +1,10 @@ +""" +This modules contains functionality that is shared amongst ooni-probe and +ooni-backend. If the code in here starts growing too much I think it would +make sense to either: + + * Make the code in here into it's own package that is imported by + ooni-probe and ooni-backend. + + * Merge ooniprobe with oonibackend. +""" diff --git a/ooni/common/http_utils.py b/ooni/common/http_utils.py new file mode 100644 index 0000000..6d636d5 --- /dev/null +++ b/ooni/common/http_utils.py @@ -0,0 +1,55 @@ +import re +import codecs +from base64 import b64encode + +META_CHARSET_REGEXP = re.compile('<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"\']*([^\s"\'/>]*)') + +def representBody(body): + if not body: + return body + # XXX perhaps add support for decoding gzip in the future. + body = body.replace('\0', '') + decoded = False + charsets = ['ascii', 'utf-8'] + + # If we are able to detect the charset of body from the meta tag + # try to decode using that one first + charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE) + if charset: + try: + encoding = charset.group(1).lower() + codecs.lookup(encoding) + charsets.insert(0, encoding) + except (LookupError, IndexError): + # Skip invalid codecs and partial regexp match + pass + + for encoding in charsets: + try: + body = unicode(body, encoding) + decoded = True + break + except UnicodeDecodeError: + pass + if not decoded: + body = { + 'data': b64encode(body), + 'format': 'base64' + } + return body + +TITLE_REGEXP = re.compile("<title>(.*?)</title>", re.IGNORECASE | re.DOTALL) + +def extractTitle(body): + m = TITLE_REGEXP.search(body, re.IGNORECASE | re.DOTALL) + if m: + return unicode(m.group(1), errors='ignore') + return '' + +REQUEST_HEADERS = { + 'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, ' + 'like Gecko) Chrome/47.0.2526.106 Safari/537.36'], + 'Accept-Language': ['en-US;q=0.8,en;q=0.5'], + 'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,' + '*/*;q=0.8'] +} diff --git a/ooni/common/tcp_utils.py b/ooni/common/tcp_utils.py new file mode 100644 index 0000000..7b7a8a4 --- /dev/null +++ b/ooni/common/tcp_utils.py @@ -0,0 +1,10 @@ +from twisted.internet.protocol import Factory, Protocol + +class TCPConnectProtocol(Protocol): + def connectionMade(self): + self.transport.loseConnection() + +class TCPConnectFactory(Factory): + noisy = False + def buildProtocol(self, addr): + return TCPConnectProtocol() diff --git a/ooni/common/txextra.py b/ooni/common/txextra.py new file mode 100644 index 0000000..7a84592 --- /dev/null +++ b/ooni/common/txextra.py @@ -0,0 +1,202 @@ +import itertools +from copy import copy + +from twisted.web.http_headers import Headers +from twisted.web import error + +from twisted.web.client import BrowserLikeRedirectAgent +from twisted.web._newclient import ResponseFailed +from twisted.web._newclient import HTTPClientParser, ParseError +from twisted.python.failure import Failure + +from twisted.web import client, _newclient + +from twisted.web._newclient import RequestNotSent, RequestGenerationFailed +from twisted.web._newclient import TransportProxyProducer, STATUS + +from twisted.internet import reactor +from twisted.internet.defer import Deferred, fail, maybeDeferred, failure + +from twisted.python import log + +class TrueHeaders(Headers): + def __init__(self, rawHeaders=None): + self._rawHeaders = dict() + if rawHeaders is not None: + for name, values in rawHeaders.iteritems(): + if type(values) is list: + self.setRawHeaders(name, values[:]) + elif type(values) is str: + self.setRawHeaders(name, values) + + def setRawHeaders(self, name, values): + if name.lower() not in self._rawHeaders: + self._rawHeaders[name.lower()] = dict() + self._rawHeaders[name.lower()]['name'] = name + self._rawHeaders[name.lower()]['values'] = values + + def getAllRawHeaders(self): + for _, v in self._rawHeaders.iteritems(): + yield v['name'], v['values'] + + def getRawHeaders(self, name, default=None): + if name.lower() in self._rawHeaders: + return self._rawHeaders[name.lower()]['values'] + return default + + + def getDiff(self, headers, ignore=[]): + """ + + Args: + + headers: a TrueHeaders object + + ignore: specify a list of header fields to ignore + + Returns: + + a set containing the header names that are not present in + header_dict or not present in self. + """ + diff = set() + field_names = [] + + headers_a = copy(self) + headers_b = copy(headers) + for name in ignore: + try: + del headers_a._rawHeaders[name.lower()] + except KeyError: + pass + try: + del headers_b._rawHeaders[name.lower()] + except KeyError: + pass + + for k, v in itertools.chain(headers_a.getAllRawHeaders(), + headers_b.getAllRawHeaders()): + field_names.append(k) + + for name in field_names: + if self.getRawHeaders(name) and headers.getRawHeaders(name): + pass + else: + diff.add(name) + return list(diff) + +class HTTPClientParser(_newclient.HTTPClientParser): + def logPrefix(self): + return 'HTTPClientParser' + + def connectionMade(self): + self.headers = TrueHeaders() + self.connHeaders = TrueHeaders() + self.state = STATUS + self._partialHeader = None + + def headerReceived(self, name, value): + if self.isConnectionControlHeader(name.lower()): + headers = self.connHeaders + else: + headers = self.headers + headers.addRawHeader(name, value) + + def statusReceived(self, status): + # This is a fix for invalid number of parts + try: + return _newclient.HTTPClientParser.statusReceived(self, status) + except ParseError as exc: + if exc.args[0] == 'wrong number of parts': + return _newclient.HTTPClientParser.statusReceived(self, + status + " XXX") + raise + +class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol): + def request(self, request): + if self._state != 'QUIESCENT': + return fail(RequestNotSent()) + + self._state = 'TRANSMITTING' + _requestDeferred = maybeDeferred(request.writeTo, self.transport) + self._finishedRequest = Deferred() + + self._currentRequest = request + + self._transportProxy = TransportProxyProducer(self.transport) + self._parser = HTTPClientParser(request, self._finishResponse) + self._parser.makeConnection(self._transportProxy) + self._responseDeferred = self._parser._responseDeferred + + def cbRequestWrotten(ignored): + if self._state == 'TRANSMITTING': + self._state = 'WAITING' + self._responseDeferred.chainDeferred(self._finishedRequest) + + def ebRequestWriting(err): + if self._state == 'TRANSMITTING': + self._state = 'GENERATION_FAILED' + self.transport.loseConnection() + self._finishedRequest.errback( + failure.Failure(RequestGenerationFailed([err]))) + else: + log.err(err, 'Error writing request, but not in valid state ' + 'to finalize request: %s' % self._state) + + _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting) + + return self._finishedRequest + + +class _HTTP11ClientFactory(client._HTTP11ClientFactory): + noisy = False + + def buildProtocol(self, addr): + return HTTP11ClientProtocol(self._quiescentCallback) + + +class HTTPConnectionPool(client.HTTPConnectionPool): + _factory = _HTTP11ClientFactory + +class TrueHeadersAgent(client.Agent): + def __init__(self, *args, **kw): + super(TrueHeadersAgent, self).__init__(*args, **kw) + self._pool = HTTPConnectionPool(reactor, False) + +class FixedRedirectAgent(BrowserLikeRedirectAgent): + """ + This is a redirect agent with this patch manually applied: + https://twistedmatrix.com/trac/ticket/8265 + """ + def _handleRedirect(self, response, method, uri, headers, redirectCount): + """ + Handle a redirect response, checking the number of redirects already + followed, and extracting the location header fields. + + This is patched to fix a bug in infinite redirect loop. + """ + if redirectCount >= self._redirectLimit: + err = error.InfiniteRedirection( + response.code, + b'Infinite redirection detected', + location=uri) + raise ResponseFailed([Failure(err)], response) + locationHeaders = response.headers.getRawHeaders(b'location', []) + if not locationHeaders: + err = error.RedirectWithNoLocation( + response.code, b'No location header field', uri) + raise ResponseFailed([Failure(err)], response) + location = self._resolveLocation( + # This is the fix to properly handle redirects + response.request.absoluteURI, + locationHeaders[0] + ) + deferred = self._agent.request(method, location, headers) + + def _chainResponse(newResponse): + newResponse.setPreviousResponse(response) + return newResponse + + deferred.addCallback(_chainResponse) + return deferred.addCallback( + self._handleResponse, method, uri, headers, redirectCount + 1) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 41f6c2a..655bf76 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -7,7 +7,6 @@ from ipaddr import IPv4Address, AddressValueError from twisted.web.client import GzipDecoder from twisted.internet import reactor -from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.names import client, dns @@ -19,31 +18,17 @@ from ooni.utils import log from ooni.backend_client import WebConnectivityClient -from ooni.utils.net import COMMON_SERVER_HEADERS, extract_title +from ooni.common.http_utils import extractTitle +from ooni.utils.net import COMMON_SERVER_HEADERS from ooni.templates import httpt, dnst from ooni.errors import failureToString -REQUEST_HEADERS = { - 'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, ' - 'like Gecko) Chrome/47.0.2526.106 Safari/537.36'], - 'Accept-Language': ['en-US;q=0.8,en;q=0.5'], - 'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,' - '*/*;q=0.8'] -} +from ooni.common.tcp_utils import TCPConnectFactory +from ooni.common.http_utils import REQUEST_HEADERS class InvalidControlResponse(Exception): pass -class TCPConnectProtocol(Protocol): - def connectionMade(self): - self.transport.loseConnection() - -class TCPConnectFactory(Factory): - noisy = False - def buildProtocol(self, addr): - return TCPConnectProtocol() - - class UsageOptions(usage.Options): optParameters = [ ['url', 'u', None, 'Specify a single URL to test'], @@ -306,7 +291,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): return False def compare_titles(self, experiment_http_response): - experiment_title = extract_title(experiment_http_response.body).strip() + experiment_title = extractTitle(experiment_http_response.body).strip() control_title = self.control['http_request']['title'].strip() first_exp_word = experiment_title.split(' ')[0] first_ctrl_word = control_title.split(' ')[0] diff --git a/ooni/nettests/manipulation/http_header_field_manipulation.py b/ooni/nettests/manipulation/http_header_field_manipulation.py index 6fbaf7d..fcd5e0e 100644 --- a/ooni/nettests/manipulation/http_header_field_manipulation.py +++ b/ooni/nettests/manipulation/http_header_field_manipulation.py @@ -11,7 +11,7 @@ from twisted.python import usage from ooni.utils import log, net, randomStr from ooni.templates import httpt -from ooni.utils.trueheaders import TrueHeaders +from ooni.common.txextra import TrueHeaders def random_capitalization(string): diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 6e0154c..f8ea941 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -10,15 +10,16 @@ from twisted.web.client import ContentDecoderAgent from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint -from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent -from ooni.utils.trueheaders import FixedRedirectAgent +from ooni.utils.socks import TrueHeadersSOCKS5Agent from ooni.nettest import NetTestCase -from ooni.utils import log, base64Dict +from ooni.utils import log from ooni.settings import config from ooni.utils.net import StringProducer, userAgents -from ooni.utils.trueheaders import TrueHeaders +from ooni.common.txextra import TrueHeaders +from ooni.common.txextra import FixedRedirectAgent, TrueHeadersAgent +from ooni.common.http_utils import representBody from ooni.errors import handleAllFailures META_CHARSET_REGEXP = re.compile('<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"\']*([^\s"\'/>!;]+)') @@ -48,37 +49,6 @@ def _representHeaders(headers): represented_headers[name] = unicode(value[0], errors='ignore') return represented_headers -def _representBody(body): - if not body: - return body - # XXX perhaps add support for decoding gzip in the future. - body = body.replace('\0', '') - decoded = False - charsets = ['ascii', 'utf-8'] - - # If we are able to detect the charset of body from the meta tag - # try to decode using that one first - charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE) - if charset: - try: - encoding = charset.group(1).lower() - codecs.lookup(encoding) - charsets.insert(0, encoding) - except (LookupError, IndexError): - # Skip invalid codecs and partial regexp match - pass - - for encoding in charsets: - try: - body = unicode(body, encoding) - decoded = True - break - except UnicodeDecodeError: - pass - if not decoded: - body = base64Dict(body) - return body - class HTTPTest(NetTestCase): """ A utility class for dealing with HTTP based testing. It provides methods to @@ -189,7 +159,7 @@ class HTTPTest(NetTestCase): } if response: if self.localOptions.get('withoutbody', 0) is 0: - response_body = _representBody(response_body) + response_body = representBody(response_body) else: response_body = '' # Attempt to redact the IP address of the probe from the responses diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 2bb9139..1c726d8 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -1,5 +1,6 @@ +from base64 import b64encode from ooni.nettest import NetTestCase -from ooni.utils import log, base64Dict +from ooni.utils import log from ooni.settings import config from ooni.utils.net import hasRawSocketPermission @@ -8,7 +9,10 @@ from ooni.utils.txscapy import ScapySender, ScapyFactory def _representPacket(packet): return { - "raw_packet": base64Dict(str(packet)), + "raw_packet": { + 'data': b64encode(str(packet)), + 'format': 'base64' + }, "summary": repr(packet) } diff --git a/ooni/tests/test_trueheaders.py b/ooni/tests/test_trueheaders.py index f6c7812..191dafd 100644 --- a/ooni/tests/test_trueheaders.py +++ b/ooni/tests/test_trueheaders.py @@ -1,6 +1,6 @@ from twisted.trial import unittest -from ooni.utils.trueheaders import TrueHeaders +from ooni.common.txextra import TrueHeaders dummy_headers_dict = { 'Header1': ['Value1', 'Value2'], diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 22af062..f85cb38 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -1,12 +1,10 @@ import shutil import string import random -import glob import os from datetime import datetime import gzip -from base64 import b64encode from zipfile import ZipFile from ooni import errors @@ -155,9 +153,3 @@ def gunzip(filename, dst): def get_ooni_root(): script = os.path.join(__file__, '..') return os.path.dirname(os.path.realpath(script)) - -def base64Dict(data): - return { - 'format': 'base64', - 'data': b64encode(data) - } diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 20f5a42..9918b60 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -81,14 +81,6 @@ COMMON_SERVER_HEADERS = ( # fetching URLs over some proxy. GOOGLE_HUMANS = ('http://www.google.com/humans.txt', 'Google is built by a large') -TITLE_REGEXP = re.compile("<title>(.*?)</title>", re.IGNORECASE | re.DOTALL) - -def extract_title(body): - m = TITLE_REGEXP.search(body) - if m: - return m.group(1) - return '' - class StringProducer(object): implements(IBodyProducer) diff --git a/ooni/utils/socks.py b/ooni/utils/socks.py new file mode 100644 index 0000000..783db55 --- /dev/null +++ b/ooni/utils/socks.py @@ -0,0 +1,25 @@ +from twisted.internet import reactor +from ooni.common.txextra import HTTPConnectionPool + +from twisted import version as twisted_version +from twisted.python.versions import Version +_twisted_15_0 = Version('twisted', 15, 0, 0) + +from txsocksx.http import SOCKS5Agent +from txsocksx.client import SOCKS5ClientFactory + +SOCKS5ClientFactory.noisy = False + +class TrueHeadersSOCKS5Agent(SOCKS5Agent): + def __init__(self, *args, **kw): + super(TrueHeadersSOCKS5Agent, self).__init__(*args, **kw) + pool = HTTPConnectionPool(reactor, False) + # + # With Twisted > 15.0 txsocksx wraps the twisted agent using a + # wrapper class, hence we must set the _pool attribute in the + # inner class rather than into its external wrapper. + # + if twisted_version >= _twisted_15_0: + self._wrappedAgent._pool = pool + else: + self._pool = pool diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py deleted file mode 100644 index 5f2b399..0000000 --- a/ooni/utils/trueheaders.py +++ /dev/null @@ -1,241 +0,0 @@ -# :authors: Giovanni Pellerano -# :licence: see LICENSE -# -# Here we make sure that the HTTP Headers sent and received are True. By this -# we mean that they are not normalized and that the ordering is maintained. - -import itertools -from copy import copy - -from twisted.python.failure import Failure - -from twisted.web import client, _newclient, http_headers, error - -from twisted.web._newclient import RequestNotSent, RequestGenerationFailed -from twisted.web._newclient import TransportProxyProducer, STATUS -from twisted.web._newclient import ResponseFailed - -from twisted.internet import reactor -from twisted.internet.defer import Deferred, fail, maybeDeferred, failure - -from txsocksx.http import SOCKS5Agent -from txsocksx.client import SOCKS5ClientFactory - -SOCKS5ClientFactory.noisy = False - -from ooni.utils import log - -import twisted -from twisted.python.versions import Version - - -class TrueHeaders(http_headers.Headers): - def __init__(self, rawHeaders=None): - self._rawHeaders = dict() - if rawHeaders is not None: - for name, values in rawHeaders.iteritems(): - if type(values) is list: - self.setRawHeaders(name, values[:]) - elif type(values) is dict: - self._rawHeaders[name.lower()] = values - elif type(values) is str: - self.setRawHeaders(name, values) - - def setRawHeaders(self, name, values): - if name.lower() not in self._rawHeaders: - self._rawHeaders[name.lower()] = dict() - self._rawHeaders[name.lower()]['name'] = name - self._rawHeaders[name.lower()]['values'] = values - - def getDiff(self, headers, ignore=[]): - """ - - Args: - - headers: a TrueHeaders object - - ignore: specify a list of header fields to ignore - - Returns: - - a set containing the header names that are not present in - header_dict or not present in self. - """ - diff = set() - field_names = [] - - headers_a = copy(self) - headers_b = copy(headers) - for name in ignore: - try: - del headers_a._rawHeaders[name.lower()] - except KeyError: - pass - try: - del headers_b._rawHeaders[name.lower()] - except KeyError: - pass - - for k, v in itertools.chain(headers_a.getAllRawHeaders(), - headers_b.getAllRawHeaders()): - field_names.append(k) - - for name in field_names: - if self.getRawHeaders(name) and headers.getRawHeaders(name): - pass - else: - diff.add(name) - return list(diff) - - def getAllRawHeaders(self): - for k, v in self._rawHeaders.iteritems(): - yield v['name'], v['values'] - - def getRawHeaders(self, name, default=None): - if name.lower() in self._rawHeaders: - return self._rawHeaders[name.lower()]['values'] - return default - - -class HTTPClientParser(_newclient.HTTPClientParser): - def logPrefix(self): - return 'HTTPClientParser' - - def connectionMade(self): - self.headers = TrueHeaders() - self.connHeaders = TrueHeaders() - self.state = STATUS - self._partialHeader = None - - def headerReceived(self, name, value): - if self.isConnectionControlHeader(name.lower()): - headers = self.connHeaders - else: - headers = self.headers - headers.addRawHeader(name, value) - - - def statusReceived(self, status): - parts = status.split(b' ', 2) - if len(parts) != 3: - # Here we add the extra missing part. - parts.append("XXX") - - try: - statusCode = int(parts[1]) - except ValueError: - raise _newclient.ParseError(u"non-integer status code", status) - - self.response = _newclient.Response._construct( - self.parseVersion(parts[0]), - statusCode, - parts[2], - self.headers, - self.transport, - self.request) - - -class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol): - def request(self, request): - if self._state != 'QUIESCENT': - return fail(RequestNotSent()) - - self._state = 'TRANSMITTING' - _requestDeferred = maybeDeferred(request.writeTo, self.transport) - self._finishedRequest = Deferred() - - self._currentRequest = request - - self._transportProxy = TransportProxyProducer(self.transport) - self._parser = HTTPClientParser(request, self._finishResponse) - self._parser.makeConnection(self._transportProxy) - self._responseDeferred = self._parser._responseDeferred - - def cbRequestWrotten(ignored): - if self._state == 'TRANSMITTING': - self._state = 'WAITING' - self._responseDeferred.chainDeferred(self._finishedRequest) - - def ebRequestWriting(err): - if self._state == 'TRANSMITTING': - self._state = 'GENERATION_FAILED' - self.transport.loseConnection() - self._finishedRequest.errback( - failure.Failure(RequestGenerationFailed([err]))) - else: - log.err(err, 'Error writing request, but not in valid state ' - 'to finalize request: %s' % self._state) - - _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting) - - return self._finishedRequest - - -class _HTTP11ClientFactory(client._HTTP11ClientFactory): - noisy = False - - def buildProtocol(self, addr): - return HTTP11ClientProtocol(self._quiescentCallback) - - -class HTTPConnectionPool(client.HTTPConnectionPool): - _factory = _HTTP11ClientFactory - - -class TrueHeadersAgent(client.Agent): - def __init__(self, *args, **kw): - super(TrueHeadersAgent, self).__init__(*args, **kw) - self._pool = HTTPConnectionPool(reactor, False) - - -_twisted_15_0 = Version('twisted', 15, 0, 0) - - -class TrueHeadersSOCKS5Agent(SOCKS5Agent): - def __init__(self, *args, **kw): - super(TrueHeadersSOCKS5Agent, self).__init__(*args, **kw) - pool = HTTPConnectionPool(reactor, False) - # - # With Twisted > 15.0 txsocksx wraps the twisted agent using a - # wrapper class, hence we must set the _pool attribute in the - # inner class rather than into its external wrapper. - # - if twisted.version >= _twisted_15_0: - self._wrappedAgent._pool = pool - else: - self._pool = pool - -class FixedRedirectAgent(client.BrowserLikeRedirectAgent): - """ - This is a redirect agent with this patch manually applied: - https://twistedmatrix.com/trac/ticket/8265 - """ - def _handleRedirect(self, response, method, uri, headers, redirectCount): - """ - Handle a redirect response, checking the number of redirects already - followed, and extracting the location header fields. - - This is pathed to fix a bug in infinite redirect loop. - """ - if redirectCount >= self._redirectLimit: - err = error.InfiniteRedirection( - response.code, - b'Infinite redirection detected', - location=uri) - raise ResponseFailed([Failure(err)], response) - locationHeaders = response.headers.getRawHeaders(b'location', []) - if not locationHeaders: - err = error.RedirectWithNoLocation( - response.code, b'No location header field', uri) - raise ResponseFailed([Failure(err)], response) - location = self._resolveLocation(response.request.absoluteURI, locationHeaders[0]) - deferred = self._agent.request(method, location, headers) - - def _chainResponse(newResponse): - newResponse.setPreviousResponse(response) - return newResponse - - deferred.addCallback(_chainResponse) - # This is the fix to properly handle redirects - return deferred.addCallback( - self._handleResponse, method, uri, headers, redirectCount + 1)

1 0

[ooni-probe/master] Write a msg when we perform the control lookup
by art＠torproject.org 30 May '16

30 May '16

commit c046653fbd63ff6beb35aa1d87d0bbe5e0014fef Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 30 15:18:31 2016 +0200 Write a msg when we perform the control lookup Be more lax when detecting header key similarity --- ooni/nettests/blocking/web_connectivity.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index b4ccff1..0dba239 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -229,6 +229,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): @defer.inlineCallbacks def control_request(self, sockets): + log.msg("* performing control request with backend") self.control = yield self.web_connectivity_client.control( http_request=self.input, tcp_connect=sockets @@ -261,6 +262,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): experiment_http_response.headers.getAllRawHeaders() } + if (set(control_headers_lower.keys()) == + set(experiment_headers_lower.keys())): + return True + uncommon_ctrl_headers = (set(control_headers_lower.keys()) - set(COMMON_SERVER_HEADERS)) uncommon_exp_headers = (set(experiment_headers_lower.keys()) -

1 0

[ooni-probe/master] Address feedback by @bassosimone
by art＠torproject.org 30 May '16

30 May '16

commit 8e7199d85c877353b83b3da98936c602d4c4a5b4 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 30 10:42:29 2016 +0200 Address feedback by @bassosimone * Test the next words if the word is less than 5 chars in web_connectivity test. * Move the charset detection charset into the correct module and change the unittest accordingly. * Include ooni.common in setup.py * Various code style fixes --- ooni/backend_client.py | 6 +----- ooni/common/http_utils.py | 2 +- ooni/nettests/blocking/web_connectivity.py | 21 ++++++++++++--------- ooni/templates/httpt.py | 4 ---- ooni/tests/test_common.py | 19 +++++++++++++++++++ ooni/tests/test_templates.py | 16 ---------------- ooni/utils/net.py | 1 - setup.py | 1 + 8 files changed, 34 insertions(+), 36 deletions(-) diff --git a/ooni/backend_client.py b/ooni/backend_client.py index d4c463e..0e85dd7 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -207,9 +207,7 @@ class CollectorClient(OONIBClient): @d.addErrback def err(failure): failure.trap(Error) - if failure.value.status == '404': - return True - return False + return failure.value.status == '404' return d @@ -345,8 +343,6 @@ class CollectorClient(OONIBClient): class WebConnectivityClient(OONIBClient): def isReachable(self): - # XXX maybe in the future we can have a dedicated API endpoint to - # test the reachability of the collector. d = self.queryBackend('GET', '/status') @d.addCallback diff --git a/ooni/common/http_utils.py b/ooni/common/http_utils.py index 6d636d5..57c3a15 100644 --- a/ooni/common/http_utils.py +++ b/ooni/common/http_utils.py @@ -2,7 +2,7 @@ import re import codecs from base64 import b64encode -META_CHARSET_REGEXP = re.compile('<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"\']*([^\s"\'/>]*)') +META_CHARSET_REGEXP = re.compile('<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"\']*([^\s"\'/>!;]+)') def representBody(body): if not body: diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 655bf76..b4ccff1 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -42,10 +42,9 @@ class UsageOptions(usage.Options): def is_public_ipv4_address(address): try: ip_address = IPv4Address(address) - if not any([ip_address.is_private, - ip_address.is_loopback]): - return True - return False + return not any( + [ip_address.is_private, ip_address.is_loopback] + ) except AddressValueError: return None @@ -293,13 +292,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): def compare_titles(self, experiment_http_response): experiment_title = extractTitle(experiment_http_response.body).strip() control_title = self.control['http_request']['title'].strip() - first_exp_word = experiment_title.split(' ')[0] - first_ctrl_word = control_title.split(' ')[0] - if len(first_exp_word) < 5: + + control_words = control_title.split(' ') + for exp_word, idx in enumerate(experiment_title.split(' ')): # We don't consider to match words that are shorter than 5 # characters (5 is the average word length for english) - return False - return (first_ctrl_word.lower() == first_exp_word.lower()) + if len(exp_word) < 5: + continue + try: + return control_words[idx].lower() == exp_word.lower() + except IndexError: + return False def compare_http_experiments(self, experiment_http_response): diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index f8ea941..3f7e77d 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -1,5 +1,3 @@ -import re -import codecs import random from txtorcon.interface import StreamListenerMixin @@ -22,8 +20,6 @@ from ooni.common.txextra import FixedRedirectAgent, TrueHeadersAgent from ooni.common.http_utils import representBody from ooni.errors import handleAllFailures -META_CHARSET_REGEXP = re.compile('<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"\']*([^\s"\'/>!;]+)') - class InvalidSocksProxyOption(Exception): pass diff --git a/ooni/tests/test_common.py b/ooni/tests/test_common.py new file mode 100644 index 0000000..1cd77cf --- /dev/null +++ b/ooni/tests/test_common.py @@ -0,0 +1,19 @@ +from twisted.trial import unittest +from ooni.common.http_utils import META_CHARSET_REGEXP + +class TestHTTPUtils(unittest.TestCase): + def test_charset_detection(self): + no_charset_html = """ + <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html> +<head> + <title>Foo</title> +""" + with_charset_html = no_charset_html + '\n<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">' + with_empty_charset = no_charset_html + '\n<meta http-equiv="Content-Type" content="text/html; charset=">' + with_two_charsets = no_charset_html + '\n<meta http-equiv="Content-Type" content="text/html; charset=UTF-8;charset=utf-8">' + self.assertEqual(META_CHARSET_REGEXP.search(no_charset_html), None) + self.assertEqual(META_CHARSET_REGEXP.search(with_charset_html).group(1), 'iso-8859-1') + self.assertEqual(META_CHARSET_REGEXP.search( + with_two_charsets).group(1), 'UTF-8') + self.assertEqual(META_CHARSET_REGEXP.search(with_empty_charset), None) diff --git a/ooni/tests/test_templates.py b/ooni/tests/test_templates.py index ebd5b2e..7aa399b 100644 --- a/ooni/tests/test_templates.py +++ b/ooni/tests/test_templates.py @@ -46,22 +46,6 @@ class TestHTTPT(unittest.TestCase): yield self.assertFailure(http_test.doRequest('http://invaliddomain/'), DNSLookupError) assert http_test.report['requests'][0]['failure'] == 'dns_lookup_error' - def test_charset_detection(self): - no_charset_html = """ - <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html> -<head> - <title>Foo</title> -""" - with_charset_html = no_charset_html + '\n<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">' - with_empty_charset = no_charset_html + '\n<meta http-equiv="Content-Type" content="text/html; charset=">' - with_two_charsets = no_charset_html + '\n<meta http-equiv="Content-Type" content="text/html; charset=UTF-8;charset=utf-8">' - self.assertEqual(httpt.META_CHARSET_REGEXP.search(no_charset_html), None) - self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_charset_html).group(1), 'iso-8859-1') - self.assertEqual(httpt.META_CHARSET_REGEXP.search( - with_two_charsets).group(1), 'UTF-8') - self.assertEqual(httpt.META_CHARSET_REGEXP.search(with_empty_charset), None) - class TestDNST(unittest.TestCase): def setUp(self): if not is_internet_connected(): diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 9918b60..79d17f4 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -1,4 +1,3 @@ -import re import sys import socket from random import randint diff --git a/setup.py b/setup.py index da0b967..3bfc5cd 100644 --- a/setup.py +++ b/setup.py @@ -171,6 +171,7 @@ data_files = [] packages = [ 'ooni', 'ooni.api', + 'ooni.common', 'ooni.deckgen', 'ooni.deckgen.processors', 'ooni.kit',

1 0

[ooni-probe/master] Merge pull request #514 from TheTorProject/feature/web_connectivity
by art＠torproject.org 30 May '16

30 May '16

1 0

[ooni-probe/master] Update to version 1.5.0
by art＠torproject.org 30 May '16

30 May '16

commit 5fdaa3ebe0928bbbc8cd8088d160cf1a40dd9730 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 30 17:53:28 2016 +0200 Update to version 1.5.0 --- ChangeLog.rst | 10 ++++++++++ ooni/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 56dd061..5470570 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,16 @@ Changelog ========= +v1.5.0 (Mon, 30 May 2016) +------------------------- +codename: The Big Wave + +* Implement web_connectivity test that measures for both DNS and HTTP + censorship. + +* Fix a regression bug that lead to Tor exit ip address not being included in + reports. + v1.4.2 (Fri, 29 Apr 2016) ------------------------- diff --git a/ooni/__init__.py b/ooni/__init__.py index e02b430..59c0143 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- __author__ = "Open Observatory of Network Interference" -__version__ = "1.5.0.dev0" +__version__ = "1.5.0" # This is the version number of resources to be downloaded # when a release is made it should be aligned to __version__ __resources_version__ = "1.4.2"

1 0

[ooni-probe/master] Make the geoip lookup code more robust
by art＠torproject.org 30 May '16

30 May '16

commit 2627cd5ad2058d616cc037cd590b9f64112882a9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon May 2 17:54:36 2016 +0200 Make the geoip lookup code more robust --- ooni/geoip.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/ooni/geoip.py b/ooni/geoip.py index dcc799e..fa6d1ae 100644 --- a/ooni/geoip.py +++ b/ooni/geoip.py @@ -34,25 +34,22 @@ def IPToLocation(ipaddr): asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat') location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'} - - def error(): + if not asn_file or not country_file: log.err("Could not find GeoIP data file in data directories." "Try running ooniresources or" " edit your ooniprobe.conf") + return location - try: - country_dat = GeoIP(country_file) - location['countrycode'] = country_dat.country_code_by_addr(ipaddr) - if not location['countrycode']: - location['countrycode'] = 'ZZ' - except IOError: - error() + country_dat = GeoIP(country_file) + asn_dat = GeoIP(asn_file) - try: - asn_dat = GeoIP(asn_file) - location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0] - except: - error() + country_code = country_dat.country_code_by_addr(ipaddr) + if country_code is not None: + location['countrycode'] = country_code + + asn = asn_dat.org_by_addr(ipaddr) + if asn is not None: + location['asn'] = asn.split(' ')[0] return location

1 0

[ooni-probe/master] Also record all the redirects
by art＠torproject.org 30 May '16

30 May '16

commit 8a745093ecd27240053b7e251de46eb4c303df6f Author: Arturo Filastò <arturo(a)filasto.net> Date: Mon Apr 18 18:45:29 2016 +0200 Also record all the redirects * Add monkey patch for bug in twisted RedirectAgent: https://twistedmatrix.com/trac/ticket/8265 --- ooni/templates/httpt.py | 22 ++++++++++++++-------- ooni/utils/trueheaders.py | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 2b280f0..2a17f5b 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -7,6 +7,7 @@ from twisted.web.client import readBody, PartialDownloadError from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent +from ooni.utils.trueheaders import FixedRedirectAgent from ooni.nettest import NetTestCase from ooni.utils import log, base64Dict @@ -37,8 +38,15 @@ class StreamListener(StreamListenerMixin): log.err("Tor Exit ip detection failed") +def _representHeaders(headers): + represented_headers = {} + for name, value in headers.getAllRawHeaders(): + represented_headers[name] = value[0] + return represented_headers def _representBody(body): + if not body: + return body # XXX perhaps add support for decoding gzip in the future. body = body.replace('\0', '') decoded = False @@ -115,9 +123,8 @@ class HTTPTest(NetTestCase): if self.followRedirects: try: - from twisted.web.client import RedirectAgent - self.control_agent = RedirectAgent(self.control_agent) - self.agent = RedirectAgent(self.agent) + self.control_agent = FixedRedirectAgent(self.control_agent) + self.agent = FixedRedirectAgent(self.agent) self.report['agent'] = 'redirect' except: log.err("Warning! You are running an old version of twisted"\ @@ -147,11 +154,10 @@ class HTTPTest(NetTestCase): failure (instance): An instance of :class:twisted.internet.failure.Failure """ - def _representHeaders(headers): - represented_headers = {} - for name, value in headers.getAllRawHeaders(): - represented_headers[name] = value[0] - return represented_headers + if response and response.previousResponse: + self.addToReport(request, response.previousResponse, + response_body=None, + failure_string=None) log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers']) diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py index d5aa206..a5b2fe3 100644 --- a/ooni/utils/trueheaders.py +++ b/ooni/utils/trueheaders.py @@ -7,9 +7,14 @@ import itertools from copy import copy -from twisted.web import client, _newclient, http_headers +from twisted.python.failure import Failure + +from twisted.web import client, _newclient, http_headers, error + from twisted.web._newclient import RequestNotSent, RequestGenerationFailed from twisted.web._newclient import TransportProxyProducer, STATUS +from twisted.web._newclient import ResponseFailed + from twisted.internet import reactor from twisted.internet.defer import Deferred, fail, maybeDeferred, failure @@ -179,3 +184,38 @@ class TrueHeadersSOCKS5Agent(SOCKS5Agent): self._wrappedAgent._pool = pool else: self._pool = pool + +class FixedRedirectAgent(client.BrowserLikeRedirectAgent): + """ + This is a redirect agent with this patch manually applied: + https://twistedmatrix.com/trac/ticket/8265 + """ + def _handleRedirect(self, response, method, uri, headers, redirectCount): + """ + Handle a redirect response, checking the number of redirects already + followed, and extracting the location header fields. + + This is pathed to fix a bug in infinite redirect loop. + """ + if redirectCount >= self._redirectLimit: + err = error.InfiniteRedirection( + response.code, + b'Infinite redirection detected', + location=uri) + raise ResponseFailed([Failure(err)], response) + locationHeaders = response.headers.getRawHeaders(b'location', []) + if not locationHeaders: + err = error.RedirectWithNoLocation( + response.code, b'No location header field', uri) + raise ResponseFailed([Failure(err)], response) + location = self._resolveLocation(response.request.absoluteURI, locationHeaders[0]) + deferred = self._agent.request(method, location, headers) + + def _chainResponse(newResponse): + newResponse.setPreviousResponse(response) + return newResponse + + deferred.addCallback(_chainResponse) + # This is the fix to properly handle redirects + return deferred.addCallback( + self._handleResponse, method, uri, headers, redirectCount + 1)

1 0