commit 3aa17317012aa0c85cff77d6942249f0bd68f8e6 Author: Arturo Filastò arturo@filasto.net Date: Sat May 28 15:45:05 2016 +0200

Share code from ooni-backend via the ooni.common module --- ooni/backend_client.py | 2 +- ooni/common/__init__.py | 10 + ooni/common/http_utils.py | 55 +++++ ooni/common/tcp_utils.py | 10 + ooni/common/txextra.py | 202 +++++++++++++++++ ooni/nettests/blocking/web_connectivity.py | 25 +-- .../manipulation/http_header_field_manipulation.py | 2 +- ooni/templates/httpt.py | 42 +--- ooni/templates/scapyt.py | 8 +- ooni/tests/test_trueheaders.py | 2 +- ooni/utils/__init__.py | 8 - ooni/utils/net.py | 8 - ooni/utils/socks.py | 25 +++ ooni/utils/trueheaders.py | 241 --------------------- 14 files changed, 322 insertions(+), 318 deletions(-)

diff --git a/ooni/backend_client.py b/ooni/backend_client.py index de01ceb..d4c463e 100644 --- a/ooni/backend_client.py +++ b/ooni/backend_client.py @@ -16,7 +16,7 @@ from ooni import errors as e from ooni.settings import config from ooni.utils import log, onion from ooni.utils.net import BodyReceiver, StringProducer, Downloader -from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent +from ooni.utils.socks import TrueHeadersSOCKS5Agent

class OONIBClient(object): diff --git a/ooni/common/__init__.py b/ooni/common/__init__.py new file mode 100644 index 0000000..7f6cf73 --- /dev/null +++ b/ooni/common/__init__.py @@ -0,0 +1,10 @@ +""" +This modules contains functionality that is shared amongst ooni-probe and +ooni-backend. If the code in here starts growing too much I think it would +make sense to either: + + * Make the code in here into it's own package that is imported by + ooni-probe and ooni-backend. + + * Merge ooniprobe with oonibackend. +""" diff --git a/ooni/common/http_utils.py b/ooni/common/http_utils.py new file mode 100644 index 0000000..6d636d5 --- /dev/null +++ b/ooni/common/http_utils.py @@ -0,0 +1,55 @@ +import re +import codecs +from base64 import b64encode + +META_CHARSET_REGEXP = re.compile('<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)') + +def representBody(body): + if not body: + return body + # XXX perhaps add support for decoding gzip in the future. + body = body.replace('\0', '') + decoded = False + charsets = ['ascii', 'utf-8'] + + # If we are able to detect the charset of body from the meta tag + # try to decode using that one first + charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE) + if charset: + try: + encoding = charset.group(1).lower() + codecs.lookup(encoding) + charsets.insert(0, encoding) + except (LookupError, IndexError): + # Skip invalid codecs and partial regexp match + pass + + for encoding in charsets: + try: + body = unicode(body, encoding) + decoded = True + break + except UnicodeDecodeError: + pass + if not decoded: + body = { + 'data': b64encode(body), + 'format': 'base64' + } + return body + +TITLE_REGEXP = re.compile("<title>(.*?)</title>", re.IGNORECASE | re.DOTALL) + +def extractTitle(body): + m = TITLE_REGEXP.search(body, re.IGNORECASE | re.DOTALL) + if m: + return unicode(m.group(1), errors='ignore') + return '' + +REQUEST_HEADERS = { + 'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, ' + 'like Gecko) Chrome/47.0.2526.106 Safari/537.36'], + 'Accept-Language': ['en-US;q=0.8,en;q=0.5'], + 'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,' + '*/*;q=0.8'] +} diff --git a/ooni/common/tcp_utils.py b/ooni/common/tcp_utils.py new file mode 100644 index 0000000..7b7a8a4 --- /dev/null +++ b/ooni/common/tcp_utils.py @@ -0,0 +1,10 @@ +from twisted.internet.protocol import Factory, Protocol + +class TCPConnectProtocol(Protocol): + def connectionMade(self): + self.transport.loseConnection() + +class TCPConnectFactory(Factory): + noisy = False + def buildProtocol(self, addr): + return TCPConnectProtocol() diff --git a/ooni/common/txextra.py b/ooni/common/txextra.py new file mode 100644 index 0000000..7a84592 --- /dev/null +++ b/ooni/common/txextra.py @@ -0,0 +1,202 @@ +import itertools +from copy import copy + +from twisted.web.http_headers import Headers +from twisted.web import error + +from twisted.web.client import BrowserLikeRedirectAgent +from twisted.web._newclient import ResponseFailed +from twisted.web._newclient import HTTPClientParser, ParseError +from twisted.python.failure import Failure + +from twisted.web import client, _newclient + +from twisted.web._newclient import RequestNotSent, RequestGenerationFailed +from twisted.web._newclient import TransportProxyProducer, STATUS + +from twisted.internet import reactor +from twisted.internet.defer import Deferred, fail, maybeDeferred, failure + +from twisted.python import log + +class TrueHeaders(Headers): + def __init__(self, rawHeaders=None): + self._rawHeaders = dict() + if rawHeaders is not None: + for name, values in rawHeaders.iteritems(): + if type(values) is list: + self.setRawHeaders(name, values[:]) + elif type(values) is str: + self.setRawHeaders(name, values) + + def setRawHeaders(self, name, values): + if name.lower() not in self._rawHeaders: + self._rawHeaders[name.lower()] = dict() + self._rawHeaders[name.lower()]['name'] = name + self._rawHeaders[name.lower()]['values'] = values + + def getAllRawHeaders(self): + for _, v in self._rawHeaders.iteritems(): + yield v['name'], v['values'] + + def getRawHeaders(self, name, default=None): + if name.lower() in self._rawHeaders: + return self._rawHeaders[name.lower()]['values'] + return default + + + def getDiff(self, headers, ignore=[]): + """ + + Args: + + headers: a TrueHeaders object + + ignore: specify a list of header fields to ignore + + Returns: + + a set containing the header names that are not present in + header_dict or not present in self. + """ + diff = set() + field_names = [] + + headers_a = copy(self) + headers_b = copy(headers) + for name in ignore: + try: + del headers_a._rawHeaders[name.lower()] + except KeyError: + pass + try: + del headers_b._rawHeaders[name.lower()] + except KeyError: + pass + + for k, v in itertools.chain(headers_a.getAllRawHeaders(), + headers_b.getAllRawHeaders()): + field_names.append(k) + + for name in field_names: + if self.getRawHeaders(name) and headers.getRawHeaders(name): + pass + else: + diff.add(name) + return list(diff) + +class HTTPClientParser(_newclient.HTTPClientParser): + def logPrefix(self): + return 'HTTPClientParser' + + def connectionMade(self): + self.headers = TrueHeaders() + self.connHeaders = TrueHeaders() + self.state = STATUS + self._partialHeader = None + + def headerReceived(self, name, value): + if self.isConnectionControlHeader(name.lower()): + headers = self.connHeaders + else: + headers = self.headers + headers.addRawHeader(name, value) + + def statusReceived(self, status): + # This is a fix for invalid number of parts + try: + return _newclient.HTTPClientParser.statusReceived(self, status) + except ParseError as exc: + if exc.args[0] == 'wrong number of parts': + return _newclient.HTTPClientParser.statusReceived(self, + status + " XXX") + raise + +class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol): + def request(self, request): + if self._state != 'QUIESCENT': + return fail(RequestNotSent()) + + self._state = 'TRANSMITTING' + _requestDeferred = maybeDeferred(request.writeTo, self.transport) + self._finishedRequest = Deferred() + + self._currentRequest = request + + self._transportProxy = TransportProxyProducer(self.transport) + self._parser = HTTPClientParser(request, self._finishResponse) + self._parser.makeConnection(self._transportProxy) + self._responseDeferred = self._parser._responseDeferred + + def cbRequestWrotten(ignored): + if self._state == 'TRANSMITTING': + self._state = 'WAITING' + self._responseDeferred.chainDeferred(self._finishedRequest) + + def ebRequestWriting(err): + if self._state == 'TRANSMITTING': + self._state = 'GENERATION_FAILED' + self.transport.loseConnection() + self._finishedRequest.errback( + failure.Failure(RequestGenerationFailed([err]))) + else: + log.err(err, 'Error writing request, but not in valid state ' + 'to finalize request: %s' % self._state) + + _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting) + + return self._finishedRequest + + +class _HTTP11ClientFactory(client._HTTP11ClientFactory): + noisy = False + + def buildProtocol(self, addr): + return HTTP11ClientProtocol(self._quiescentCallback) + + +class HTTPConnectionPool(client.HTTPConnectionPool): + _factory = _HTTP11ClientFactory + +class TrueHeadersAgent(client.Agent): + def __init__(self, *args, **kw): + super(TrueHeadersAgent, self).__init__(*args, **kw) + self._pool = HTTPConnectionPool(reactor, False) + +class FixedRedirectAgent(BrowserLikeRedirectAgent): + """ + This is a redirect agent with this patch manually applied: + https://twistedmatrix.com/trac/ticket/8265 + """ + def _handleRedirect(self, response, method, uri, headers, redirectCount): + """ + Handle a redirect response, checking the number of redirects already + followed, and extracting the location header fields. + + This is patched to fix a bug in infinite redirect loop. + """ + if redirectCount >= self._redirectLimit: + err = error.InfiniteRedirection( + response.code, + b'Infinite redirection detected', + location=uri) + raise ResponseFailed([Failure(err)], response) + locationHeaders = response.headers.getRawHeaders(b'location', []) + if not locationHeaders: + err = error.RedirectWithNoLocation( + response.code, b'No location header field', uri) + raise ResponseFailed([Failure(err)], response) + location = self._resolveLocation( + # This is the fix to properly handle redirects + response.request.absoluteURI, + locationHeaders[0] + ) + deferred = self._agent.request(method, location, headers) + + def _chainResponse(newResponse): + newResponse.setPreviousResponse(response) + return newResponse + + deferred.addCallback(_chainResponse) + return deferred.addCallback( + self._handleResponse, method, uri, headers, redirectCount + 1) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 41f6c2a..655bf76 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -7,7 +7,6 @@ from ipaddr import IPv4Address, AddressValueError

from twisted.web.client import GzipDecoder from twisted.internet import reactor -from twisted.internet.protocol import Factory, Protocol from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.names import client, dns

@@ -19,31 +18,17 @@ from ooni.utils import log

from ooni.backend_client import WebConnectivityClient

-from ooni.utils.net import COMMON_SERVER_HEADERS, extract_title +from ooni.common.http_utils import extractTitle +from ooni.utils.net import COMMON_SERVER_HEADERS from ooni.templates import httpt, dnst from ooni.errors import failureToString

-REQUEST_HEADERS = { - 'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, ' - 'like Gecko) Chrome/47.0.2526.106 Safari/537.36'], - 'Accept-Language': ['en-US;q=0.8,en;q=0.5'], - 'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,' - '*/*;q=0.8'] -} +from ooni.common.tcp_utils import TCPConnectFactory +from ooni.common.http_utils import REQUEST_HEADERS

class InvalidControlResponse(Exception): pass

-class TCPConnectProtocol(Protocol): - def connectionMade(self): - self.transport.loseConnection() - -class TCPConnectFactory(Factory): - noisy = False - def buildProtocol(self, addr): - return TCPConnectProtocol() - - class UsageOptions(usage.Options): optParameters = [ ['url', 'u', None, 'Specify a single URL to test'], @@ -306,7 +291,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): return False

def compare_titles(self, experiment_http_response): - experiment_title = extract_title(experiment_http_response.body).strip() + experiment_title = extractTitle(experiment_http_response.body).strip() control_title = self.control['http_request']['title'].strip() first_exp_word = experiment_title.split(' ')[0] first_ctrl_word = control_title.split(' ')[0] diff --git a/ooni/nettests/manipulation/http_header_field_manipulation.py b/ooni/nettests/manipulation/http_header_field_manipulation.py index 6fbaf7d..fcd5e0e 100644 --- a/ooni/nettests/manipulation/http_header_field_manipulation.py +++ b/ooni/nettests/manipulation/http_header_field_manipulation.py @@ -11,7 +11,7 @@ from twisted.python import usage

from ooni.utils import log, net, randomStr from ooni.templates import httpt -from ooni.utils.trueheaders import TrueHeaders +from ooni.common.txextra import TrueHeaders

def random_capitalization(string): diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 6e0154c..f8ea941 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -10,15 +10,16 @@ from twisted.web.client import ContentDecoderAgent from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint

-from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent -from ooni.utils.trueheaders import FixedRedirectAgent +from ooni.utils.socks import TrueHeadersSOCKS5Agent

from ooni.nettest import NetTestCase -from ooni.utils import log, base64Dict +from ooni.utils import log from ooni.settings import config

from ooni.utils.net import StringProducer, userAgents -from ooni.utils.trueheaders import TrueHeaders +from ooni.common.txextra import TrueHeaders +from ooni.common.txextra import FixedRedirectAgent, TrueHeadersAgent +from ooni.common.http_utils import representBody from ooni.errors import handleAllFailures

META_CHARSET_REGEXP = re.compile('<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>!;]+)') @@ -48,37 +49,6 @@ def _representHeaders(headers): represented_headers[name] = unicode(value[0], errors='ignore') return represented_headers

-def _representBody(body): - if not body: - return body - # XXX perhaps add support for decoding gzip in the future. - body = body.replace('\0', '') - decoded = False - charsets = ['ascii', 'utf-8'] - - # If we are able to detect the charset of body from the meta tag - # try to decode using that one first - charset = META_CHARSET_REGEXP.search(body, re.IGNORECASE) - if charset: - try: - encoding = charset.group(1).lower() - codecs.lookup(encoding) - charsets.insert(0, encoding) - except (LookupError, IndexError): - # Skip invalid codecs and partial regexp match - pass - - for encoding in charsets: - try: - body = unicode(body, encoding) - decoded = True - break - except UnicodeDecodeError: - pass - if not decoded: - body = base64Dict(body) - return body - class HTTPTest(NetTestCase): """ A utility class for dealing with HTTP based testing. It provides methods to @@ -189,7 +159,7 @@ class HTTPTest(NetTestCase): } if response: if self.localOptions.get('withoutbody', 0) is 0: - response_body = _representBody(response_body) + response_body = representBody(response_body) else: response_body = '' # Attempt to redact the IP address of the probe from the responses diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 2bb9139..1c726d8 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -1,5 +1,6 @@ +from base64 import b64encode from ooni.nettest import NetTestCase -from ooni.utils import log, base64Dict +from ooni.utils import log from ooni.settings import config from ooni.utils.net import hasRawSocketPermission

@@ -8,7 +9,10 @@ from ooni.utils.txscapy import ScapySender, ScapyFactory

def _representPacket(packet): return { - "raw_packet": base64Dict(str(packet)), + "raw_packet": { + 'data': b64encode(str(packet)), + 'format': 'base64' + }, "summary": repr(packet) }

diff --git a/ooni/tests/test_trueheaders.py b/ooni/tests/test_trueheaders.py index f6c7812..191dafd 100644 --- a/ooni/tests/test_trueheaders.py +++ b/ooni/tests/test_trueheaders.py @@ -1,6 +1,6 @@ from twisted.trial import unittest

-from ooni.utils.trueheaders import TrueHeaders +from ooni.common.txextra import TrueHeaders

dummy_headers_dict = { 'Header1': ['Value1', 'Value2'], diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 22af062..f85cb38 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -1,12 +1,10 @@ import shutil import string import random -import glob import os from datetime import datetime

import gzip -from base64 import b64encode from zipfile import ZipFile

from ooni import errors @@ -155,9 +153,3 @@ def gunzip(filename, dst): def get_ooni_root(): script = os.path.join(__file__, '..') return os.path.dirname(os.path.realpath(script)) - -def base64Dict(data): - return { - 'format': 'base64', - 'data': b64encode(data) - } diff --git a/ooni/utils/net.py b/ooni/utils/net.py index 20f5a42..9918b60 100644 --- a/ooni/utils/net.py +++ b/ooni/utils/net.py @@ -81,14 +81,6 @@ COMMON_SERVER_HEADERS = ( # fetching URLs over some proxy. GOOGLE_HUMANS = ('http://www.google.com/humans.txt', 'Google is built by a large')

-TITLE_REGEXP = re.compile("<title>(.*?)</title>", re.IGNORECASE | re.DOTALL) - -def extract_title(body): - m = TITLE_REGEXP.search(body) - if m: - return m.group(1) - return '' - class StringProducer(object): implements(IBodyProducer)

diff --git a/ooni/utils/socks.py b/ooni/utils/socks.py new file mode 100644 index 0000000..783db55 --- /dev/null +++ b/ooni/utils/socks.py @@ -0,0 +1,25 @@ +from twisted.internet import reactor +from ooni.common.txextra import HTTPConnectionPool + +from twisted import version as twisted_version +from twisted.python.versions import Version +_twisted_15_0 = Version('twisted', 15, 0, 0) + +from txsocksx.http import SOCKS5Agent +from txsocksx.client import SOCKS5ClientFactory + +SOCKS5ClientFactory.noisy = False + +class TrueHeadersSOCKS5Agent(SOCKS5Agent): + def __init__(self, *args, **kw): + super(TrueHeadersSOCKS5Agent, self).__init__(*args, **kw) + pool = HTTPConnectionPool(reactor, False) + # + # With Twisted > 15.0 txsocksx wraps the twisted agent using a + # wrapper class, hence we must set the _pool attribute in the + # inner class rather than into its external wrapper. + # + if twisted_version >= _twisted_15_0: + self._wrappedAgent._pool = pool + else: + self._pool = pool diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py deleted file mode 100644 index 5f2b399..0000000 --- a/ooni/utils/trueheaders.py +++ /dev/null @@ -1,241 +0,0 @@ -# :authors: Giovanni Pellerano -# :licence: see LICENSE -# -# Here we make sure that the HTTP Headers sent and received are True. By this -# we mean that they are not normalized and that the ordering is maintained. - -import itertools -from copy import copy - -from twisted.python.failure import Failure - -from twisted.web import client, _newclient, http_headers, error - -from twisted.web._newclient import RequestNotSent, RequestGenerationFailed -from twisted.web._newclient import TransportProxyProducer, STATUS -from twisted.web._newclient import ResponseFailed - -from twisted.internet import reactor -from twisted.internet.defer import Deferred, fail, maybeDeferred, failure - -from txsocksx.http import SOCKS5Agent -from txsocksx.client import SOCKS5ClientFactory - -SOCKS5ClientFactory.noisy = False - -from ooni.utils import log - -import twisted -from twisted.python.versions import Version - - -class TrueHeaders(http_headers.Headers): - def __init__(self, rawHeaders=None): - self._rawHeaders = dict() - if rawHeaders is not None: - for name, values in rawHeaders.iteritems(): - if type(values) is list: - self.setRawHeaders(name, values[:]) - elif type(values) is dict: - self._rawHeaders[name.lower()] = values - elif type(values) is str: - self.setRawHeaders(name, values) - - def setRawHeaders(self, name, values): - if name.lower() not in self._rawHeaders: - self._rawHeaders[name.lower()] = dict() - self._rawHeaders[name.lower()]['name'] = name - self._rawHeaders[name.lower()]['values'] = values - - def getDiff(self, headers, ignore=[]): - """ - - Args: - - headers: a TrueHeaders object - - ignore: specify a list of header fields to ignore - - Returns: - - a set containing the header names that are not present in - header_dict or not present in self. - """ - diff = set() - field_names = [] - - headers_a = copy(self) - headers_b = copy(headers) - for name in ignore: - try: - del headers_a._rawHeaders[name.lower()] - except KeyError: - pass - try: - del headers_b._rawHeaders[name.lower()] - except KeyError: - pass - - for k, v in itertools.chain(headers_a.getAllRawHeaders(), - headers_b.getAllRawHeaders()): - field_names.append(k) - - for name in field_names: - if self.getRawHeaders(name) and headers.getRawHeaders(name): - pass - else: - diff.add(name) - return list(diff) - - def getAllRawHeaders(self): - for k, v in self._rawHeaders.iteritems(): - yield v['name'], v['values'] - - def getRawHeaders(self, name, default=None): - if name.lower() in self._rawHeaders: - return self._rawHeaders[name.lower()]['values'] - return default - - -class HTTPClientParser(_newclient.HTTPClientParser): - def logPrefix(self): - return 'HTTPClientParser' - - def connectionMade(self): - self.headers = TrueHeaders() - self.connHeaders = TrueHeaders() - self.state = STATUS - self._partialHeader = None - - def headerReceived(self, name, value): - if self.isConnectionControlHeader(name.lower()): - headers = self.connHeaders - else: - headers = self.headers - headers.addRawHeader(name, value) - - - def statusReceived(self, status): - parts = status.split(b' ', 2) - if len(parts) != 3: - # Here we add the extra missing part. - parts.append("XXX") - - try: - statusCode = int(parts[1]) - except ValueError: - raise _newclient.ParseError(u"non-integer status code", status) - - self.response = _newclient.Response._construct( - self.parseVersion(parts[0]), - statusCode, - parts[2], - self.headers, - self.transport, - self.request) - - -class HTTP11ClientProtocol(_newclient.HTTP11ClientProtocol): - def request(self, request): - if self._state != 'QUIESCENT': - return fail(RequestNotSent()) - - self._state = 'TRANSMITTING' - _requestDeferred = maybeDeferred(request.writeTo, self.transport) - self._finishedRequest = Deferred() - - self._currentRequest = request - - self._transportProxy = TransportProxyProducer(self.transport) - self._parser = HTTPClientParser(request, self._finishResponse) - self._parser.makeConnection(self._transportProxy) - self._responseDeferred = self._parser._responseDeferred - - def cbRequestWrotten(ignored): - if self._state == 'TRANSMITTING': - self._state = 'WAITING' - self._responseDeferred.chainDeferred(self._finishedRequest) - - def ebRequestWriting(err): - if self._state == 'TRANSMITTING': - self._state = 'GENERATION_FAILED' - self.transport.loseConnection() - self._finishedRequest.errback( - failure.Failure(RequestGenerationFailed([err]))) - else: - log.err(err, 'Error writing request, but not in valid state ' - 'to finalize request: %s' % self._state) - - _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting) - - return self._finishedRequest - - -class _HTTP11ClientFactory(client._HTTP11ClientFactory): - noisy = False - - def buildProtocol(self, addr): - return HTTP11ClientProtocol(self._quiescentCallback) - - -class HTTPConnectionPool(client.HTTPConnectionPool): - _factory = _HTTP11ClientFactory - - -class TrueHeadersAgent(client.Agent): - def __init__(self, *args, **kw): - super(TrueHeadersAgent, self).__init__(*args, **kw) - self._pool = HTTPConnectionPool(reactor, False) - - -_twisted_15_0 = Version('twisted', 15, 0, 0) - - -class TrueHeadersSOCKS5Agent(SOCKS5Agent): - def __init__(self, *args, **kw): - super(TrueHeadersSOCKS5Agent, self).__init__(*args, **kw) - pool = HTTPConnectionPool(reactor, False) - # - # With Twisted > 15.0 txsocksx wraps the twisted agent using a - # wrapper class, hence we must set the _pool attribute in the - # inner class rather than into its external wrapper. - # - if twisted.version >= _twisted_15_0: - self._wrappedAgent._pool = pool - else: - self._pool = pool - -class FixedRedirectAgent(client.BrowserLikeRedirectAgent): - """ - This is a redirect agent with this patch manually applied: - https://twistedmatrix.com/trac/ticket/8265 - """ - def _handleRedirect(self, response, method, uri, headers, redirectCount): - """ - Handle a redirect response, checking the number of redirects already - followed, and extracting the location header fields. - - This is pathed to fix a bug in infinite redirect loop. - """ - if redirectCount >= self._redirectLimit: - err = error.InfiniteRedirection( - response.code, - b'Infinite redirection detected', - location=uri) - raise ResponseFailed([Failure(err)], response) - locationHeaders = response.headers.getRawHeaders(b'location', []) - if not locationHeaders: - err = error.RedirectWithNoLocation( - response.code, b'No location header field', uri) - raise ResponseFailed([Failure(err)], response) - location = self._resolveLocation(response.request.absoluteURI, locationHeaders[0]) - deferred = self._agent.request(method, location, headers) - - def _chainResponse(newResponse): - newResponse.setPreviousResponse(response) - return newResponse - - deferred.addCallback(_chainResponse) - # This is the fix to properly handle redirects - return deferred.addCallback( - self._handleResponse, method, uri, headers, redirectCount + 1)