commit 4f0c2af37093070e4cec0593a0f57568a4db231c Author: Arturo Filastò <arturo@filasto.net> Date: Wed Nov 23 16:08:27 2016 +0000 Add support for ignoring redirects to IPs in private IP space --- ooni/common/ip_utils.py | 29 +++++++++++++++++++++++++++++ ooni/common/txextra.py | 18 +++++++++++++++++- ooni/nettests/blocking/web_connectivity.py | 30 +++++++++--------------------- ooni/templates/httpt.py | 26 +++++++++++++++++++++----- 4 files changed, 76 insertions(+), 27 deletions(-) diff --git a/ooni/common/ip_utils.py b/ooni/common/ip_utils.py new file mode 100644 index 0000000..8745b7a --- /dev/null +++ b/ooni/common/ip_utils.py @@ -0,0 +1,29 @@ +from ipaddr import IPv4Address, IPv6Address +from ipaddr import AddressValueError + + +def is_public_ipv4_address(address): + return not is_private_ipv4_address(address) + + +def is_private_ipv4_address(address): + try: + ip_address = IPv4Address(address) + return any( + [ip_address.is_private, ip_address.is_loopback] + ) + except AddressValueError: + return False + +def is_private_address(address): + try: + ip_address = IPv4Address(address) + except AddressValueError: + try: + ip_address = IPv6Address(address) + except AddressValueError: + return False + + return any( + [ip_address.is_private, ip_address.is_loopback] + ) diff --git a/ooni/common/txextra.py b/ooni/common/txextra.py index 7a84592..39b8996 100644 --- a/ooni/common/txextra.py +++ b/ooni/common/txextra.py @@ -19,6 +19,8 @@ from twisted.internet.defer import Deferred, fail, maybeDeferred, failure from twisted.python import log +from .ip_utils import is_private_address + class TrueHeaders(Headers): def __init__(self, rawHeaders=None): self._rawHeaders = dict() @@ -168,6 +170,10 @@ class FixedRedirectAgent(BrowserLikeRedirectAgent): This is a redirect agent with this patch manually applied: https://twistedmatrix.com/trac/ticket/8265 """ + def __init__(self, agent, redirectLimit=20, ignorePrivateRedirects=False): + self.ignorePrivateRedirects = ignorePrivateRedirects + BrowserLikeRedirectAgent.__init__(self, agent, redirectLimit) + def _handleRedirect(self, response, method, uri, headers, redirectCount): """ Handle a redirect response, checking the number of redirects already @@ -191,12 +197,22 @@ class FixedRedirectAgent(BrowserLikeRedirectAgent): response.request.absoluteURI, locationHeaders[0] ) + uri = client.URI.fromBytes(location) + if self.ignorePrivateRedirects and (is_private_address(uri.host) or + uri.host == "localhost"): + return response + deferred = self._agent.request(method, location, headers) def _chainResponse(newResponse): + if isinstance(newResponse, Failure): + # This is needed to write the response even in case of failure + newResponse.previousResponse = response + newResponse.requestLocation = location + return newResponse newResponse.setPreviousResponse(response) return newResponse - deferred.addCallback(_chainResponse) + deferred.addBoth(_chainResponse) return deferred.addCallback( self._handleResponse, method, uri, headers, redirectCount + 1) diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py index 1d1b742..629ef1c 100644 --- a/ooni/nettests/blocking/web_connectivity.py +++ b/ooni/nettests/blocking/web_connectivity.py @@ -3,28 +3,24 @@ import csv from urlparse import urlparse -from ipaddr import IPv4Address, AddressValueError - -from twisted.web.client import GzipDecoder +from twisted.internet import defer from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.names import client - -from twisted.internet import defer from twisted.python import usage +from twisted.web.client import GzipDecoder from ooni import geoip -from ooni.utils import log - from ooni.backend_client import WebConnectivityClient - +from ooni.common.http_utils import REQUEST_HEADERS from ooni.common.http_utils import extractTitle -from ooni.utils.net import COMMON_SERVER_HEADERS -from ooni.templates import httpt, dnst +from ooni.common.ip_utils import is_public_ipv4_address +from ooni.common.tcp_utils import TCPConnectFactory from ooni.errors import failureToString +from ooni.templates import httpt, dnst +from ooni.utils import log +from ooni.utils.net import COMMON_SERVER_HEADERS -from ooni.common.tcp_utils import TCPConnectFactory -from ooni.common.http_utils import REQUEST_HEADERS class InvalidControlResponse(Exception): pass @@ -42,15 +38,6 @@ class UsageOptions(usage.Options): ] -def is_public_ipv4_address(address): - try: - ip_address = IPv4Address(address) - return not any( - [ip_address.is_private, ip_address.is_loopback] - ) - except AddressValueError: - return None - class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): """ Web connectivity @@ -79,6 +66,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest): requiresRoot = False requiresTor = False followRedirects = True + ignorePrivateRedirects = True # These are the options to be shown on the GUI simpleOptions = [ diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index d1aa1dc..e57463b 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -66,6 +66,10 @@ class HTTPTest(NetTestCase): randomizeUA = False followRedirects = False + # When this is set to False we will follow redirects pointing to IPs in + # rfc1918 + ignorePrivateRedirects = False + # You can specify a list of tuples in the format of (CONTENT_TYPE, # DECODER) # For example to support Gzip decoding you should specify @@ -107,7 +111,10 @@ class HTTPTest(NetTestCase): if self.followRedirects: try: self.control_agent = FixedRedirectAgent(self.control_agent) - self.agent = FixedRedirectAgent(self.agent) + self.agent = FixedRedirectAgent( + self.agent, + ignorePrivateRedirects=self.ignorePrivateRedirects + ) self.report['agent'] = 'redirect' except: log.err("Warning! You are running an old version of twisted " @@ -129,7 +136,8 @@ class HTTPTest(NetTestCase): def processInputs(self): pass - def addToReport(self, request, response=None, response_body=None, failure_string=None): + def addToReport(self, request, response=None, response_body=None, + failure_string=None, previous_response=None): """ Adds to the report the specified request and response. @@ -176,9 +184,10 @@ class HTTPTest(NetTestCase): session['failure'] = failure_string self.report['requests'].append(session) - if response and response.previousResponse: - self.addToReport(request, response.previousResponse, + previous_response = response.previousResponse + if previous_response: + self.addToReport(request, previous_response, response_body=None, failure_string=None) @@ -367,7 +376,14 @@ class HTTPTest(NetTestCase): else: log.err("Error performing HTTP request: %s" % request['url']) failure_string = handleAllFailures(failure) - self.addToReport(request, failure_string=failure_string) + previous_response = None + if getattr(failure, "previousResponse", None): + previous_response = failure.previousResponse + if getattr(failure, "requestLocation", None): + request['url'] = failure.requestLocation + + self.addToReport(request, failure_string=failure_string, + previous_response=previous_response) return failure if use_tor: