[tor-commits] [ooni-probe/master] Add support for ignoring redirects to IPs in private IP space

art at torproject.org art at torproject.org
Fri Jan 13 12:39:57 UTC 2017


commit 4f0c2af37093070e4cec0593a0f57568a4db231c
Author: Arturo Filastò <arturo at filasto.net>
Date:   Wed Nov 23 16:08:27 2016 +0000

    Add support for ignoring redirects to IPs in private IP space
---
 ooni/common/ip_utils.py                    | 29 +++++++++++++++++++++++++++++
 ooni/common/txextra.py                     | 18 +++++++++++++++++-
 ooni/nettests/blocking/web_connectivity.py | 30 +++++++++---------------------
 ooni/templates/httpt.py                    | 26 +++++++++++++++++++++-----
 4 files changed, 76 insertions(+), 27 deletions(-)

diff --git a/ooni/common/ip_utils.py b/ooni/common/ip_utils.py
new file mode 100644
index 0000000..8745b7a
--- /dev/null
+++ b/ooni/common/ip_utils.py
@@ -0,0 +1,29 @@
+from ipaddr import IPv4Address, IPv6Address
+from ipaddr import AddressValueError
+
+
+def is_public_ipv4_address(address):
+    return not is_private_ipv4_address(address)
+
+
+def is_private_ipv4_address(address):
+    try:
+        ip_address = IPv4Address(address)
+        return any(
+            [ip_address.is_private, ip_address.is_loopback]
+        )
+    except AddressValueError:
+        return False
+
+def is_private_address(address):
+    try:
+        ip_address = IPv4Address(address)
+    except AddressValueError:
+        try:
+            ip_address = IPv6Address(address)
+        except AddressValueError:
+            return False
+
+    return any(
+        [ip_address.is_private, ip_address.is_loopback]
+    )
diff --git a/ooni/common/txextra.py b/ooni/common/txextra.py
index 7a84592..39b8996 100644
--- a/ooni/common/txextra.py
+++ b/ooni/common/txextra.py
@@ -19,6 +19,8 @@ from twisted.internet.defer import Deferred, fail, maybeDeferred, failure
 
 from twisted.python import log
 
+from .ip_utils import is_private_address
+
 class TrueHeaders(Headers):
     def __init__(self, rawHeaders=None):
         self._rawHeaders = dict()
@@ -168,6 +170,10 @@ class FixedRedirectAgent(BrowserLikeRedirectAgent):
     This is a redirect agent with this patch manually applied:
     https://twistedmatrix.com/trac/ticket/8265
     """
+    def __init__(self, agent, redirectLimit=20, ignorePrivateRedirects=False):
+        self.ignorePrivateRedirects = ignorePrivateRedirects
+        BrowserLikeRedirectAgent.__init__(self, agent, redirectLimit)
+
     def _handleRedirect(self, response, method, uri, headers, redirectCount):
         """
         Handle a redirect response, checking the number of redirects already
@@ -191,12 +197,22 @@ class FixedRedirectAgent(BrowserLikeRedirectAgent):
             response.request.absoluteURI,
             locationHeaders[0]
         )
+        uri = client.URI.fromBytes(location)
+        if self.ignorePrivateRedirects and (is_private_address(uri.host) or
+                                            uri.host == "localhost"):
+            return response
+
         deferred = self._agent.request(method, location, headers)
 
         def _chainResponse(newResponse):
+            if isinstance(newResponse, Failure):
+                # This is needed to write the response even in case of failure
+                newResponse.previousResponse = response
+                newResponse.requestLocation = location
+                return newResponse
             newResponse.setPreviousResponse(response)
             return newResponse
 
-        deferred.addCallback(_chainResponse)
+        deferred.addBoth(_chainResponse)
         return deferred.addCallback(
             self._handleResponse, method, uri, headers, redirectCount + 1)
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index 1d1b742..629ef1c 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -3,28 +3,24 @@
 import csv
 from urlparse import urlparse
 
-from ipaddr import IPv4Address, AddressValueError
-
-from twisted.web.client import GzipDecoder
+from twisted.internet import defer
 from twisted.internet import reactor
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.names import client
-
-from twisted.internet import defer
 from twisted.python import usage
+from twisted.web.client import GzipDecoder
 
 from ooni import geoip
-from ooni.utils import log
-
 from ooni.backend_client import WebConnectivityClient
-
+from ooni.common.http_utils import REQUEST_HEADERS
 from ooni.common.http_utils import extractTitle
-from ooni.utils.net import COMMON_SERVER_HEADERS
-from ooni.templates import httpt, dnst
+from ooni.common.ip_utils import is_public_ipv4_address
+from ooni.common.tcp_utils import TCPConnectFactory
 from ooni.errors import failureToString
+from ooni.templates import httpt, dnst
+from ooni.utils import log
+from ooni.utils.net import COMMON_SERVER_HEADERS
 
-from ooni.common.tcp_utils import TCPConnectFactory
-from ooni.common.http_utils import REQUEST_HEADERS
 
 class InvalidControlResponse(Exception):
     pass
@@ -42,15 +38,6 @@ class UsageOptions(usage.Options):
     ]
 
 
-def is_public_ipv4_address(address):
-    try:
-        ip_address = IPv4Address(address)
-        return not any(
-            [ip_address.is_private, ip_address.is_loopback]
-        )
-    except AddressValueError:
-        return None
-
 class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     """
     Web connectivity
@@ -79,6 +66,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     requiresRoot = False
     requiresTor = False
     followRedirects = True
+    ignorePrivateRedirects = True
 
     # These are the options to be shown on the GUI
     simpleOptions = [
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index d1aa1dc..e57463b 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -66,6 +66,10 @@ class HTTPTest(NetTestCase):
     randomizeUA = False
     followRedirects = False
 
+    # When this is set to False we will follow redirects pointing to IPs in
+    # rfc1918
+    ignorePrivateRedirects = False
+
     # You can specify a list of tuples in the format of (CONTENT_TYPE,
     # DECODER)
     # For example to support Gzip decoding you should specify
@@ -107,7 +111,10 @@ class HTTPTest(NetTestCase):
         if self.followRedirects:
             try:
                 self.control_agent = FixedRedirectAgent(self.control_agent)
-                self.agent = FixedRedirectAgent(self.agent)
+                self.agent = FixedRedirectAgent(
+                    self.agent,
+                    ignorePrivateRedirects=self.ignorePrivateRedirects
+                )
                 self.report['agent'] = 'redirect'
             except:
                 log.err("Warning! You are running an old version of twisted "
@@ -129,7 +136,8 @@ class HTTPTest(NetTestCase):
     def processInputs(self):
         pass
 
-    def addToReport(self, request, response=None, response_body=None, failure_string=None):
+    def addToReport(self, request, response=None, response_body=None,
+                    failure_string=None, previous_response=None):
         """
         Adds to the report the specified request and response.
 
@@ -176,9 +184,10 @@ class HTTPTest(NetTestCase):
             session['failure'] = failure_string
 
         self.report['requests'].append(session)
-
         if response and response.previousResponse:
-            self.addToReport(request, response.previousResponse,
+            previous_response = response.previousResponse
+        if previous_response:
+            self.addToReport(request, previous_response,
                              response_body=None,
                              failure_string=None)
 
@@ -367,7 +376,14 @@ class HTTPTest(NetTestCase):
             else:
                 log.err("Error performing HTTP request: %s" % request['url'])
             failure_string = handleAllFailures(failure)
-            self.addToReport(request, failure_string=failure_string)
+            previous_response = None
+            if getattr(failure, "previousResponse", None):
+                previous_response  = failure.previousResponse
+            if getattr(failure, "requestLocation", None):
+                request['url'] = failure.requestLocation
+
+            self.addToReport(request, failure_string=failure_string,
+                             previous_response=previous_response)
             return failure
 
         if use_tor:





More information about the tor-commits mailing list