commit d2ace6fc566daec6c4b9549b5f89830929d2d908 Author: Arturo Filastò arturo@filasto.net Date: Tue Jan 31 15:03:35 2017 +0000

Fix bug in handling of request headers (#729)

* Write a failing unittest demonstrating the bug

* Fix bug that leads to Headers not being preserved across redirects

* Fix backward compatibility with twisted < 15.0.0 when handling redirects

* Skip the redirect test when the network is absent --- ooni/common/txextra.py | 12 +++++++++++- ooni/tests/test_common.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/ooni/common/txextra.py b/ooni/common/txextra.py index e3507bf6..8b32f69f 100644 --- a/ooni/common/txextra.py +++ b/ooni/common/txextra.py @@ -37,6 +37,12 @@ class TrueHeaders(Headers): self._rawHeaders[name.lower()]['name'] = name self._rawHeaders[name.lower()]['values'] = values

+ def copy(self): + rawHeaders = {} + for k, v in self.getAllRawHeaders(): + rawHeaders[k] = v + return self.__class__(rawHeaders) + def getAllRawHeaders(self): for _, v in self._rawHeaders.iteritems(): yield v['name'], v['values'] @@ -197,7 +203,11 @@ class FixedRedirectAgent(BrowserLikeRedirectAgent): response.request.absoluteURI, locationHeaders[0] ) - uri = client.URI.fromBytes(location) + if getattr(client, 'URI', None): + uri = client.URI.fromBytes(location) + else: + # Backward compatibility with twisted 14.0.2 + uri = client._URI.fromBytes(location) if self.ignorePrivateRedirects and is_private_address(uri.host, only_loopback=True): return response diff --git a/ooni/tests/test_common.py b/ooni/tests/test_common.py index eb8e2432..40d3859f 100644 --- a/ooni/tests/test_common.py +++ b/ooni/tests/test_common.py @@ -1,6 +1,13 @@ +import json from twisted.trial import unittest +from twisted.internet import defer, reactor +from twisted.web.client import readBody + +from . import is_internet_connected + from ooni.common.http_utils import META_CHARSET_REGEXP from ooni.common.ip_utils import is_public_ipv4_address, is_private_ipv4_address +from ooni.common.txextra import FixedRedirectAgent, TrueHeadersAgent, TrueHeaders

class TestHTTPUtils(unittest.TestCase): def test_charset_detection(self): @@ -31,3 +38,17 @@ class TestIPUtils(unittest.TestCase): self.assertFalse(is_private_ipv4_address('example.com')) self.assertTrue(is_private_ipv4_address('127.0.0.1')) self.assertTrue(is_private_ipv4_address('192.168.2.2')) + +class TestTxExtra(unittest.TestCase): + @defer.inlineCallbacks + def test_redirect_works(self): + if not is_internet_connected(): + raise unittest.SkipTest("Internet connection missing") + + agent = FixedRedirectAgent(TrueHeadersAgent(reactor)) + headers = TrueHeaders({"Spam": ["ham"]}) + url = "http://httpbin.org/absolute-redirect/3" + response = yield agent.request('GET', url, headers) + body = yield readBody(response) + j = json.loads(body) + self.assertEqual(j['headers']['Spam'], 'ham')