diff --git a/docs/news.rst b/docs/news.rst index ea79fbd98..6c45fcf5a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -37,6 +37,12 @@ Scrapy 1.8.4 (unreleased) ``scrapy.downloadermiddlewares.decompression`` module is discouraged and will trigger a warning. +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + .. _release-1.8.3: Scrapy 1.8.3 (2022-07-25) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 300041617..05072feb3 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -15,11 +15,17 @@ def _build_redirect_request(source_request, url, **kwargs): kwargs['url'] = url kwargs['cookies'] = None redirect_request = source_request.replace(**kwargs) - if 'Cookie' in redirect_request.headers: + has_cookie_header = "Cookie" in redirect_request.headers + has_authorization_header = "Authorization" in redirect_request.headers + if has_cookie_header or has_authorization_header: source_request_netloc = urlparse_cached(source_request).netloc redirect_request_netloc = urlparse_cached(redirect_request).netloc if source_request_netloc != redirect_request_netloc: - del redirect_request.headers['Cookie'] + if has_cookie_header: + del redirect_request.headers["Cookie"] + # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name + if has_authorization_header: + del redirect_request.headers["Authorization"] return redirect_request diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 0e841489d..e30b32a32 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -186,6 +186,39 @@ class RedirectMiddlewareTest(unittest.TestCase): perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' self.assertEqual(perc_encoded_utf8_url, req_result.url) + def test_cross_domain_header_dropping(self): + safe_headers = {"a": "b"} + headers = {"Cookie": "a=b", "Authorization": "a"} + headers.update(safe_headers) + original_request = Request( + "https://example.com", + headers=headers, + ) + + internal_response = Response( + "https://example.com", + headers={"Location": "https://example.com/a"}, + status=301, + ) + internal_redirect_request = self.mw.process_response( + original_request, internal_response, self.spider + ) + self.assertIsInstance(internal_redirect_request, Request) + self.assertEqual(original_request.headers, internal_redirect_request.headers) + + external_response = Response( + "https://example.com", + headers={"Location": "https://example.org/a"}, + status=301, + ) + external_redirect_request = self.mw.process_response( + original_request, external_response, self.spider + ) + self.assertIsInstance(external_redirect_request, Request) + self.assertEqual( + safe_headers, external_redirect_request.headers.to_unicode_dict() + ) + class MetaRefreshMiddlewareTest(unittest.TestCase):