diff --git a/docs/news.rst b/docs/news.rst index c26cef22c..5a3d5e783 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,15 +5,26 @@ Release notes .. _release-2.11.1: -Scrapy 2.11.1 (YYYY-MM-DD) +Scrapy 2.11.1 (unreleased) -------------------------- Highlights: +- Security bug fixes. + - Support for Twisted >= 23.8.0. - Documentation improvements. +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + Modified requirements ~~~~~~~~~~~~~~~~~~~~~ @@ -61,6 +72,7 @@ Quality assurance - Fixed a test issue on PyPy 7.3.14. (:issue:`6204`, :issue:`6205`) + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) @@ -2929,6 +2941,19 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (unreleased) +------------------------- + +**Security bug fixes:** + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + .. _release-1.8.3: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 65f1d2224..3176ed930 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -17,11 +17,17 @@ def _build_redirect_request(source_request, *, url, **kwargs): **kwargs, cookies=None, ) - if "Cookie" in redirect_request.headers: + has_cookie_header = "Cookie" in redirect_request.headers + has_authorization_header = "Authorization" in redirect_request.headers + if has_cookie_header or has_authorization_header: source_request_netloc = urlparse_cached(source_request).netloc redirect_request_netloc = urlparse_cached(redirect_request).netloc if source_request_netloc != redirect_request_netloc: - del redirect_request.headers["Cookie"] + if has_cookie_header: + del redirect_request.headers["Cookie"] + # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name + if has_authorization_header: + del redirect_request.headers["Authorization"] return redirect_request diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index dc15b672c..10b8ca9af 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -247,6 +247,37 @@ class RedirectMiddlewareTest(unittest.TestCase): perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" self.assertEqual(perc_encoded_utf8_url, req_result.url) + def test_cross_domain_header_dropping(self): + safe_headers = {"A": "B"} + original_request = Request( + "https://example.com", + headers={"Cookie": "a=b", "Authorization": "a", **safe_headers}, + ) + + internal_response = Response( + "https://example.com", + headers={"Location": "https://example.com/a"}, + status=301, + ) + internal_redirect_request = self.mw.process_response( + original_request, internal_response, self.spider + ) + self.assertIsInstance(internal_redirect_request, Request) + self.assertEqual(original_request.headers, internal_redirect_request.headers) + + external_response = Response( + "https://example.com", + headers={"Location": "https://example.org/a"}, + status=301, + ) + external_redirect_request = self.mw.process_response( + original_request, external_response, self.spider + ) + self.assertIsInstance(external_redirect_request, Request) + self.assertEqual( + safe_headers, external_redirect_request.headers.to_unicode_dict() + ) + class MetaRefreshMiddlewareTest(unittest.TestCase): def setUp(self):