diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 61f62ecfc..0ab2f58bc 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -43,7 +43,8 @@ def _build_redirect_request( redirect_request.headers.pop(b"Proxy-Authorization", None) has_cookie_header = "Cookie" in redirect_request.headers has_authorization_header = "Authorization" in redirect_request.headers - if has_cookie_header or has_authorization_header: + has_request_body = bool(getattr(redirect_request, "body", None)) + if has_cookie_header or has_authorization_header or has_request_body: default_ports = {"http": 80, "https": 443} parsed_source_request = urlparse_cached(source_request) @@ -76,6 +77,11 @@ def _build_redirect_request( ): del redirect_request.headers["Authorization"] + if has_request_body and source_host != redirect_host: + redirect_request = redirect_request.replace(body=b"") + redirect_request.headers.pop("Content-Type", None) + redirect_request.headers.pop("Content-Length", None) + return redirect_request @@ -174,7 +180,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): if urlparse_cached(redirected).scheme not in {"http", "https"}: return response - if response.status in (301, 307, 308) or request.method == "HEAD": + if response.status in (307, 308) or request.method == "HEAD": return self._redirect(redirected, request, response.status) redirected = self._redirect_request_using_get(request, redirected_url) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 281bc898a..b8cebba84 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1004,7 +1004,7 @@ class TestRedirectMiddleware(Base.Test): return Response(request.url, status=status, headers=headers) def test_redirect_3xx_permanent(self): - def _test(method, status=301): + def _test(method, status: int): url = f"http://www.example.com/{status}" url2 = "http://www.example.com/redirected" req = Request(url, method=method) @@ -1019,10 +1019,6 @@ class TestRedirectMiddleware(Base.Test): del rsp.headers["Location"] assert self.mw.process_response(req, rsp) is rsp - _test("GET") - _test("POST") - _test("HEAD") - _test("GET", status=307) _test("POST", status=307) _test("HEAD", status=307) @@ -1031,6 +1027,45 @@ class TestRedirectMiddleware(Base.Test): _test("POST", status=308) _test("HEAD", status=308) + @pytest.mark.parametrize("status", [301, 302, 303]) + def test_post_method_converted_on_301_302_303(self, status): + source_url = f"http://www.example.com/{status}" + target_url = "http://www.example.com/redirected2" + request = Request( + source_url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + response = Response(source_url, headers={"Location": target_url}, status=status) + redirect_request = self.mw.process_response(request, response) + assert isinstance(redirect_request, Request) + assert redirect_request.url == target_url + assert redirect_request.method == "GET" + assert "Content-Type" not in redirect_request.headers + assert "Content-Length" not in redirect_request.headers + assert not redirect_request.body + + @pytest.mark.parametrize("status", [307, 308]) + def test_cross_origin_strip_body(self, status): + source_url = "https://example.com" + target_url = "https://attacker.example" + body = b"secret" + request = Request( + source_url, + method="POST", + body=body, + headers={"Content-Type": "application/json", "Content-Length": str(len(body))}, + ) + response1 = Response(source_url, headers={"Location": target_url}, status=status) + redirect_request = self.mw.process_response(request, response1) + assert isinstance(redirect_request, Request) + assert redirect_request.url == target_url + assert redirect_request.method == "POST" + assert not getattr(redirect_request, "body", None) + assert b"Content-Type" not in redirect_request.headers + assert b"Content-Length" not in redirect_request.headers + def test_redirect_302_head(self): url = "http://www.example.com/302" url2 = "http://www.example.com/redirected2"