diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 73c2e7dd4..9ddb0d224 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -19,6 +19,7 @@ from typing import ( TypeVar, overload, ) +from urllib.parse import urlsplit, urlunsplit from w3lib.url import safe_url_string @@ -263,6 +264,19 @@ class Request(object_ref): self._url = url else: self._url = safe_url_string(url, self.encoding) + # An http(s) URL with a host and a query but no path (e.g. + # "http://example.com?a=1") would produce a malformed request + # line ("GET ?a=1 HTTP/1.1") that many servers reject with a + # 400 response. Restore the "/" like web browsers do. + # See https://github.com/scrapy/scrapy/issues/6574 + parts = urlsplit(self._url) + if ( + parts.scheme in {"http", "https"} + and parts.netloc + and not parts.path + and parts.query + ): + self._url = urlunsplit(parts._replace(path="/")) if ( "://" not in self._url diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 1941b826f..41f879bd2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -165,6 +165,27 @@ class TestRequest: r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") assert r4.url == "http://www.example.org/r%E9sum%E9.html" + def test_url_empty_path_with_query(self): + # A missing path must be replaced with "/" if there is a query, + # otherwise the request line sent to the server would be malformed, + # e.g. "GET ?a=1 HTTP/1.1" (see #6574) + r = self.request_class(url="http://www.scrapy.org?a=1") + assert r.url == "http://www.scrapy.org/?a=1" + r = self.request_class(url="https://www.scrapy.org:8080?a=1&b=2") + assert r.url == "https://www.scrapy.org:8080/?a=1&b=2" + r = self.request_class(url="http://www.scrapy.org?a=1#frag") + assert r.url == "http://www.scrapy.org/?a=1#frag" + + # URLs without a query are left unchanged + r = self.request_class(url="http://www.scrapy.org") + assert r.url == "http://www.scrapy.org" + r = self.request_class(url="http://www.scrapy.org#frag") + assert r.url == "http://www.scrapy.org#frag" + + # non-HTTP schemes are left unchanged + r = self.request_class(url="s3://bucket?a=1") + assert r.url == "s3://bucket?a=1" + def test_url_verbatim(self): r = self.request_class( url="http://www.scrapy.org/price/£",