Add a leading slash to request URLs with a host and query but no path (#6574)

Co-authored-by: Cursor <cursoragent@cursor.com>

scrapy/http/request/__init__.py

@@ -19,6 +19,7 @@ from typing import (
     TypeVar,
     overload,
 )
+from urllib.parse import urlsplit, urlunsplit
 
 from w3lib.url import safe_url_string
 
@@ -263,6 +264,19 @@ class Request(object_ref):
             self._url = url
         else:
             self._url = safe_url_string(url, self.encoding)
+            # An http(s) URL with a host and a query but no path (e.g.
+            # "http://example.com?a=1") would produce a malformed request
+            # line ("GET ?a=1 HTTP/1.1") that many servers reject with a
+            # 400 response. Restore the "/" like web browsers do.
+            # See https://github.com/scrapy/scrapy/issues/6574
+            parts = urlsplit(self._url)
+            if (
+                parts.scheme in {"http", "https"}
+                and parts.netloc
+                and not parts.path
+                and parts.query
+            ):
+                self._url = urlunsplit(parts._replace(path="/"))
 
         if (
             "://" not in self._url

tests/test_http_request.py

@@ -165,6 +165,27 @@ class TestRequest:
         r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
         assert r4.url == "http://www.example.org/r%E9sum%E9.html"
 
+    def test_url_empty_path_with_query(self):
+        # A missing path must be replaced with "/" if there is a query,
+        # otherwise the request line sent to the server would be malformed,
+        # e.g. "GET ?a=1 HTTP/1.1" (see #6574)
+        r = self.request_class(url="http://www.scrapy.org?a=1")
+        assert r.url == "http://www.scrapy.org/?a=1"
+        r = self.request_class(url="https://www.scrapy.org:8080?a=1&b=2")
+        assert r.url == "https://www.scrapy.org:8080/?a=1&b=2"
+        r = self.request_class(url="http://www.scrapy.org?a=1#frag")
+        assert r.url == "http://www.scrapy.org/?a=1#frag"
+
+        # URLs without a query are left unchanged
+        r = self.request_class(url="http://www.scrapy.org")
+        assert r.url == "http://www.scrapy.org"
+        r = self.request_class(url="http://www.scrapy.org#frag")
+        assert r.url == "http://www.scrapy.org#frag"
+
+        # non-HTTP schemes are left unchanged
+        r = self.request_class(url="s3://bucket?a=1")
+        assert r.url == "s3://bucket?a=1"
+
     def test_url_verbatim(self):
         r = self.request_class(
             url="http://www.scrapy.org/price/£",