Add a leading slash to request URLs with a host and query/fragment but no path

A Request URL like "http://host?query" kept an empty path, so the HTTP request
line started with "?query" instead of "/?query", which some servers reject with
a 400 (the request never reaches the application). Normalize such URLs to keep
the "/" before the query or fragment, the same way w3lib's safe_download_url
does. URLs that already have a path, and hosts with neither a query nor a
fragment, are unchanged.

scrapy/http/request/__init__.py

@@ -19,6 +19,7 @@ from typing import (
     TypeVar,
     overload,
 )
+from urllib.parse import urlsplit, urlunsplit
 
 from w3lib.url import safe_url_string
 
@@ -263,6 +264,18 @@ class Request(object_ref):
             self._url = url
         else:
             self._url = safe_url_string(url, self.encoding)
+            # An http(s) URL with a host and a query or fragment but no path
+            # keeps an empty path, so the request line starts with "?..." and
+            # some servers reject it with a 400. Put the "/" back, the same
+            # normalization w3lib's safe_download_url applies. See #6574.
+            parts = urlsplit(self._url)
+            if (
+                parts.scheme in ("http", "https")
+                and parts.netloc
+                and not parts.path
+                and (parts.query or parts.fragment)
+            ):
+                self._url = urlunsplit(parts._replace(path="/"))
 
         if (
             "://" not in self._url

tests/test_http_request.py

@@ -100,6 +100,21 @@ class TestRequest:
         r = self.request_class(url="http://www.scrapy.org/path")
         assert r.url == "http://www.scrapy.org/path"
 
+    def test_url_empty_path_with_query(self):
+        # #6574: an http(s) url with a host and a query or fragment but no path
+        # must keep a leading "/" so the request line is "/?..." and not "?...",
+        # which some servers reject with a 400.
+        r = self.request_class(url="http://www.scrapy.org?url=x")
+        assert r.url == "http://www.scrapy.org/?url=x"
+        r = self.request_class(url="http://www.scrapy.org#frag")
+        assert r.url == "http://www.scrapy.org/#frag"
+        # a host with no query or fragment is left unchanged
+        r = self.request_class(url="http://www.scrapy.org")
+        assert r.url == "http://www.scrapy.org"
+        # an existing path is untouched
+        r = self.request_class(url="http://www.scrapy.org/p?url=x")
+        assert r.url == "http://www.scrapy.org/p?url=x"
+
     def test_url_quoting(self):
         r = self.request_class(url="http://www.scrapy.org/blank%20space")
         assert r.url == "http://www.scrapy.org/blank%20space"