From 21daf7c66ebffd728cb8100eea5438ffda3c0996 Mon Sep 17 00:00:00 2001 From: Apoorva Verma Date: Mon, 29 Jun 2026 12:05:48 +0530 Subject: [PATCH] Add a leading slash to request URLs with a host and query/fragment but no path A Request URL like "http://host?query" kept an empty path, so the HTTP request line started with "?query" instead of "/?query", which some servers reject with a 400 (the request never reaches the application). Normalize such URLs to keep the "/" before the query or fragment, the same way w3lib's safe_download_url does. URLs that already have a path, and hosts with neither a query nor a fragment, are unchanged. --- scrapy/http/request/__init__.py | 13 +++++++++++++ tests/test_http_request.py | 15 +++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 7db648a45..c30b9e42b 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -19,6 +19,7 @@ from typing import ( TypeVar, overload, ) +from urllib.parse import urlsplit, urlunsplit from w3lib.url import safe_url_string @@ -263,6 +264,18 @@ class Request(object_ref): self._url = url else: self._url = safe_url_string(url, self.encoding) + # An http(s) URL with a host and a query or fragment but no path + # keeps an empty path, so the request line starts with "?..." and + # some servers reject it with a 400. Put the "/" back, the same + # normalization w3lib's safe_download_url applies. See #6574. + parts = urlsplit(self._url) + if ( + parts.scheme in ("http", "https") + and parts.netloc + and not parts.path + and (parts.query or parts.fragment) + ): + self._url = urlunsplit(parts._replace(path="/")) if ( "://" not in self._url diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 1941b826f..ee51efe15 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -100,6 +100,21 @@ class TestRequest: r = self.request_class(url="http://www.scrapy.org/path") assert r.url == "http://www.scrapy.org/path" + def test_url_empty_path_with_query(self): + # #6574: an http(s) url with a host and a query or fragment but no path + # must keep a leading "/" so the request line is "/?..." and not "?...", + # which some servers reject with a 400. + r = self.request_class(url="http://www.scrapy.org?url=x") + assert r.url == "http://www.scrapy.org/?url=x" + r = self.request_class(url="http://www.scrapy.org#frag") + assert r.url == "http://www.scrapy.org/#frag" + # a host with no query or fragment is left unchanged + r = self.request_class(url="http://www.scrapy.org") + assert r.url == "http://www.scrapy.org" + # an existing path is untouched + r = self.request_class(url="http://www.scrapy.org/p?url=x") + assert r.url == "http://www.scrapy.org/p?url=x" + def test_url_quoting(self): r = self.request_class(url="http://www.scrapy.org/blank%20space") assert r.url == "http://www.scrapy.org/blank%20space"