From b2d8b06be61df40fd652af2f3fa8238a82b8645d Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 5 Jun 2026 11:51:03 +0200 Subject: [PATCH] Support sending requests with "unsafe" URLs (#7473) --- docs/topics/request-response.rst | 34 +++++++++++++++++-- scrapy/http/request/__init__.py | 10 ++++-- scrapy/utils/request.py | 17 +++++++--- tests/mockserver/http.py | 2 ++ tests/test_downloader_handlers_http_base.py | 19 +++++++++++ tests/test_http_request.py | 13 ++++++++ tests/test_utils_request.py | 36 +++++++++++++++++++-- tox.ini | 7 +++- 8 files changed, 124 insertions(+), 14 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d1e4851d9..a4f031803 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -117,6 +117,9 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the body to bytes (if given as a string). + + To disable URL percent-encoding for a request, use the + :reqmeta:`verbatim_url` request meta key. :type encoding: str :param priority: sets :attr:`priority`, defaults to ``0``. @@ -136,9 +139,13 @@ Request objects .. attribute:: Request.url - A string containing the URL of this request. Keep in mind that this - attribute contains the escaped URL, so it can differ from the URL passed in - the ``__init__()`` method. + A string containing the URL of this request. + + Keep in mind that this attribute contains the escaped URL, so it can + differ from the URL passed in the ``__init__()`` method. + + If :reqmeta:`verbatim_url` is set to ``True``, the URL is kept as + passed to ``__init__()``. This attribute is read-only. To change the URL of a Request use :meth:`replace`. @@ -541,6 +548,11 @@ in your :meth:`fingerprint` method implementation: .. autofunction:: scrapy.utils.request.fingerprint +By default, request fingerprinting canonicalizes the request URL. If +:reqmeta:`verbatim_url` is set to ``True``, fingerprinting does not +canonicalize the URL, and the ``keep_fragments`` parameter is ignored (it is +effectively true). + For example, to take the value of a request header named ``X-ID`` into account: @@ -710,6 +722,7 @@ Those are: * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` +* :reqmeta:`verbatim_url` .. reqmeta:: bindaddress @@ -786,6 +799,21 @@ The meta key is used set retry times per request. When initialized, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. +.. reqmeta:: verbatim_url + +verbatim_url +------------ + +Set this key to ``True`` to keep the request URL as passed to +:class:`~scrapy.Request`, without URL percent-encoding. + +When this key is enabled, :func:`~scrapy.utils.request.fingerprint` does not +canonicalize the request URL, so requests whose URLs differ only in +characters that would otherwise be canonicalized get different fingerprints. + +In this mode, the ``keep_fragments`` parameter is ignored, and it is +effectively true. + .. _topics-stop-response-download: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 00042e093..7db648a45 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -138,6 +138,7 @@ class Request(object_ref): ) -> None: self._encoding: str = encoding # this one has to be set first self.method: str = str(method).upper() + self._meta: dict[str, Any] | None = dict(meta) if meta else None self._set_url(url) self._set_body(body) if not isinstance(priority, int): @@ -232,7 +233,6 @@ class Request(object_ref): #: default. See :meth:`~scrapy.Spider.start`. self.dont_filter: bool = dont_filter - self._meta: dict[str, Any] | None = dict(meta) if meta else None self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None self._flags: list[str] | None = list(flags) if flags else None @@ -252,11 +252,17 @@ class Request(object_ref): def url(self) -> str: return self._url + def _url_is_verbatim(self) -> bool: + return bool(self._meta and self._meta.get("verbatim_url")) + def _set_url(self, url: str) -> None: if not isinstance(url, str): raise TypeError(f"Request url must be str, got {type(url).__name__}") - self._url = safe_url_string(url, self.encoding) + if self._url_is_verbatim(): + self._url = url + else: + self._url = safe_url_string(url, self.encoding) if ( "://" not in self._url diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 27d669e71..ffb7fae49 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -28,7 +28,7 @@ if TYPE_CHECKING: _fingerprint_cache: WeakKeyDictionary[ - Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes] + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] ] = WeakKeyDictionary() @@ -71,8 +71,10 @@ def fingerprint( processed_include_headers = tuple( to_bytes(h.lower()) for h in sorted(include_headers) ) + verbatim_url = bool(request.meta.get("verbatim_url")) + effective_keep_fragments = keep_fragments and not verbatim_url cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (processed_include_headers, keep_fragments) + cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url) if cache_key not in cache: # To decode bytes reliably (JSON does not support bytes), regardless of # character encoding, we use bytes.hex() @@ -84,9 +86,13 @@ def fingerprint( header_value.hex() for header_value in request.headers.getlist(header) ] + if verbatim_url: + url = request.url + else: + url = canonicalize_url(request.url, keep_fragments=keep_fragments) fingerprint_data = { "method": to_unicode(request.method), - "url": canonicalize_url(request.url, keep_fragments=keep_fragments), + "url": url, "body": (request.body or b"").hex(), "headers": headers, } @@ -108,8 +114,9 @@ class RequestFingerprinter: (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url ` and the values of :attr:`request.method ` and :attr:`request.body - `. It then generates an `SHA1 - `_ hash. + `, unless :reqmeta:`verbatim_url` is true for that + request. It then generates an `SHA1 `_ + hash. """ @classmethod diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index 622324a10..7ad873c02 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -34,6 +34,7 @@ from .http_resources import ( ResponseHeadersResource, SetCookie, Status, + UriResource, ) @@ -84,6 +85,7 @@ class Root(resource.Resource): self.putChild(b"duplicate-header", DuplicateHeaderResource()) self.putChild(b"response-headers", ResponseHeadersResource()) self.putChild(b"set-cookie", SetCookie()) + self.putChild(b"uri", UriResource()) def getChild(self, path, request): return self diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 51da8b38f..3b60fa555 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -803,6 +803,25 @@ class TestHttpBase(ABC): "The 'bindaddress' request meta key is not supported by" in caplog.text ) + @coroutine_test + async def test_verbatim_url(self, mockserver: MockServer) -> None: + # Square brackets are encoded by safe_url_string (w3lib). + path = "/uri/items?data[0]=a" + url = mockserver.url(path, is_secure=self.is_secure) + + # Without verbatim_url, the brackets are percent-encoded before the + # request reaches the server. + request = Request(url) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"/uri/items?data%5B0%5D=a" + + # With verbatim_url=True the URL is sent to the server as-is. + request = Request(url, meta={"verbatim_url": True}) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == path.encode() + class TestHttpsBase(TestHttpBase): is_secure = True diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 4c77bb1ec..fed5dbab7 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -162,6 +162,19 @@ class TestRequest: r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") assert r4.url == "http://www.example.org/r%E9sum%E9.html" + def test_url_verbatim(self): + r = self.request_class( + url="http://www.scrapy.org/price/£", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/price/£" + + r = self.request_class( + url="http://www.scrapy.org/blank space", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/blank space" + def test_body(self): r1 = self.request_class(url="http://www.example.com/") assert r1.body == b"" diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 70800dcec..55c46059e 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -59,10 +59,14 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None: class TestFingerprint: function: staticmethod[[Request], bytes] = staticmethod(fingerprint) cache: ( - WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]] - | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]] + WeakKeyDictionary[ + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] + ] + | WeakKeyDictionary[ + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str] + ] ) = _fingerprint_cache - default_cache_key = (None, False) + default_cache_key = (None, False, False) known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = ( ( Request("http://example.org"), @@ -192,6 +196,32 @@ class TestFingerprint: assert self.function(r2) != self.function(r2, keep_fragments=True) assert self.function(r1) != self.function(r2, keep_fragments=True) + def test_verbatim_url(self): + # verbatim_url requests skip URL canonicalization + r1 = Request( + "http://www.example.com/query?a=1&b=2", meta={"verbatim_url": True} + ) + r2 = Request( + "http://www.example.com/query?b=2&a=1", meta={"verbatim_url": True} + ) + assert self.function(r1) != self.function(r2) + + # without verbatim_url, canonicalization makes query-param order irrelevant + r3 = Request("http://www.example.com/query?a=1&b=2") + r4 = Request("http://www.example.com/query?b=2&a=1") + assert self.function(r3) == self.function(r4) + + # with verbatim_url, the fragment is always kept in the fingerprint + r5 = Request( + "http://www.example.com/test#fragment", meta={"verbatim_url": True} + ) + r6 = Request("http://www.example.com/test", meta={"verbatim_url": True}) + assert self.function(r5) != self.function(r6) + + # keep_fragments parameter is ignored for verbatim_url requests + assert self.function(r5) == self.function(r5, keep_fragments=True) + assert self.function(r5) == self.function(r5, keep_fragments=False) + def test_method_and_body(self): r1 = Request("http://www.example.com") r2 = Request("http://www.example.com", method="POST") diff --git a/tox.ini b/tox.ini index 5fef37e5b..cc916caea 100644 --- a/tox.ini +++ b/tox.ini @@ -236,7 +236,12 @@ deps = pyOpenSSL==24.3.0 queuelib==1.4.2 service_identity==23.1.0 - w3lib==1.20.0 + # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses + # an inline flag placement that Python 3.11 treats as an error: global + # flags not at the start of the expression. w3lib 1.18 stopped encoding [] + # in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url + # rely on that encoding. + w3lib==2.1.0 zope.interface==5.1.0 commands = ; disabling coverage