Support sending requests with "unsafe" URLs (#7473)

docs/topics/request-response.rst

@@ -117,6 +117,9 @@ Request objects
     :param encoding: the encoding of this request (defaults to ``'utf-8'``).
        This encoding will be used to percent-encode the URL and to convert the
        body to bytes (if given as a string).
+
+       To disable URL percent-encoding for a request, use the
+       :reqmeta:`verbatim_url` request meta key.
     :type encoding: str
 
     :param priority: sets :attr:`priority`, defaults to ``0``.
@@ -136,9 +139,13 @@ Request objects
 
     .. attribute:: Request.url
 
-        A string containing the URL of this request. Keep in mind that this
-        attribute contains the escaped URL, so it can differ from the URL passed in
-        the ``__init__()`` method.
+        A string containing the URL of this request.
+
+        Keep in mind that this attribute contains the escaped URL, so it can
+        differ from the URL passed in the ``__init__()`` method.
+
+        If :reqmeta:`verbatim_url` is set to ``True``, the URL is kept as
+        passed to ``__init__()``.
 
         This attribute is read-only. To change the URL of a Request use
         :meth:`replace`.
@@ -541,6 +548,11 @@ in your :meth:`fingerprint` method implementation:
 
 .. autofunction:: scrapy.utils.request.fingerprint
 
+By default, request fingerprinting canonicalizes the request URL. If
+:reqmeta:`verbatim_url` is set to ``True``, fingerprinting does not
+canonicalize the URL, and the ``keep_fragments`` parameter is ignored (it is
+effectively true).
+
 For example, to take the value of a request header named ``X-ID`` into
 account:
 
@@ -710,6 +722,7 @@ Those are:
 * :reqmeta:`redirect_reasons`
 * :reqmeta:`redirect_urls`
 * :reqmeta:`referrer_policy`
+* :reqmeta:`verbatim_url`
 
 .. reqmeta:: bindaddress
 
@@ -786,6 +799,21 @@ The meta key is used set retry times per request. When initialized, the
 :reqmeta:`max_retry_times` meta key takes higher precedence over the
 :setting:`RETRY_TIMES` setting.
 
+.. reqmeta:: verbatim_url
+
+verbatim_url
+------------
+
+Set this key to ``True`` to keep the request URL as passed to
+:class:`~scrapy.Request`, without URL percent-encoding.
+
+When this key is enabled, :func:`~scrapy.utils.request.fingerprint` does not
+canonicalize the request URL, so requests whose URLs differ only in
+characters that would otherwise be canonicalized get different fingerprints.
+
+In this mode, the ``keep_fragments`` parameter is ignored, and it is
+effectively true.
+
 
 .. _topics-stop-response-download:
 

scrapy/http/request/__init__.py

@@ -138,6 +138,7 @@ class Request(object_ref):
     ) -> None:
         self._encoding: str = encoding  # this one has to be set first
         self.method: str = str(method).upper()
+        self._meta: dict[str, Any] | None = dict(meta) if meta else None
         self._set_url(url)
         self._set_body(body)
         if not isinstance(priority, int):
@@ -232,7 +233,6 @@ class Request(object_ref):
         #: default. See :meth:`~scrapy.Spider.start`.
         self.dont_filter: bool = dont_filter
 
-        self._meta: dict[str, Any] | None = dict(meta) if meta else None
         self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None
         self._flags: list[str] | None = list(flags) if flags else None
 
@@ -252,11 +252,17 @@ class Request(object_ref):
     def url(self) -> str:
         return self._url
 
+    def _url_is_verbatim(self) -> bool:
+        return bool(self._meta and self._meta.get("verbatim_url"))
+
     def _set_url(self, url: str) -> None:
         if not isinstance(url, str):
             raise TypeError(f"Request url must be str, got {type(url).__name__}")
 
-        self._url = safe_url_string(url, self.encoding)
+        if self._url_is_verbatim():
+            self._url = url
+        else:
+            self._url = safe_url_string(url, self.encoding)
 
         if (
             "://" not in self._url

scrapy/utils/request.py

@@ -28,7 +28,7 @@ if TYPE_CHECKING:
 
 
 _fingerprint_cache: WeakKeyDictionary[
-    Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]
+    Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
 ] = WeakKeyDictionary()
 
 
@@ -71,8 +71,10 @@ def fingerprint(
         processed_include_headers = tuple(
             to_bytes(h.lower()) for h in sorted(include_headers)
         )
+    verbatim_url = bool(request.meta.get("verbatim_url"))
+    effective_keep_fragments = keep_fragments and not verbatim_url
     cache = _fingerprint_cache.setdefault(request, {})
-    cache_key = (processed_include_headers, keep_fragments)
+    cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url)
     if cache_key not in cache:
         # To decode bytes reliably (JSON does not support bytes), regardless of
         # character encoding, we use bytes.hex()
@@ -84,9 +86,13 @@ def fingerprint(
                         header_value.hex()
                         for header_value in request.headers.getlist(header)
                     ]
+        if verbatim_url:
+            url = request.url
+        else:
+            url = canonicalize_url(request.url, keep_fragments=keep_fragments)
         fingerprint_data = {
             "method": to_unicode(request.method),
-            "url": canonicalize_url(request.url, keep_fragments=keep_fragments),
+            "url": url,
             "body": (request.body or b"").hex(),
             "headers": headers,
         }
@@ -108,8 +114,9 @@ class RequestFingerprinter:
     (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
     <scrapy.Request.url>` and the values of :attr:`request.method
     <scrapy.Request.method>` and :attr:`request.body
-    <scrapy.Request.body>`. It then generates an `SHA1
-    <https://en.wikipedia.org/wiki/SHA-1>`_ hash.
+    <scrapy.Request.body>`, unless :reqmeta:`verbatim_url` is true for that
+    request. It then generates an `SHA1 <https://en.wikipedia.org/wiki/SHA-1>`_
+    hash.
     """
 
     @classmethod

tests/mockserver/http.py

@@ -34,6 +34,7 @@ from .http_resources import (
     ResponseHeadersResource,
     SetCookie,
     Status,
+    UriResource,
 )
 
 
@@ -84,6 +85,7 @@ class Root(resource.Resource):
         self.putChild(b"duplicate-header", DuplicateHeaderResource())
         self.putChild(b"response-headers", ResponseHeadersResource())
         self.putChild(b"set-cookie", SetCookie())
+        self.putChild(b"uri", UriResource())
 
     def getChild(self, path, request):
         return self

tests/test_downloader_handlers_http_base.py

@@ -803,6 +803,25 @@ class TestHttpBase(ABC):
                 "The 'bindaddress' request meta key is not supported by" in caplog.text
             )
 
+    @coroutine_test
+    async def test_verbatim_url(self, mockserver: MockServer) -> None:
+        # Square brackets are encoded by safe_url_string (w3lib).
+        path = "/uri/items?data[0]=a"
+        url = mockserver.url(path, is_secure=self.is_secure)
+
+        # Without verbatim_url, the brackets are percent-encoded before the
+        # request reaches the server.
+        request = Request(url)
+        async with self.get_dh() as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.body == b"/uri/items?data%5B0%5D=a"
+
+        # With verbatim_url=True the URL is sent to the server as-is.
+        request = Request(url, meta={"verbatim_url": True})
+        async with self.get_dh() as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.body == path.encode()
+
 
 class TestHttpsBase(TestHttpBase):
     is_secure = True

tests/test_http_request.py

@@ -162,6 +162,19 @@ class TestRequest:
         r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
         assert r4.url == "http://www.example.org/r%E9sum%E9.html"
 
+    def test_url_verbatim(self):
+        r = self.request_class(
+            url="http://www.scrapy.org/price/£",
+            meta={"verbatim_url": True},
+        )
+        assert r.url == "http://www.scrapy.org/price/£"
+
+        r = self.request_class(
+            url="http://www.scrapy.org/blank space",
+            meta={"verbatim_url": True},
+        )
+        assert r.url == "http://www.scrapy.org/blank space"
+
     def test_body(self):
         r1 = self.request_class(url="http://www.example.com/")
         assert r1.body == b""

tests/test_utils_request.py

@@ -59,10 +59,14 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:
 class TestFingerprint:
     function: staticmethod[[Request], bytes] = staticmethod(fingerprint)
     cache: (
-        WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]
-        | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]]
+        WeakKeyDictionary[
+            Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
+        ]
+        | WeakKeyDictionary[
+            Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str]
+        ]
     ) = _fingerprint_cache
-    default_cache_key = (None, False)
+    default_cache_key = (None, False, False)
     known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = (
         (
             Request("http://example.org"),
@@ -192,6 +196,32 @@ class TestFingerprint:
         assert self.function(r2) != self.function(r2, keep_fragments=True)
         assert self.function(r1) != self.function(r2, keep_fragments=True)
 
+    def test_verbatim_url(self):
+        # verbatim_url requests skip URL canonicalization
+        r1 = Request(
+            "http://www.example.com/query?a=1&b=2", meta={"verbatim_url": True}
+        )
+        r2 = Request(
+            "http://www.example.com/query?b=2&a=1", meta={"verbatim_url": True}
+        )
+        assert self.function(r1) != self.function(r2)
+
+        # without verbatim_url, canonicalization makes query-param order irrelevant
+        r3 = Request("http://www.example.com/query?a=1&b=2")
+        r4 = Request("http://www.example.com/query?b=2&a=1")
+        assert self.function(r3) == self.function(r4)
+
+        # with verbatim_url, the fragment is always kept in the fingerprint
+        r5 = Request(
+            "http://www.example.com/test#fragment", meta={"verbatim_url": True}
+        )
+        r6 = Request("http://www.example.com/test", meta={"verbatim_url": True})
+        assert self.function(r5) != self.function(r6)
+
+        # keep_fragments parameter is ignored for verbatim_url requests
+        assert self.function(r5) == self.function(r5, keep_fragments=True)
+        assert self.function(r5) == self.function(r5, keep_fragments=False)
+
     def test_method_and_body(self):
         r1 = Request("http://www.example.com")
         r2 = Request("http://www.example.com", method="POST")

tox.ini

@@ -236,7 +236,12 @@ deps =
     pyOpenSSL==24.3.0
     queuelib==1.4.2
     service_identity==23.1.0
-    w3lib==1.20.0
+    # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses
+    # an inline flag placement that Python 3.11 treats as an error: global
+    # flags not at the start of the expression. w3lib 1.18 stopped encoding []
+    # in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url
+    # rely on that encoding.
+    w3lib==2.1.0
     zope.interface==5.1.0
 commands =
     ; disabling coverage