Rewrite tests

tests/mockserver/http.py

@@ -34,6 +34,7 @@ from .http_resources import (
     ResponseHeadersResource,
     SetCookie,
     Status,
+    UriResource,
 )
 
 
@@ -84,6 +85,7 @@ class Root(resource.Resource):
         self.putChild(b"duplicate-header", DuplicateHeaderResource())
         self.putChild(b"response-headers", ResponseHeadersResource())
         self.putChild(b"set-cookie", SetCookie())
+        self.putChild(b"uri", UriResource())
 
     def getChild(self, path, request):
         return self

tests/test_downloader_handlers_http_base.py

@@ -803,6 +803,25 @@ class TestHttpBase(ABC):
                 "The 'bindaddress' request meta key is not supported by" in caplog.text
             )
 
+    @coroutine_test
+    async def test_verbatim_url(self, mockserver: MockServer) -> None:
+        # Square brackets are encoded by safe_url_string (w3lib).
+        path = "/uri/items?data[0]=a"
+        url = mockserver.url(path, is_secure=self.is_secure)
+
+        # Without verbatim_url, the brackets are percent-encoded before the
+        # request reaches the server.
+        request = Request(url)
+        async with self.get_dh() as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.body == b"/uri/items?data%5B0%5D=a"
+
+        # With verbatim_url=True the URL is sent to the server as-is.
+        request = Request(url, meta={"verbatim_url": True})
+        async with self.get_dh() as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.body == path.encode()
+
 
 class TestHttpsBase(TestHttpBase):
     is_secure = True

tests/test_http_request.py

@@ -162,6 +162,19 @@ class TestRequest:
         r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
         assert r4.url == "http://www.example.org/r%E9sum%E9.html"
 
+    def test_url_verbatim(self):
+        r = self.request_class(
+            url="http://www.scrapy.org/price/£",
+            meta={"verbatim_url": True},
+        )
+        assert r.url == "http://www.scrapy.org/price/£"
+
+        r = self.request_class(
+            url="http://www.scrapy.org/blank space",
+            meta={"verbatim_url": True},
+        )
+        assert r.url == "http://www.scrapy.org/blank space"
+
     def test_body(self):
         r1 = self.request_class(url="http://www.example.com/")
         assert r1.body == b""

tests/test_utils_request.py

@@ -59,10 +59,14 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:
 class TestFingerprint:
     function: staticmethod[[Request], bytes] = staticmethod(fingerprint)
     cache: (
-        WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]
-        | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]]
+        WeakKeyDictionary[
+            Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
+        ]
+        | WeakKeyDictionary[
+            Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str]
+        ]
     ) = _fingerprint_cache
-    default_cache_key = (None, False)
+    default_cache_key = (None, False, False)
     known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = (
         (
             Request("http://example.org"),
@@ -192,6 +196,32 @@ class TestFingerprint:
         assert self.function(r2) != self.function(r2, keep_fragments=True)
         assert self.function(r1) != self.function(r2, keep_fragments=True)
 
+    def test_verbatim_url(self):
+        # verbatim_url requests skip URL canonicalization
+        r1 = Request(
+            "http://www.example.com/query?a=1&b=2", meta={"verbatim_url": True}
+        )
+        r2 = Request(
+            "http://www.example.com/query?b=2&a=1", meta={"verbatim_url": True}
+        )
+        assert self.function(r1) != self.function(r2)
+
+        # without verbatim_url, canonicalization makes query-param order irrelevant
+        r3 = Request("http://www.example.com/query?a=1&b=2")
+        r4 = Request("http://www.example.com/query?b=2&a=1")
+        assert self.function(r3) == self.function(r4)
+
+        # with verbatim_url, the fragment is always kept in the fingerprint
+        r5 = Request(
+            "http://www.example.com/test#fragment", meta={"verbatim_url": True}
+        )
+        r6 = Request("http://www.example.com/test", meta={"verbatim_url": True})
+        assert self.function(r5) != self.function(r6)
+
+        # keep_fragments parameter is ignored for verbatim_url requests
+        assert self.function(r5) == self.function(r5, keep_fragments=True)
+        assert self.function(r5) == self.function(r5, keep_fragments=False)
+
     def test_method_and_body(self):
         r1 = Request("http://www.example.com")
         r2 = Request("http://www.example.com", method="POST")