Merge fc382cd453 into ad43bf0c56

docs/topics/download-handlers.rst

@@ -141,6 +141,7 @@ TLS implementation  ``cryptography``  ``cryptography``      Stdlib ``ssl``
 HTTP proxies        No                Yes                   Yes
 SOCKS proxies       No                No                    Yes
 Bad header handling Not applicable    Skip bad              Fail
+Header name case    Lowercase         Capitalized           As written
 =================== ================= ===================== ====================
 
 Bad header handling is what a handler does when a response has a bad header
@@ -148,6 +149,9 @@ line, e.g. one with no colon in it, which some servers send. Handlers that skip
 bad header lines, like web browsers do, still parse the header lines that follow
 them; other handlers also lose those, or cannot download such responses at all.
 
+Because HTTP/2 requires lowercase header names, handlers only keep the case of
+your header names over HTTP/1.1.
+
 You can find additional HTTP download handlers in the
 scrapy-download-handlers-incubator_ package. This package is made by the Scrapy
 developers and contains experimental handlers that may be included in some

docs/topics/request-response.rst

@@ -108,6 +108,13 @@ Request objects
         A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains
         the request headers.
 
+        .. versionchanged:: VERSION
+            Header names keep the case you write them in, instead of being
+            converted to ``Title-Case``.
+
+        Lookups are case-insensitive. Whether your case reaches the server
+        depends on the :ref:`download handler <download-handlers-ref>`.
+
     .. attribute:: Request.body
 
         The request body as bytes.

scrapy/core/_http2/stream.py

@@ -250,11 +250,10 @@ class Stream:
         content_length = str(len(self._request.body))
         headers.append(("Content-Length", content_length))
 
-        content_length_name = self._request.headers.normkey(b"Content-Length")
         for name, values in self._request.headers.items():
             for value_bytes in values:
                 value = str(value_bytes, "utf-8")
-                if name == content_length_name:
+                if name.lower() == b"content-length":
                     if value != content_length:
                         logger.warning(
                             "Ignoring bad Content-Length header %r of request %r, "

scrapy/core/downloader/handlers/_httpx.py

@@ -184,7 +184,7 @@ class HttpxDownloadHandler(_Base):
 
     @staticmethod
     def _extract_headers(response: httpx.Response) -> Headers:
-        return Headers(response.headers.multi_items())
+        return Headers(response.headers.raw)
 
     @staticmethod
     def _build_base_response_args(

scrapy/core/downloader/handlers/ftp.py

@@ -123,7 +123,7 @@ class FTPDownloadHandler(BaseDownloadHandler):
             protocol.close()
             assert client.transport
             client.transport.loseConnection()
-        headers = {"local filename": protocol.filename or b"", "size": protocol.size}
+        headers = {"Local Filename": protocol.filename or b"", "Size": protocol.size}
         body = protocol.filename or protocol.body.read()
         respcls = responsetypes.from_args(url=request.url, body=body)
         return respcls(url=request.url, status=200, body=body, headers=headers)

scrapy/http/headers.py

@@ -42,13 +42,26 @@ class Headers(CaselessDict):
     ) -> None:
         seq = seq.items() if isinstance(seq, Mapping) else seq
         iseq: dict[bytes, list[bytes]] = {}
+        # normkey() only sees keys already stored, so keys from seq that differ
+        # only in case are mapped to a single spelling here.
+        spellings: dict[bytes, bytes] = {}
         for k, v in seq:
-            iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
+            key = self.normkey(k)
+            key = spellings.setdefault(key.lower(), key)
+            iseq.setdefault(key, []).extend(self.normvalue(v))
         super().update(iseq)
 
     def normkey(self, key: str | bytes) -> bytes:
-        """Normalize key to bytes"""
-        return self._tobytes(key.title())
+        """Normalize key to bytes, matching the case of an existing key if any"""
+        key = self._tobytes(key)
+        if dict.__contains__(self, key):
+            return key
+        lower_key = key.lower()
+        existing_key: bytes
+        for existing_key in dict.keys(self):
+            if existing_key.lower() == lower_key:
+                return existing_key
+        return key
 
     def normvalue(self, value: _RawValue | Iterable[_RawValue]) -> list[bytes]:
         """Normalize values to bytes"""

tests/test_http2_client_protocol.py

@@ -780,7 +780,9 @@ class TestHttps2ClientProtocol:
 
         response_headers = json.loads(str(response.body, "utf-8"))
         assert isinstance(response_headers, dict)
+        # The server reports header names in its own case.
+        received = {k.lower(): v for k, v in response_headers.items()}
         for k, v in request.headers.items():
-            k_decoded, v_decoded = str(k, "utf-8"), str(v[0], "utf-8")
-            assert k_decoded in response_headers
-            assert v_decoded == response_headers[k_decoded]
+            k_decoded, v_decoded = str(k, "utf-8").lower(), str(v[0], "utf-8")
+            assert k_decoded in received
+            assert v_decoded == received[k_decoded]

tests/test_http_headers.py

@@ -101,6 +101,25 @@ class TestHeaders:
         assert h.getlist("Content-Type") == [b"text/html"]
         assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]
 
+    def test_key_case_kept(self):
+        h = Headers({"accept": "text/html", "access_token": "foo"})
+        assert sorted(h.keys()) == [b"accept", b"access_token"]
+
+    def test_key_case_of_first_spelling_wins(self):
+        h = Headers({"accept": "a", "Accept": "b"})
+        assert h.getlist("ACCEPT") == [b"a", b"b"]
+        assert list(h.keys()) == [b"accept"]
+
+        h["ACCEPT"] = "c"
+        h.appendlist("aCCept", "d")
+        h.update({"ACCEPT": "e"})
+        assert list(h.keys()) == [b"accept"]
+        assert h.getlist("accept") == [b"e"]
+
+        del h["ACCEPT"]
+        h["ACCEPT"] = "f"
+        assert list(h.keys()) == [b"ACCEPT"]
+
     def test_copy(self):
         h1 = Headers({"header1": ["value1", "value2"]})
         h2 = copy.copy(h1)

tests/test_utils_request.py

@@ -39,7 +39,7 @@ if TYPE_CHECKING:
                 headers={"Content-type": b"text/html"},
                 body=b"Some body",
             ),
-            b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",
+            b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-type: text/html\r\n\r\nSome body",
         ),
     ],
 )

tests/utils/bases/download_handlers_http.py

@@ -308,12 +308,12 @@ class TestHttpBase(ABC):
         async with self.get_dh() as download_handler:
             response = await download_handler.download_request(request)
         assert response.status == 200
-        received_headers = set(response.headers.keys())
+        received_headers = {key.lower() for key in response.headers}
         allowed_headers = {
-            b"Content-Length",
-            b"Content-Type",
-            b"Date",
-            b"Server",
+            b"content-length",
+            b"content-type",
+            b"date",
+            b"server",
         }
         extra_headers = received_headers - allowed_headers
         assert not extra_headers, response.headers