From 2e59189f8440bd95f305cc66c8ee1dbdc9030758 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 5 Aug 2026 18:19:56 +0200 Subject: [PATCH] Keep the case of header names --- docs/topics/download-handlers.rst | 4 ++++ docs/topics/request-response.rst | 7 +++++++ scrapy/core/downloader/handlers/_httpx.py | 2 +- scrapy/core/downloader/handlers/ftp.py | 2 +- scrapy/core/http2/stream.py | 3 +-- scrapy/http/headers.py | 19 ++++++++++++++++--- tests/test_http_headers.py | 19 +++++++++++++++++++ tests/test_utils_request.py | 2 +- tests/utils/bases/download_handlers_http.py | 10 +++++----- 9 files changed, 55 insertions(+), 13 deletions(-) diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index e0501c169..0011c7ed5 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -140,8 +140,12 @@ HTTP/2 Yes No Yes TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` HTTP proxies No Yes Yes SOCKS proxies No No Yes +Header name case Lowercase Capitalized As written ================== ================= ===================== ==================== +Because HTTP/2 requires lowercase header names, handlers only keep the case of +your header names over HTTP/1.1. + You can find additional HTTP download handlers in the scrapy-download-handlers-incubator_ package. This package is made by the Scrapy developers and contains experimental handlers that may be included in some diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 75158440b..09b5e4767 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -160,6 +160,13 @@ Request objects A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains the request headers. + .. versionchanged:: VERSION + Header names keep the case you write them in, instead of being + converted to ``Title-Case``. + + Lookups are case-insensitive. Whether your case reaches the server + depends on the :ref:`download handler `. + .. attribute:: Request.body The request body as bytes. diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 8bbffb233..553c626bc 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -184,7 +184,7 @@ class HttpxDownloadHandler(_Base): @staticmethod def _extract_headers(response: httpx.Response) -> Headers: - return Headers(response.headers.multi_items()) + return Headers(response.headers.raw) @staticmethod def _build_base_response_args( diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 29b3e3c0f..9b4b2c01b 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -123,7 +123,7 @@ class FTPDownloadHandler(BaseDownloadHandler): protocol.close() assert client.transport client.transport.loseConnection() - headers = {"local filename": protocol.filename or b"", "size": protocol.size} + headers = {"Local Filename": protocol.filename or b"", "Size": protocol.size} body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) return respcls(url=request.url, status=200, body=body, headers=headers) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c6226bbca..811629fa8 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -239,11 +239,10 @@ class Stream: content_length = str(len(self._request.body)) headers.append(("Content-Length", content_length)) - content_length_name = self._request.headers.normkey(b"Content-Length") for name, values in self._request.headers.items(): for value_bytes in values: value = str(value_bytes, "utf-8") - if name == content_length_name: + if name.lower() == b"content-length": if value != content_length: logger.warning( "Ignoring bad Content-Length header %r of request %r, " diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index b55ef6191..770c07252 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -42,13 +42,26 @@ class Headers(CaselessDict): ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq iseq: dict[bytes, list[bytes]] = {} + # normkey() only sees keys already stored, so keys from seq that differ + # only in case are mapped to a single spelling here. + spellings: dict[bytes, bytes] = {} for k, v in seq: - iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) + key = self.normkey(k) + key = spellings.setdefault(key.lower(), key) + iseq.setdefault(key, []).extend(self.normvalue(v)) super().update(iseq) def normkey(self, key: str | bytes) -> bytes: - """Normalize key to bytes""" - return self._tobytes(key.title()) + """Normalize key to bytes, matching the case of an existing key if any""" + key = self._tobytes(key) + if dict.__contains__(self, key): + return key + lower_key = key.lower() + existing_key: bytes + for existing_key in dict.keys(self): + if existing_key.lower() == lower_key: + return existing_key + return key def normvalue(self, value: _RawValue | Iterable[_RawValue]) -> list[bytes]: """Normalize values to bytes""" diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index e7ed17615..ffd965e11 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -101,6 +101,25 @@ class TestHeaders: assert h.getlist("Content-Type") == [b"text/html"] assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] + def test_key_case_kept(self): + h = Headers({"accept": "text/html", "access_token": "foo"}) + assert sorted(h.keys()) == [b"accept", b"access_token"] + + def test_key_case_of_first_spelling_wins(self): + h = Headers({"accept": "a", "Accept": "b"}) + assert h.getlist("ACCEPT") == [b"a", b"b"] + assert list(h.keys()) == [b"accept"] + + h["ACCEPT"] = "c" + h.appendlist("aCCept", "d") + h.update({"ACCEPT": "e"}) + assert list(h.keys()) == [b"accept"] + assert h.getlist("accept") == [b"e"] + + del h["ACCEPT"] + h["ACCEPT"] = "f" + assert list(h.keys()) == [b"ACCEPT"] + def test_copy(self): h1 = Headers({"header1": ["value1", "value2"]}) h2 = copy.copy(h1) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 935447bc4..ba3e632dc 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -39,7 +39,7 @@ if TYPE_CHECKING: headers={"Content-type": b"text/html"}, body=b"Some body", ), - b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body", + b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-type: text/html\r\n\r\nSome body", ), ], ) diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py index e44f9bcb8..dbd937625 100644 --- a/tests/utils/bases/download_handlers_http.py +++ b/tests/utils/bases/download_handlers_http.py @@ -297,12 +297,12 @@ class TestHttpBase(ABC): async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.status == 200 - received_headers = set(response.headers.keys()) + received_headers = {key.lower() for key in response.headers} allowed_headers = { - b"Content-Length", - b"Content-Type", - b"Date", - b"Server", + b"content-length", + b"content-type", + b"date", + b"server", } extra_headers = received_headers - allowed_headers assert not extra_headers, response.headers