diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 2d04b621c..17020a41a 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -141,6 +141,7 @@ TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` HTTP proxies No Yes Yes SOCKS proxies No No Yes Bad header handling Not applicable Skip bad Fail +Header name case Lowercase Capitalized As written =================== ================= ===================== ==================== Bad header handling is what a handler does when a response has a bad header @@ -148,6 +149,9 @@ line, e.g. one with no colon in it, which some servers send. Handlers that skip bad header lines, like web browsers do, still parse the header lines that follow them; other handlers also lose those, or cannot download such responses at all. +Because HTTP/2 requires lowercase header names, handlers only keep the case of +your header names over HTTP/1.1. + You can find additional HTTP download handlers in the scrapy-download-handlers-incubator_ package. This package is made by the Scrapy developers and contains experimental handlers that may be included in some diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1d97e39b6..d4a647a62 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -108,6 +108,13 @@ Request objects A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains the request headers. + .. versionchanged:: VERSION + Header names keep the case you write them in, instead of being + converted to ``Title-Case``. + + Lookups are case-insensitive. Whether your case reaches the server + depends on the :ref:`download handler `. + .. attribute:: Request.body The request body as bytes. diff --git a/scrapy/core/_http2/stream.py b/scrapy/core/_http2/stream.py index 4a07d198b..19fb854c3 100644 --- a/scrapy/core/_http2/stream.py +++ b/scrapy/core/_http2/stream.py @@ -250,11 +250,10 @@ class Stream: content_length = str(len(self._request.body)) headers.append(("Content-Length", content_length)) - content_length_name = self._request.headers.normkey(b"Content-Length") for name, values in self._request.headers.items(): for value_bytes in values: value = str(value_bytes, "utf-8") - if name == content_length_name: + if name.lower() == b"content-length": if value != content_length: logger.warning( "Ignoring bad Content-Length header %r of request %r, " diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 8bbffb233..553c626bc 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -184,7 +184,7 @@ class HttpxDownloadHandler(_Base): @staticmethod def _extract_headers(response: httpx.Response) -> Headers: - return Headers(response.headers.multi_items()) + return Headers(response.headers.raw) @staticmethod def _build_base_response_args( diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 29b3e3c0f..9b4b2c01b 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -123,7 +123,7 @@ class FTPDownloadHandler(BaseDownloadHandler): protocol.close() assert client.transport client.transport.loseConnection() - headers = {"local filename": protocol.filename or b"", "size": protocol.size} + headers = {"Local Filename": protocol.filename or b"", "Size": protocol.size} body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) return respcls(url=request.url, status=200, body=body, headers=headers) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index b55ef6191..770c07252 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -42,13 +42,26 @@ class Headers(CaselessDict): ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq iseq: dict[bytes, list[bytes]] = {} + # normkey() only sees keys already stored, so keys from seq that differ + # only in case are mapped to a single spelling here. + spellings: dict[bytes, bytes] = {} for k, v in seq: - iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) + key = self.normkey(k) + key = spellings.setdefault(key.lower(), key) + iseq.setdefault(key, []).extend(self.normvalue(v)) super().update(iseq) def normkey(self, key: str | bytes) -> bytes: - """Normalize key to bytes""" - return self._tobytes(key.title()) + """Normalize key to bytes, matching the case of an existing key if any""" + key = self._tobytes(key) + if dict.__contains__(self, key): + return key + lower_key = key.lower() + existing_key: bytes + for existing_key in dict.keys(self): + if existing_key.lower() == lower_key: + return existing_key + return key def normvalue(self, value: _RawValue | Iterable[_RawValue]) -> list[bytes]: """Normalize values to bytes""" diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index e96a5484e..021edd7ca 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -780,7 +780,9 @@ class TestHttps2ClientProtocol: response_headers = json.loads(str(response.body, "utf-8")) assert isinstance(response_headers, dict) + # The server reports header names in its own case. + received = {k.lower(): v for k, v in response_headers.items()} for k, v in request.headers.items(): - k_decoded, v_decoded = str(k, "utf-8"), str(v[0], "utf-8") - assert k_decoded in response_headers - assert v_decoded == response_headers[k_decoded] + k_decoded, v_decoded = str(k, "utf-8").lower(), str(v[0], "utf-8") + assert k_decoded in received + assert v_decoded == received[k_decoded] diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index e7ed17615..ffd965e11 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -101,6 +101,25 @@ class TestHeaders: assert h.getlist("Content-Type") == [b"text/html"] assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] + def test_key_case_kept(self): + h = Headers({"accept": "text/html", "access_token": "foo"}) + assert sorted(h.keys()) == [b"accept", b"access_token"] + + def test_key_case_of_first_spelling_wins(self): + h = Headers({"accept": "a", "Accept": "b"}) + assert h.getlist("ACCEPT") == [b"a", b"b"] + assert list(h.keys()) == [b"accept"] + + h["ACCEPT"] = "c" + h.appendlist("aCCept", "d") + h.update({"ACCEPT": "e"}) + assert list(h.keys()) == [b"accept"] + assert h.getlist("accept") == [b"e"] + + del h["ACCEPT"] + h["ACCEPT"] = "f" + assert list(h.keys()) == [b"ACCEPT"] + def test_copy(self): h1 = Headers({"header1": ["value1", "value2"]}) h2 = copy.copy(h1) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 935447bc4..ba3e632dc 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -39,7 +39,7 @@ if TYPE_CHECKING: headers={"Content-type": b"text/html"}, body=b"Some body", ), - b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body", + b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-type: text/html\r\n\r\nSome body", ), ], ) diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py index abf153817..16714b59d 100644 --- a/tests/utils/bases/download_handlers_http.py +++ b/tests/utils/bases/download_handlers_http.py @@ -308,12 +308,12 @@ class TestHttpBase(ABC): async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.status == 200 - received_headers = set(response.headers.keys()) + received_headers = {key.lower() for key in response.headers} allowed_headers = { - b"Content-Length", - b"Content-Type", - b"Date", - b"Server", + b"content-length", + b"content-type", + b"date", + b"server", } extra_headers = received_headers - allowed_headers assert not extra_headers, response.headers