From 18ed0c0f7cea0fc23e1f837b72059889191e735f Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 13:58:09 +0200 Subject: [PATCH] Fall back to the response encoding in TextResponse.json() (#7897) --- docs/topics/request-response.rst | 3 --- scrapy/http/response/text.py | 16 ++++++++++++++-- tests/test_http_response_text.py | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f810146e4..a177d1ad5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1432,9 +1432,6 @@ TextResponse objects .. automethod:: TextResponse.json() - Returns a Python object from deserialized JSON document. - The result is cached after the first call. - .. method:: TextResponse.urljoin(url) Constructs an absolute url by combining the Response's base url with diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index d01e23e47..64251780a 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -84,9 +84,21 @@ class TextResponse(Response): ) def json(self) -> Any: - """Deserialize a JSON document to a Python object.""" + """Deserialize a JSON document to a Python object. + + .. versionchanged:: VERSION + Bodies that cannot be decoded as UTF-8, UTF-16 or UTF-32, as the + JSON specification requires, are now decoded using + :attr:`TextResponse.encoding` instead of raising + :exc:`UnicodeDecodeError`. + + The result is cached after the first call. + """ if self._cached_decoded_json is _NONE: - self._cached_decoded_json = json.loads(self.body) + try: + self._cached_decoded_json = json.loads(self.body) + except UnicodeDecodeError: + self._cached_decoded_json = json.loads(self.text) return self._cached_decoded_json @property diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index efa63e049..f705dbcee 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -481,6 +481,22 @@ class TestTextResponse(TestResponseBase): ): text_response.json() + def test_json_response_non_utf8(self): + response = self.response_class( + "http://www.example.com", + body='{"message": "café"}'.encode("cp1252"), + headers={"Content-Type": "application/json"}, + ) + assert response.json() == {"message": "café"} + + def test_json_response_wrong_charset(self): + response = self.response_class( + "http://www.example.com", + body='{"message": "café"}'.encode(), + headers={"Content-Type": "application/json; charset=iso-8859-1"}, + ) + assert response.json() == {"message": "café"} + def test_cache_json_response(self): json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] for json_body in json_valid_bodies: