Fall back to the response encoding in TextResponse.json() (#7897)

docs/topics/request-response.rst

@@ -1432,9 +1432,6 @@ TextResponse objects
 
     .. automethod:: TextResponse.json()
 
-        Returns a Python object from deserialized JSON document.
-        The result is cached after the first call.
-
     .. method:: TextResponse.urljoin(url)
 
         Constructs an absolute url by combining the Response's base url with

scrapy/http/response/text.py

@@ -84,9 +84,21 @@ class TextResponse(Response):
         )
 
     def json(self) -> Any:
-        """Deserialize a JSON document to a Python object."""
+        """Deserialize a JSON document to a Python object.
+
+        .. versionchanged:: VERSION
+           Bodies that cannot be decoded as UTF-8, UTF-16 or UTF-32, as the
+           JSON specification requires, are now decoded using
+           :attr:`TextResponse.encoding` instead of raising
+           :exc:`UnicodeDecodeError`.
+
+        The result is cached after the first call.
+        """
         if self._cached_decoded_json is _NONE:
-            self._cached_decoded_json = json.loads(self.body)
+            try:
+                self._cached_decoded_json = json.loads(self.body)
+            except UnicodeDecodeError:
+                self._cached_decoded_json = json.loads(self.text)
         return self._cached_decoded_json
 
     @property

tests/test_http_response_text.py

@@ -481,6 +481,22 @@ class TestTextResponse(TestResponseBase):
         ):
             text_response.json()
 
+    def test_json_response_non_utf8(self):
+        response = self.response_class(
+            "http://www.example.com",
+            body='{"message": "café"}'.encode("cp1252"),
+            headers={"Content-Type": "application/json"},
+        )
+        assert response.json() == {"message": "café"}
+
+    def test_json_response_wrong_charset(self):
+        response = self.response_class(
+            "http://www.example.com",
+            body='{"message": "café"}'.encode(),
+            headers={"Content-Type": "application/json; charset=iso-8859-1"},
+        )
+        assert response.json() == {"message": "café"}
+
     def test_cache_json_response(self):
         json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]
         for json_body in json_valid_bodies: