Merge pull request #6016 from jxlil/fix/response.json

Fix: response.json() call makes unnecessary memory allocation

docs/news.rst

@@ -3,6 +3,23 @@
 Release notes
 =============
 
+.. _release-2.11.0:
+
+Scrapy 2.11.0 (to be released)
+------------------------------
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-   The :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method now
+    requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or
+    UTF-32).
+
+    If you need to deal with JSON documents in an invalid encoding, use
+    ``json.loads(response.text)`` instead.
+
+    (:issue:`5968`)
+
 .. _release-2.10.0:
 
 Scrapy 2.10.0 (2023-08-04)

scrapy/http/response/text.py

@@ -82,7 +82,7 @@ class TextResponse(Response):
         Deserialize a JSON document to a Python object.
         """
         if self._cached_decoded_json is _NONE:
-            self._cached_decoded_json = json.loads(self.text)
+            self._cached_decoded_json = json.loads(self.body)
         return self._cached_decoded_json
 
     @property

tests/test_http_response.py

@@ -844,7 +844,7 @@ class TextResponseTest(BaseResponseTest):
             with mock.patch("json.loads") as mock_json:
                 for _ in range(2):
                     json_response.json()
-                mock_json.assert_called_once_with(json_body.decode())
+                mock_json.assert_called_once_with(json_body)
 
 
 class HtmlResponseTest(TextResponseTest):