diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ae25ff7e4..bbd715766 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -879,6 +879,11 @@ TextResponse objects .. automethod:: TextResponse.follow_all + .. automethod:: TextResponse.json() + + Returns a Python object from deserialized JSON document. + The result is cached after the first call. + HtmlResponse objects -------------------- diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0603b6653..40cf3f483 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ +import json import warnings from contextlib import suppress from typing import Generator @@ -21,10 +22,13 @@ from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url +_NONE = object() + class TextResponse(Response): _DEFAULT_ENCODING = 'ascii' + _cached_decoded_json = _NONE def __init__(self, *args, **kwargs): self._encoding = kwargs.pop('encoding', None) @@ -68,6 +72,14 @@ class TextResponse(Response): ScrapyDeprecationWarning, stacklevel=2) return self.text + def json(self): + """ + Deserialize a JSON document to a Python object. + """ + if self._cached_decoded_json is _NONE: + self._cached_decoded_json = json.loads(self.text) + return self._cached_decoded_json + @property def text(self): """ Body as unicode """ diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 039e863f4..e0ca3c0e6 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,4 +1,5 @@ import unittest +from unittest import mock from warnings import catch_warnings from w3lib.encoding import resolve_encoding @@ -685,6 +686,26 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + def test_json_response(self): + json_body = b"""{"ip": "109.187.217.200"}""" + json_response = self.response_class("http://www.example.com", body=json_body) + self.assertEqual(json_response.json(), {'ip': '109.187.217.200'}) + + text_body = b"""text""" + text_response = self.response_class("http://www.example.com", body=text_body) + with self.assertRaises(ValueError): + text_response.json() + + def test_cache_json_response(self): + json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] + for json_body in json_valid_bodies: + json_response = self.response_class("http://www.example.com", body=json_body) + + with mock.patch('json.loads') as mock_json: + for _ in range(2): + json_response.json() + mock_json.assert_called_once_with(json_body.decode()) + class HtmlResponseTest(TextResponseTest):