RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement TextResponse.json() (#4574)

This commit is contained in:
Bulat Khabibullin 2020-06-01 07:57:23 +03:00 committed by GitHub
parent 6aab3badfa
commit 5cef927944
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 38 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/topics/request-response.rst
View File

@ -879,6 +879,11 @@ TextResponse objects
.. automethod:: TextResponse.follow_all
.. automethod:: TextResponse.json()
Returns a Python object from deserialized JSON document.
The result is cached after the first call.
HtmlResponse objects
--------------------

12
scrapy/http/response/text.py
View File

@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
import json
import warnings
from contextlib import suppress
from typing import Generator
@ -21,10 +22,13 @@ from scrapy.http.response import Response
from scrapy.utils.python import memoizemethod_noargs, to_unicode
from scrapy.utils.response import get_base_url
_NONE = object()
class TextResponse(Response):
_DEFAULT_ENCODING = 'ascii'
_cached_decoded_json = _NONE
def __init__(self, *args, **kwargs):
self._encoding = kwargs.pop('encoding', None)
@ -68,6 +72,14 @@ class TextResponse(Response):
ScrapyDeprecationWarning, stacklevel=2)
return self.text
def json(self):
"""
Deserialize a JSON document to a Python object.
"""
if self._cached_decoded_json is _NONE:
self._cached_decoded_json = json.loads(self.text)
return self._cached_decoded_json
@property
def text(self):
""" Body as unicode """

21
tests/test_http_response.py
View File

@ -1,4 +1,5 @@
import unittest
from unittest import mock
from warnings import catch_warnings
from w3lib.encoding import resolve_encoding
@ -685,6 +686,26 @@ class TextResponseTest(BaseResponseTest):
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
def test_json_response(self):
json_body = b"""{"ip": "109.187.217.200"}"""
json_response = self.response_class("http://www.example.com", body=json_body)
self.assertEqual(json_response.json(), {'ip': '109.187.217.200'})
text_body = b"""<html><body>text</body></html>"""
text_response = self.response_class("http://www.example.com", body=text_body)
with self.assertRaises(ValueError):
text_response.json()
def test_cache_json_response(self):
json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]
for json_body in json_valid_bodies:
json_response = self.response_class("http://www.example.com", body=json_body)
with mock.patch('json.loads') as mock_json:
for _ in range(2):
json_response.json()
mock_json.assert_called_once_with(json_body.decode())
class HtmlResponseTest(TextResponseTest):