mirror of https://github.com/scrapy/scrapy.git
Merge pull request #3505 from kasun/json_request
[MRG+2] Request subclass for json requests #3504
This commit is contained in:
Konstantin Lopuhin
GitHub
commit
af2b666595
|
|
@ -512,6 +512,44 @@ method for this job. Here's an example spider which uses it::
|
|||
|
||||
# continue scraping with authenticated session...
|
||||
|
||||
JSONRequest
|
||||
-----------
|
||||
|
||||
The JSONRequest class extends the base :class:`Request` class with functionality for
|
||||
dealing with JSON requests.
|
||||
|
||||
.. class:: JSONRequest(url, [... data, dumps_kwargs])
|
||||
|
||||
The :class:`JSONRequest` class adds two new argument to the constructor. The
|
||||
remaining arguments are the same as for the :class:`Request` class and are
|
||||
not documented here.
|
||||
|
||||
Using the :class:`JSONRequest` will set the ``Content-Type`` header to ``application/json``
|
||||
and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01``
|
||||
|
||||
:param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
|
||||
if :attr:`Request.body` argument is provided this parameter will be ignored.
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
set to ``'POST'`` automatically.
|
||||
:type data: JSON serializable object
|
||||
|
||||
:param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
|
||||
data into JSON format.
|
||||
:type dumps_kwargs: dict
|
||||
|
||||
.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
|
||||
|
||||
JSONRequest usage example
|
||||
-------------------------
|
||||
|
||||
Sending a JSON POST request with a JSON payload::
|
||||
|
||||
data = {
|
||||
'name1': 'value1',
|
||||
'name2': 'value2',
|
||||
}
|
||||
yield JSONRequest(url='http://www.example.com/post/action', data=data)
|
||||
|
||||
|
||||
Response objects
|
||||
================
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from scrapy.http.headers import Headers
|
|||
from scrapy.http.request import Request
|
||||
from scrapy.http.request.form import FormRequest
|
||||
from scrapy.http.request.rpc import XmlRpcRequest
|
||||
from scrapy.http.request.json_request import JSONRequest
|
||||
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.http.response.html import HtmlResponse
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
"""
|
||||
This module implements the JSONRequest class which is a more convenient class
|
||||
(than Request) to generate JSON Requests.
|
||||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import warnings
|
||||
|
||||
from scrapy.http.request import Request
|
||||
|
||||
|
||||
class JSONRequest(Request):
|
||||
def __init__(self, *args, **kwargs):
|
||||
dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
|
||||
dumps_kwargs.setdefault('sort_keys', True)
|
||||
self._dumps_kwargs = dumps_kwargs
|
||||
|
||||
body_passed = kwargs.get('body', None) is not None
|
||||
data = kwargs.pop('data', None)
|
||||
data_passed = data is not None
|
||||
|
||||
if body_passed and data_passed:
|
||||
warnings.warn('Both body and data passed. data will be ignored')
|
||||
|
||||
elif not body_passed and data_passed:
|
||||
kwargs['body'] = self._dumps(data)
|
||||
|
||||
if 'method' not in kwargs:
|
||||
kwargs['method'] = 'POST'
|
||||
|
||||
super(JSONRequest, self).__init__(*args, **kwargs)
|
||||
self.headers.setdefault('Content-Type', 'application/json')
|
||||
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
|
||||
|
||||
def replace(self, *args, **kwargs):
|
||||
body_passed = kwargs.get('body', None) is not None
|
||||
data = kwargs.pop('data', None)
|
||||
data_passed = data is not None
|
||||
|
||||
if body_passed and data_passed:
|
||||
warnings.warn('Both body and data passed. data will be ignored')
|
||||
|
||||
elif not body_passed and data_passed:
|
||||
kwargs['body'] = self._dumps(data)
|
||||
|
||||
return super(JSONRequest, self).replace(*args, **kwargs)
|
||||
|
||||
def _dumps(self, data):
|
||||
"""Convert to JSON """
|
||||
return json.dumps(data, **self._dumps_kwargs)
|
||||
|
|
@ -2,6 +2,8 @@
|
|||
import cgi
|
||||
import unittest
|
||||
import re
|
||||
import json
|
||||
import warnings
|
||||
|
||||
import six
|
||||
from six.moves import xmlrpc_client as xmlrpclib
|
||||
|
|
@ -9,9 +11,11 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote
|
|||
if six.PY3:
|
||||
from urllib.parse import unquote_to_bytes
|
||||
|
||||
from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse
|
||||
from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse
|
||||
from scrapy.utils.python import to_bytes, to_native_str
|
||||
|
||||
from tests import mock
|
||||
|
||||
|
||||
class RequestTest(unittest.TestCase):
|
||||
|
||||
|
|
@ -1147,5 +1151,169 @@ class XmlRpcRequestTest(RequestTest):
|
|||
self._test_request(params=(u'pas£',), encoding='latin1')
|
||||
|
||||
|
||||
class JSONRequestTest(RequestTest):
|
||||
request_class = JSONRequest
|
||||
default_method = 'GET'
|
||||
default_headers = {b'Content-Type': [b'application/json'], b'Accept': [b'application/json, text/javascript, */*; q=0.01']}
|
||||
|
||||
def setUp(self):
|
||||
warnings.simplefilter("always")
|
||||
super(JSONRequestTest, self).setUp()
|
||||
|
||||
def test_data(self):
|
||||
r1 = self.request_class(url="http://www.example.com/")
|
||||
self.assertEqual(r1.body, b'')
|
||||
|
||||
body = b'body'
|
||||
r2 = self.request_class(url="http://www.example.com/", body=body)
|
||||
self.assertEqual(r2.body, body)
|
||||
|
||||
data = {
|
||||
'name': 'value',
|
||||
}
|
||||
r3 = self.request_class(url="http://www.example.com/", data=data)
|
||||
self.assertEqual(r3.body, to_bytes(json.dumps(data)))
|
||||
|
||||
# empty data
|
||||
r4 = self.request_class(url="http://www.example.com/", data=[])
|
||||
self.assertEqual(r4.body, to_bytes(json.dumps([])))
|
||||
|
||||
def test_data_method(self):
|
||||
# data is not passed
|
||||
r1 = self.request_class(url="http://www.example.com/")
|
||||
self.assertEqual(r1.method, 'GET')
|
||||
|
||||
body = b'body'
|
||||
r2 = self.request_class(url="http://www.example.com/", body=body)
|
||||
self.assertEqual(r2.method, 'GET')
|
||||
|
||||
data = {
|
||||
'name': 'value',
|
||||
}
|
||||
r3 = self.request_class(url="http://www.example.com/", data=data)
|
||||
self.assertEqual(r3.method, 'POST')
|
||||
|
||||
# method passed explicitly
|
||||
r4 = self.request_class(url="http://www.example.com/", data=data, method='GET')
|
||||
self.assertEqual(r4.method, 'GET')
|
||||
|
||||
r5 = self.request_class(url="http://www.example.com/", data=[])
|
||||
self.assertEqual(r5.method, 'POST')
|
||||
|
||||
def test_body_data(self):
|
||||
""" passing both body and data should result a warning """
|
||||
body = b'body'
|
||||
data = {
|
||||
'name': 'value',
|
||||
}
|
||||
with warnings.catch_warnings(record=True) as _warnings:
|
||||
r5 = self.request_class(url="http://www.example.com/", body=body, data=data)
|
||||
self.assertEqual(r5.body, body)
|
||||
self.assertEqual(r5.method, 'GET')
|
||||
self.assertEqual(len(_warnings), 1)
|
||||
self.assertIn('data will be ignored', str(_warnings[0].message))
|
||||
|
||||
def test_empty_body_data(self):
|
||||
""" passing any body value and data should result a warning """
|
||||
data = {
|
||||
'name': 'value',
|
||||
}
|
||||
with warnings.catch_warnings(record=True) as _warnings:
|
||||
r6 = self.request_class(url="http://www.example.com/", body=b'', data=data)
|
||||
self.assertEqual(r6.body, b'')
|
||||
self.assertEqual(r6.method, 'GET')
|
||||
self.assertEqual(len(_warnings), 1)
|
||||
self.assertIn('data will be ignored', str(_warnings[0].message))
|
||||
|
||||
def test_body_none_data(self):
|
||||
data = {
|
||||
'name': 'value',
|
||||
}
|
||||
with warnings.catch_warnings(record=True) as _warnings:
|
||||
r7 = self.request_class(url="http://www.example.com/", body=None, data=data)
|
||||
self.assertEqual(r7.body, to_bytes(json.dumps(data)))
|
||||
self.assertEqual(r7.method, 'POST')
|
||||
self.assertEqual(len(_warnings), 0)
|
||||
|
||||
def test_body_data_none(self):
|
||||
with warnings.catch_warnings(record=True) as _warnings:
|
||||
r8 = self.request_class(url="http://www.example.com/", body=None, data=None)
|
||||
self.assertEqual(r8.method, 'GET')
|
||||
self.assertEqual(len(_warnings), 0)
|
||||
|
||||
def test_dumps_sort_keys(self):
|
||||
""" Test that sort_keys=True is passed to json.dumps by default """
|
||||
data = {
|
||||
'name': 'value',
|
||||
}
|
||||
with mock.patch('json.dumps', return_value=b'') as mock_dumps:
|
||||
self.request_class(url="http://www.example.com/", data=data)
|
||||
kwargs = mock_dumps.call_args[1]
|
||||
self.assertEqual(kwargs['sort_keys'], True)
|
||||
|
||||
def test_dumps_kwargs(self):
|
||||
""" Test that dumps_kwargs are passed to json.dumps """
|
||||
data = {
|
||||
'name': 'value',
|
||||
}
|
||||
dumps_kwargs = {
|
||||
'ensure_ascii': True,
|
||||
'allow_nan': True,
|
||||
}
|
||||
with mock.patch('json.dumps', return_value=b'') as mock_dumps:
|
||||
self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs)
|
||||
kwargs = mock_dumps.call_args[1]
|
||||
self.assertEqual(kwargs['ensure_ascii'], True)
|
||||
self.assertEqual(kwargs['allow_nan'], True)
|
||||
|
||||
def test_replace_data(self):
|
||||
data1 = {
|
||||
'name1': 'value1',
|
||||
}
|
||||
data2 = {
|
||||
'name2': 'value2',
|
||||
}
|
||||
r1 = self.request_class(url="http://www.example.com/", data=data1)
|
||||
r2 = r1.replace(data=data2)
|
||||
self.assertEqual(r2.body, to_bytes(json.dumps(data2)))
|
||||
|
||||
def test_replace_sort_keys(self):
|
||||
""" Test that replace provides sort_keys=True to json.dumps """
|
||||
data1 = {
|
||||
'name1': 'value1',
|
||||
}
|
||||
data2 = {
|
||||
'name2': 'value2',
|
||||
}
|
||||
r1 = self.request_class(url="http://www.example.com/", data=data1)
|
||||
with mock.patch('json.dumps', return_value=b'') as mock_dumps:
|
||||
r1.replace(data=data2)
|
||||
kwargs = mock_dumps.call_args[1]
|
||||
self.assertEqual(kwargs['sort_keys'], True)
|
||||
|
||||
def test_replace_dumps_kwargs(self):
|
||||
""" Test that dumps_kwargs are provided to json.dumps when replace is called """
|
||||
data1 = {
|
||||
'name1': 'value1',
|
||||
}
|
||||
data2 = {
|
||||
'name2': 'value2',
|
||||
}
|
||||
dumps_kwargs = {
|
||||
'ensure_ascii': True,
|
||||
'allow_nan': True,
|
||||
}
|
||||
r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs)
|
||||
with mock.patch('json.dumps', return_value=b'') as mock_dumps:
|
||||
r1.replace(data=data2)
|
||||
kwargs = mock_dumps.call_args[1]
|
||||
self.assertEqual(kwargs['ensure_ascii'], True)
|
||||
self.assertEqual(kwargs['allow_nan'], True)
|
||||
|
||||
def tearDown(self):
|
||||
warnings.resetwarnings()
|
||||
super(JSONRequestTest, self).tearDown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue