RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

dumps_kwargs

This commit is contained in:
kasun Herath 2018-12-17 23:14:06 +05:30
parent 71ef321b68
commit 8f1507a4a5
3 changed files with 138 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/topics/request-response.rst
View File

@ -514,9 +514,9 @@ JSONRequest
The JSONRequest class extends the base :class:`Request` class with functionality for
dealing with JSON requests.
.. class:: JSONRequest(url, [data, ...])
.. class:: JSONRequest(url, [... data])
The :class:`JSONRequest` class adds a new argument to the constructor called data. The
The :class:`JSONRequest` class adds two new argument to the constructor. The
remaining arguments are the same as for the :class:`Request` class and are
not documented here.
@ -529,6 +529,12 @@ dealing with JSON requests.
set to ``'POST'`` automatically.
:type data: JSON serializable object
:param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data
into JSON format.
:type dumps_kwargs: dict
.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
JSONRequest usage example
-------------------------

21
scrapy/http/request/json_request.py
View File

@ -13,6 +13,7 @@ from scrapy.http.request import Request
class JSONRequest(Request):
def __init__(self, *args, **kwargs):
dumps_kwargs = kwargs.pop('dumps_kwargs', {})
body_passed = kwargs.get('body', None) is not None
data = kwargs.pop('data', None)
data_passed = data is not None
@ -21,7 +22,7 @@ class JSONRequest(Request):
warnings.warn('Both body and data passed. data will be ignored')
elif not body_passed and data_passed:
kwargs['body'] = json.dumps(data, sort_keys=True)
kwargs['body'] = self.dump(data, **dumps_kwargs)
if 'method' not in kwargs:
kwargs['method'] = 'POST'
@ -29,3 +30,21 @@ class JSONRequest(Request):
super(JSONRequest, self).__init__(*args, **kwargs)
self.headers.setdefault('Content-Type', 'application/json')
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
self._dumps_kwargs = dumps_kwargs
def replace(self, *args, **kwargs):
body_passed = kwargs.get('body', None) is not None
data = kwargs.pop('data', None)
data_passed = data is not None
if body_passed and data_passed:
warnings.warn('Both body and data passed. data will be ignored')
elif not body_passed and data_passed:
kwargs['body'] = self.dump(data, **self._dumps_kwargs)
return super(JSONRequest, self).replace(*args, **kwargs)
def dump(self, data, **kwargs):
"""Convert to JSON """
return json.dumps(data, sort_keys=True, **kwargs)

114
tests/test_http_request.py
View File

@ -14,6 +14,8 @@ if six.PY3:
from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse
from scrapy.utils.python import to_bytes, to_native_str
from tests import mock
class RequestTest(unittest.TestCase):
@ -1161,24 +1163,49 @@ class JSONRequestTest(RequestTest):
def test_data(self):
r1 = self.request_class(url="http://www.example.com/")
self.assertEqual(r1.body, b'')
self.assertEqual(r1.method, 'GET')
body = b'body'
r2 = self.request_class(url="http://www.example.com/", body=body)
self.assertEqual(r2.body, body)
self.assertEqual(r2.method, 'GET')
data = {
'name': 'value',
}
r3 = self.request_class(url="http://www.example.com/", data=data)
self.assertEqual(r3.body, to_bytes(json.dumps(data)))
self.assertEqual(r3.method, 'POST')
# empty data
r4 = self.request_class(url="http://www.example.com/", data=[])
self.assertEqual(r4.body, to_bytes(json.dumps([])))
self.assertEqual(r4.method, 'POST')
def test_data_method(self):
# data is not passed
r1 = self.request_class(url="http://www.example.com/")
self.assertEqual(r1.method, 'GET')
body = b'body'
r2 = self.request_class(url="http://www.example.com/", body=body)
self.assertEqual(r2.method, 'GET')
data = {
'name': 'value',
}
r3 = self.request_class(url="http://www.example.com/", data=data)
self.assertEqual(r3.method, 'POST')
# method passed explicitly
r4 = self.request_class(url="http://www.example.com/", data=data, method='GET')
self.assertEqual(r4.method, 'GET')
r5 = self.request_class(url="http://www.example.com/", data=[])
self.assertEqual(r5.method, 'POST')
def test_body_data(self):
""" passing both body and data should result a warning """
body = b'body'
data = {
'name': 'value',
}
with warnings.catch_warnings(record=True) as _warnings:
r5 = self.request_class(url="http://www.example.com/", body=body, data=data)
self.assertEqual(r5.body, body)
@ -1186,6 +1213,11 @@ class JSONRequestTest(RequestTest):
self.assertEqual(len(_warnings), 1)
self.assertIn('data will be ignored', str(_warnings[0].message))
def test_empty_body_data(self):
""" passing any body value and data should result a warning """
data = {
'name': 'value',
}
with warnings.catch_warnings(record=True) as _warnings:
r6 = self.request_class(url="http://www.example.com/", body=b'', data=data)
self.assertEqual(r6.body, b'')
@ -1193,17 +1225,91 @@ class JSONRequestTest(RequestTest):
self.assertEqual(len(_warnings), 1)
self.assertIn('data will be ignored', str(_warnings[0].message))
def test_body_none_data(self):
data = {
'name': 'value',
}
with warnings.catch_warnings(record=True) as _warnings:
r7 = self.request_class(url="http://www.example.com/", body=None, data=data)
self.assertEqual(r7.body, to_bytes(json.dumps(data)))
self.assertEqual(r7.method, 'POST')
self.assertEqual(len(_warnings), 0)
def test_body_data_none(self):
with warnings.catch_warnings(record=True) as _warnings:
r8 = self.request_class(url="http://www.example.com/", body=None, data=None)
self.assertEqual(r8.method, 'GET')
self.assertEqual(len(_warnings), 0)
def test_dumps_sort_keys(self):
""" Test that sort_keys=True is passed to json.dumps by default """
data = {
'name': 'value',
}
with mock.patch('json.dumps', return_value=b'') as mock_dumps:
self.request_class(url="http://www.example.com/", data=data)
kwargs = mock_dumps.call_args[1]
self.assertEqual(kwargs['sort_keys'], True)
def test_dumps_kwargs(self):
""" Test that dumps_kwargs are passed to json.dumps """
data = {
'name': 'value',
}
dumps_kwargs = {
'ensure_ascii': True,
'allow_nan': True,
}
with mock.patch('json.dumps', return_value=b'') as mock_dumps:
self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs)
kwargs = mock_dumps.call_args[1]
self.assertEqual(kwargs['ensure_ascii'], True)
self.assertEqual(kwargs['allow_nan'], True)
def test_replace_data(self):
data1 = {
'name1': 'value1',
}
data2 = {
'name2': 'value2',
}
r1 = self.request_class(url="http://www.example.com/", data=data1)
r2 = r1.replace(data=data2)
self.assertEqual(r2.body, to_bytes(json.dumps(data2)))
def test_replace_sort_keys(self):
""" Test that replace provides sort_keys=True to json.dumps """
data1 = {
'name1': 'value1',
}
data2 = {
'name2': 'value2',
}
r1 = self.request_class(url="http://www.example.com/", data=data1)
with mock.patch('json.dumps', return_value=b'') as mock_dumps:
r1.replace(data=data2)
kwargs = mock_dumps.call_args[1]
self.assertEqual(kwargs['sort_keys'], True)
def test_replace_dumps_kwargs(self):
""" Test that dumps_kwargs are provided json.dumps when replace is called """
data1 = {
'name1': 'value1',
}
data2 = {
'name2': 'value2',
}
dumps_kwargs = {
'ensure_ascii': True,
'allow_nan': True,
}
r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs)
with mock.patch('json.dumps', return_value=b'') as mock_dumps:
r1.replace(data=data2)
kwargs = mock_dumps.call_args[1]
self.assertEqual(kwargs['ensure_ascii'], True)
self.assertEqual(kwargs['allow_nan'], True)
def tearDown(self):
warnings.resetwarnings()
super(JSONRequestTest, self).tearDown()