diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 02b853fc0..4e6f00bb0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -514,9 +514,9 @@ JSONRequest The JSONRequest class extends the base :class:`Request` class with functionality for dealing with JSON requests. -.. class:: JSONRequest(url, [data, ...]) +.. class:: JSONRequest(url, [... data]) - The :class:`JSONRequest` class adds a new argument to the constructor called data. The + The :class:`JSONRequest` class adds two new argument to the constructor. The remaining arguments are the same as for the :class:`Request` class and are not documented here. @@ -529,6 +529,12 @@ dealing with JSON requests. set to ``'POST'`` automatically. :type data: JSON serializable object + :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data + into JSON format. + :type dumps_kwargs: dict + +.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps + JSONRequest usage example ------------------------- diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index afc4356a3..7499610b9 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -13,6 +13,7 @@ from scrapy.http.request import Request class JSONRequest(Request): def __init__(self, *args, **kwargs): + dumps_kwargs = kwargs.pop('dumps_kwargs', {}) body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None @@ -21,7 +22,7 @@ class JSONRequest(Request): warnings.warn('Both body and data passed. data will be ignored') elif not body_passed and data_passed: - kwargs['body'] = json.dumps(data, sort_keys=True) + kwargs['body'] = self.dump(data, **dumps_kwargs) if 'method' not in kwargs: kwargs['method'] = 'POST' @@ -29,3 +30,21 @@ class JSONRequest(Request): super(JSONRequest, self).__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') + self._dumps_kwargs = dumps_kwargs + + def replace(self, *args, **kwargs): + body_passed = kwargs.get('body', None) is not None + data = kwargs.pop('data', None) + data_passed = data is not None + + if body_passed and data_passed: + warnings.warn('Both body and data passed. data will be ignored') + + elif not body_passed and data_passed: + kwargs['body'] = self.dump(data, **self._dumps_kwargs) + + return super(JSONRequest, self).replace(*args, **kwargs) + + def dump(self, data, **kwargs): + """Convert to JSON """ + return json.dumps(data, sort_keys=True, **kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5eb655c12..6dcfa25da 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -14,6 +14,8 @@ if six.PY3: from scrapy.http import Request, FormRequest, XmlRpcRequest, JSONRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str +from tests import mock + class RequestTest(unittest.TestCase): @@ -1161,24 +1163,49 @@ class JSONRequestTest(RequestTest): def test_data(self): r1 = self.request_class(url="http://www.example.com/") self.assertEqual(r1.body, b'') - self.assertEqual(r1.method, 'GET') body = b'body' r2 = self.request_class(url="http://www.example.com/", body=body) self.assertEqual(r2.body, body) - self.assertEqual(r2.method, 'GET') data = { 'name': 'value', } r3 = self.request_class(url="http://www.example.com/", data=data) self.assertEqual(r3.body, to_bytes(json.dumps(data))) - self.assertEqual(r3.method, 'POST') + # empty data r4 = self.request_class(url="http://www.example.com/", data=[]) self.assertEqual(r4.body, to_bytes(json.dumps([]))) - self.assertEqual(r4.method, 'POST') + def test_data_method(self): + # data is not passed + r1 = self.request_class(url="http://www.example.com/") + self.assertEqual(r1.method, 'GET') + + body = b'body' + r2 = self.request_class(url="http://www.example.com/", body=body) + self.assertEqual(r2.method, 'GET') + + data = { + 'name': 'value', + } + r3 = self.request_class(url="http://www.example.com/", data=data) + self.assertEqual(r3.method, 'POST') + + # method passed explicitly + r4 = self.request_class(url="http://www.example.com/", data=data, method='GET') + self.assertEqual(r4.method, 'GET') + + r5 = self.request_class(url="http://www.example.com/", data=[]) + self.assertEqual(r5.method, 'POST') + + def test_body_data(self): + """ passing both body and data should result a warning """ + body = b'body' + data = { + 'name': 'value', + } with warnings.catch_warnings(record=True) as _warnings: r5 = self.request_class(url="http://www.example.com/", body=body, data=data) self.assertEqual(r5.body, body) @@ -1186,6 +1213,11 @@ class JSONRequestTest(RequestTest): self.assertEqual(len(_warnings), 1) self.assertIn('data will be ignored', str(_warnings[0].message)) + def test_empty_body_data(self): + """ passing any body value and data should result a warning """ + data = { + 'name': 'value', + } with warnings.catch_warnings(record=True) as _warnings: r6 = self.request_class(url="http://www.example.com/", body=b'', data=data) self.assertEqual(r6.body, b'') @@ -1193,17 +1225,91 @@ class JSONRequestTest(RequestTest): self.assertEqual(len(_warnings), 1) self.assertIn('data will be ignored', str(_warnings[0].message)) + def test_body_none_data(self): + data = { + 'name': 'value', + } with warnings.catch_warnings(record=True) as _warnings: r7 = self.request_class(url="http://www.example.com/", body=None, data=data) self.assertEqual(r7.body, to_bytes(json.dumps(data))) self.assertEqual(r7.method, 'POST') self.assertEqual(len(_warnings), 0) + def test_body_data_none(self): with warnings.catch_warnings(record=True) as _warnings: r8 = self.request_class(url="http://www.example.com/", body=None, data=None) self.assertEqual(r8.method, 'GET') self.assertEqual(len(_warnings), 0) + def test_dumps_sort_keys(self): + """ Test that sort_keys=True is passed to json.dumps by default """ + data = { + 'name': 'value', + } + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + self.request_class(url="http://www.example.com/", data=data) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['sort_keys'], True) + + def test_dumps_kwargs(self): + """ Test that dumps_kwargs are passed to json.dumps """ + data = { + 'name': 'value', + } + dumps_kwargs = { + 'ensure_ascii': True, + 'allow_nan': True, + } + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['ensure_ascii'], True) + self.assertEqual(kwargs['allow_nan'], True) + + def test_replace_data(self): + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + r2 = r1.replace(data=data2) + self.assertEqual(r2.body, to_bytes(json.dumps(data2))) + + def test_replace_sort_keys(self): + """ Test that replace provides sort_keys=True to json.dumps """ + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['sort_keys'], True) + + def test_replace_dumps_kwargs(self): + """ Test that dumps_kwargs are provided json.dumps when replace is called """ + data1 = { + 'name1': 'value1', + } + data2 = { + 'name2': 'value2', + } + dumps_kwargs = { + 'ensure_ascii': True, + 'allow_nan': True, + } + r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs) + with mock.patch('json.dumps', return_value=b'') as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + self.assertEqual(kwargs['ensure_ascii'], True) + self.assertEqual(kwargs['allow_nan'], True) + def tearDown(self): warnings.resetwarnings() super(JSONRequestTest, self).tearDown()