From 26f723e4e63185b35b2f11285f50cfc3ac52b8fc Mon Sep 17 00:00:00 2001 From: Harrison Gregg Date: Sun, 30 Apr 2017 19:07:29 -0400 Subject: [PATCH 1/3] Allow formdata value to be None to drop field generated from response --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 905d8412f..d9d178a3e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -135,7 +135,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): if clickable and clickable[0] not in formdata and not clickable[0] is None: values.append(clickable) - values.extend(formdata.items()) + values.extend((k, v) for k, v in formdata.items() if v is not None) return values From 45a323024c7a7008df3e319a8a1437fae53826f9 Mon Sep 17 00:00:00 2001 From: Harrison Gregg Date: Sun, 30 Apr 2017 19:14:47 -0400 Subject: [PATCH 2/3] Add documentation for dropping fields in from_response request body --- docs/topics/request-response.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 67f8ec285..5410654ef 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -417,7 +417,9 @@ fields with form data from :class:`Response` objects. :param formdata: fields to override in the form data. If a field was already present in the response ``
`` element, its value is - overridden by the one passed in this parameter. + overridden by the one passed in this parameter. If a value passed in + this parameter is ``None``, the field will not be included in the + request, even if it was present in the response ```` element. :type formdata: dict :param clickdata: attributes to lookup the control clicked. If it's not From ffef828a8deb86520e3bd6a50d76b2a4ecf3ae71 Mon Sep 17 00:00:00 2001 From: Harrison Gregg Date: Sun, 30 Apr 2017 19:33:51 -0400 Subject: [PATCH 3/3] Add test for dropping fields in from_response request body --- tests/test_http_request.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 7eadb874f..bbce537f4 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -425,6 +425,17 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'one'], [b'1']) self.assertEqual(fs[b'two'], [b'2']) + def test_from_response_drop_params(self): + response = _buildresponse( + """ + + + """) + req = self.request_class.from_response(response, formdata={'two': None}) + fs = _qs(req) + self.assertEqual(fs[b'one'], [b'1']) + self.assertNotIn(b'two', fs) + def test_from_response_override_method(self): response = _buildresponse( '''