From 461682fc3dca72d9a34ddc22ad1896787c9dc518 Mon Sep 17 00:00:00 2001 From: Claudio Salazar Date: Sat, 25 May 2019 11:01:19 +0200 Subject: [PATCH 1/3] Whitelist form methods in FormRequest.from_response method --- scrapy/http/request/form.py | 7 ++++++- tests/test_http_request.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index c2413b431..2182b9b53 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -18,6 +18,7 @@ from scrapy.utils.response import get_base_url class FormRequest(Request): + valid_form_methods = ['GET', 'POST', 'DIALOG'] def __init__(self, *args, **kwargs): formdata = kwargs.pop('formdata', None) @@ -48,7 +49,11 @@ class FormRequest(Request): form = _get_form(response, formname, formid, formnumber, formxpath) formdata = _get_inputs(form, formdata, dont_click, clickdata, response) url = _get_form_url(form, kwargs.pop('url', None)) - method = kwargs.pop('method', form.method) + + method = kwargs.pop('method', form.method).upper() + if method not in cls.valid_form_methods: + raise ValueError('Invalid form method in chosen form') + return cls(url=url, method=method, formdata=formdata, **kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 49f148016..8fdafb286 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1100,6 +1100,19 @@ class FormRequestTest(RequestTest): self.assertRaises(ValueError, self.request_class.from_response, response, formcss="input[name='abc']") + def test_from_response_valid_form_methods(self): + body = """
+ +
""" + + for method in self.request_class.valid_form_methods: + response = _buildresponse(body % method) + r1 = self.request_class.from_response(response) + self.assertEqual(r1.method, method) + + response = _buildresponse(body % 'UNKNOWN') + self.assertRaises(ValueError, self.request_class.from_response, response) + def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) From 0c50879568dee2363df5cbe25e9bdd7adaed5da4 Mon Sep 17 00:00:00 2001 From: Claudio Salazar Date: Thu, 6 Jun 2019 22:10:59 +0200 Subject: [PATCH 2/3] Change behavior to use method GET when there are unknown methods in the form --- scrapy/http/request/form.py | 2 +- tests/test_http_request.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2182b9b53..8b29aae4b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -52,7 +52,7 @@ class FormRequest(Request): method = kwargs.pop('method', form.method).upper() if method not in cls.valid_form_methods: - raise ValueError('Invalid form method in chosen form') + method = 'GET' return cls(url=url, method=method, formdata=formdata, **kwargs) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 8fdafb286..258b48dce 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1107,11 +1107,12 @@ class FormRequestTest(RequestTest): for method in self.request_class.valid_form_methods: response = _buildresponse(body % method) - r1 = self.request_class.from_response(response) - self.assertEqual(r1.method, method) + r = self.request_class.from_response(response) + self.assertEqual(r.method, method) response = _buildresponse(body % 'UNKNOWN') - self.assertRaises(ValueError, self.request_class.from_response, response) + r = self.request_class.from_response(response) + self.assertEqual(r.method, 'GET') def _buildresponse(body, **kwargs): From 2e4dc20393c3a678a6576fb20817d4339f66235c Mon Sep 17 00:00:00 2001 From: Claudio Salazar Date: Wed, 26 Jun 2019 21:36:28 +0200 Subject: [PATCH 3/3] Add backward compability when method=None in FormRequest --- scrapy/http/request/form.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 8b29aae4b..3ce8fc48e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -18,7 +18,7 @@ from scrapy.utils.response import get_base_url class FormRequest(Request): - valid_form_methods = ['GET', 'POST', 'DIALOG'] + valid_form_methods = ['GET', 'POST'] def __init__(self, *args, **kwargs): formdata = kwargs.pop('formdata', None) @@ -50,9 +50,11 @@ class FormRequest(Request): formdata = _get_inputs(form, formdata, dont_click, clickdata, response) url = _get_form_url(form, kwargs.pop('url', None)) - method = kwargs.pop('method', form.method).upper() - if method not in cls.valid_form_methods: - method = 'GET' + method = kwargs.pop('method', form.method) + if method is not None: + method = method.upper() + if method not in cls.valid_form_methods: + method = 'GET' return cls(url=url, method=method, formdata=formdata, **kwargs)