diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 177e1e65c..f8582f444 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -250,7 +250,7 @@ objects. The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: - .. classmethod:: FormRequest.from_response(response, [formnumber=0, formdata, ...]) + .. classmethod:: FormRequest.from_response(response, [formnumber=0, formdata=None, clickdata=None, ...]) Returns a new :class:`FormRequest` object with its form field values pre-populated with those found in the HTML ``
`` element contained @@ -271,6 +271,11 @@ objects. overridden by the one passed in this parameter. :type formdata: dict + :param clickdata: The arguments in clickdata are passed directly to + ClientForm's click_request_data() method. See for more + info. + :type clickdata: dict + The other parameters of this class method are passed directly to the :class:`FormRequest` constructor. diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0f3d85c8a..15c383210 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -37,7 +37,7 @@ class FormRequest(Request): self.headers['Content-Type'] = 'application/x-www-form-urlencoded' @classmethod - def from_response(cls, response, formnumber=0, formdata=None, **kwargs): + def from_response(cls, response, formnumber=0, formdata=None, clickdata=None, **kwargs): encoding = getattr(response, 'encoding', 'utf-8') forms = ParseFile(StringIO(response.body), response.url, encoding=encoding, backwards_compat=False) @@ -51,11 +51,10 @@ class FormRequest(Request): # remove all existing fields with the same name before, so that # formdata fields properly can properly override existing ones, # which is the desired behaviour - form.controls = [c for c in form.controls if c.name not in formdata.keys()] + form.controls = [c for c in form.controls if c.name not in formdata] for k, v in formdata.iteritems(): for v2 in v if hasattr(v, '__iter__') else [v]: form.new_control('text', k, {'value': v2}) - - url, body, headers = form.click_request_data() - request = cls(url, method=form.method, body=body, headers=headers, **kwargs) - return request + + url, body, headers = form.click_request_data(**(clickdata or {})) + return cls(url, method=form.method, body=body, headers=headers, **kwargs) diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index 665915ae2..7e3c6262f 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -258,6 +258,40 @@ class FormRequestTest(RequestTest): self.assertEqual(fs['one'].value, '1') self.assertEqual(fs['two'].value, '2') + def test_from_response_submit_first_clickeable(self): + respbody = """ + + + + + + + """ + response = Response("http://www.example.com/this/list.html", body=respbody) + r1 = self.request_class.from_response(response, formdata={'two': '2'}) + urlargs = cgi.parse_qs(urlparse(r1.url).query) + self.assertEqual(urlargs['clickeable1'], ['clicked1']) + self.assertFalse('clickeable2' in urlargs, urlargs) + self.assertEqual(urlargs['one'], ['1']) + self.assertEqual(urlargs['two'], ['2']) + + def test_from_response_submit_not_first_clickeable(self): + respbody = """ +
+ + + + +
+ """ + response = Response("http://www.example.com/this/list.html", body=respbody) + r1 = self.request_class.from_response(response, formdata={'two': '2'}, clickdata={'name': 'clickeable2'}) + urlargs = cgi.parse_qs(urlparse(r1.url).query) + self.assertEqual(urlargs['clickeable2'], ['clicked2']) + self.assertFalse('clickeable1' in urlargs, urlargs) + self.assertEqual(urlargs['one'], ['1']) + self.assertEqual(urlargs['two'], ['2']) + def test_from_response_errors_noform(self): respbody = """""" response = Response("http://www.example.com/lala.html", body=respbody)