Improve FormRequest.from_response method to pass click data arguments to ClientForm library

docs/topics/request-response.rst

@@ -250,7 +250,7 @@ objects.
     The :class:`FormRequest` objects support the following class method in
     addition to the standard :class:`Request` methods:
 
-    .. classmethod:: FormRequest.from_response(response, [formnumber=0, formdata, ...])
+    .. classmethod:: FormRequest.from_response(response, [formnumber=0, formdata=None, clickdata=None, ...])
 
        Returns a new :class:`FormRequest` object with its form field values
        pre-populated with those found in the HTML ``<form>`` element contained
@@ -271,6 +271,11 @@ objects.
           overridden by the one passed in this parameter.
        :type formdata: dict
 
+       :param clickdata: The arguments in clickdata are passed directly to
+          ClientForm's click_request_data() method. See <ClientForm> for more
+          info.
+       :type clickdata: dict
+
        The other parameters of this class method are passed directly to the
        :class:`FormRequest` constructor.
 

scrapy/http/request/form.py

@@ -37,7 +37,7 @@ class FormRequest(Request):
             self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
 
     @classmethod
-    def from_response(cls, response, formnumber=0, formdata=None, **kwargs):
+    def from_response(cls, response, formnumber=0, formdata=None, clickdata=None, **kwargs):
         encoding = getattr(response, 'encoding', 'utf-8')
         forms = ParseFile(StringIO(response.body), response.url,
                           encoding=encoding, backwards_compat=False)
@@ -51,11 +51,10 @@ class FormRequest(Request):
             # remove all existing fields with the same name before, so that
             # formdata fields properly can properly override existing ones,
             # which is the desired behaviour
-            form.controls = [c for c in form.controls if c.name not in formdata.keys()]
+            form.controls = [c for c in form.controls if c.name not in formdata]
             for k, v in formdata.iteritems():
                 for v2 in v if hasattr(v, '__iter__') else [v]:
                     form.new_control('text', k, {'value': v2})
-                    
-        url, body, headers = form.click_request_data()       
-        request = cls(url, method=form.method, body=body, headers=headers, **kwargs)
-        return request
+
+        url, body, headers = form.click_request_data(**(clickdata or {}))
+        return cls(url, method=form.method, body=body, headers=headers, **kwargs)

scrapy/tests/test_http_request.py

@@ -258,6 +258,40 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs['one'].value, '1')
         self.assertEqual(fs['two'].value, '2')
 
+    def test_from_response_submit_first_clickeable(self):
+        respbody = """
+<form action="get.php" method="GET">
+<input type="submit" name="clickeable1" value="clicked1">
+<input type="hidden" name="one" value="1">
+<input type="hidden" name="two" value="3">
+<input type="submit" name="clickeable2" value="clicked2">
+</form>
+        """
+        response = Response("http://www.example.com/this/list.html", body=respbody)
+        r1 = self.request_class.from_response(response, formdata={'two': '2'})
+        urlargs = cgi.parse_qs(urlparse(r1.url).query)
+        self.assertEqual(urlargs['clickeable1'], ['clicked1'])
+        self.assertFalse('clickeable2' in urlargs, urlargs)
+        self.assertEqual(urlargs['one'], ['1'])
+        self.assertEqual(urlargs['two'], ['2'])
+
+    def test_from_response_submit_not_first_clickeable(self):
+        respbody = """
+<form action="get.php" method="GET">
+<input type="submit" name="clickeable1" value="clicked1">
+<input type="hidden" name="one" value="1">
+<input type="hidden" name="two" value="3">
+<input type="submit" name="clickeable2" value="clicked2">
+</form>
+        """
+        response = Response("http://www.example.com/this/list.html", body=respbody)
+        r1 = self.request_class.from_response(response, formdata={'two': '2'}, clickdata={'name': 'clickeable2'})
+        urlargs = cgi.parse_qs(urlparse(r1.url).query)
+        self.assertEqual(urlargs['clickeable2'], ['clicked2'])
+        self.assertFalse('clickeable1' in urlargs, urlargs)
+        self.assertEqual(urlargs['one'], ['1'])
+        self.assertEqual(urlargs['two'], ['2'])
+
     def test_from_response_errors_noform(self):
         respbody = """<html></html>"""
         response = Response("http://www.example.com/lala.html", body=respbody)