Merge pull request #1597 from Digenis/various_formreq_fixes

[MRG+1] Various FormRequest tests+fixes

scrapy/http/request/form.py

@@ -85,7 +85,8 @@ def _get_form(response, formname, formid, formnumber, formxpath):
                 el = el.getparent()
                 if el is None:
                     break
-        raise ValueError('No <form> element found with %s' % formxpath)
+        encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
+        raise ValueError('No <form> element found with %s' % encoded)
 
     # If we get here, it means that either formname was None
     # or invalid
@@ -107,8 +108,12 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
 
     inputs = form.xpath('descendant::textarea'
                         '|descendant::select'
-                        '|descendant::input[@type!="submit" and @type!="image" and @type!="reset"'
-                        'and ((@type!="checkbox" and @type!="radio") or @checked)]')
+                        '|descendant::input[not(@type) or @type['
+                        ' not(re:test(., "^(?:submit|image|reset)$", "i"))'
+                        ' and (../@checked or'
+                        '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
+                        namespaces={
+                            "re": "http://exslt.org/regular-expressions"})
     values = [(k, u'' if v is None else v)
               for k, v in (_value(e) for e in inputs)
               if k and k not in formdata]
@@ -151,9 +156,13 @@ def _get_clickable(clickdata, form):
     if the latter is given. If not, it returns the first
     clickable element found
     """
-    clickables = [el for el in form.xpath('descendant::input[@type="submit"]'
-                                          '|descendant::button[@type="submit"]'
-                                          '|descendant::button[not(@type)]')]
+    clickables = [
+        el for el in form.xpath(
+            'descendant::*[(self::input or self::button)'
+            ' and re:test(@type, "^submit$", "i")]'
+            '|descendant::button[not(@type)]',
+            namespaces={"re": "http://exslt.org/regular-expressions"})
+        ]
     if not clickables:
         return
 

tests/test_http_request.py

@@ -1,5 +1,6 @@
 import cgi
 import unittest
+import re
 
 import six
 from six.moves import xmlrpc_client as xmlrpclib
@@ -305,6 +306,19 @@ class FormRequestTest(RequestTest):
         request = FormRequest.from_response(response, url='/relative')
         self.assertEqual(request.url, 'http://example.com/relative')
 
+    def test_from_response_case_insensitive(self):
+        response = _buildresponse(
+            """<form action="get.php" method="GET">
+            <input type="SuBmIt" name="clickable1" value="clicked1">
+            <input type="iMaGe" name="i1" src="http://my.image.org/1.jpg">
+            <input type="submit" name="clickable2" value="clicked2">
+            </form>""")
+        req = self.request_class.from_response(response)
+        fs = _qs(req)
+        self.assertEqual(fs['clickable1'], ['clicked1'])
+        self.assertFalse('i1' in fs, fs)  # xpath in _get_inputs()
+        self.assertFalse('clickable2' in fs, fs)  # xpath in _get_clickable()
+
     def test_from_response_submit_first_clickable(self):
         response = _buildresponse(
             """<form action="get.php" method="GET">
@@ -662,10 +676,11 @@ class FormRequestTest(RequestTest):
             <input type="text" name="i2">
             <input type="text" value="i3v1">
             <input type="text">
+            <input name="i4" value="i4v1">
             </form>''')
         req = self.request_class.from_response(res)
         fs = _qs(req)
-        self.assertEqual(fs, {'i1': ['i1v1'], 'i2': ['']})
+        self.assertEqual(fs, {'i1': ['i1v1'], 'i2': [''], 'i4': ['i4v1']})
 
     def test_from_response_input_hidden(self):
         res = _buildresponse(
@@ -733,6 +748,18 @@ class FormRequestTest(RequestTest):
         self.assertRaises(ValueError, self.request_class.from_response,
                           response, formxpath="//form/input[@name='abc']")
 
+    def test_from_response_unicode_xpath(self):
+        response = _buildresponse(b'<form name="\xd1\x8a"></form>')
+        r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']")
+        fs = _qs(r)
+        self.assertEqual(fs, {})
+
+        xpath = u"//form[@name='\u03b1']"
+        encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
+        self.assertRaisesRegexp(ValueError, re.escape(encoded),
+                                self.request_class.from_response,
+                                response, formxpath=xpath)
+
     def test_from_response_button_submit(self):
         response = _buildresponse(
             """<form action="post.php" method="POST">