diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 4a9bd732e..f623a5aa3 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -85,7 +85,8 @@ def _get_form(response, formname, formid, formnumber, formxpath): el = el.getparent() if el is None: break - raise ValueError('No
element found with %s' % formxpath) + encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape') + raise ValueError('No element found with %s' % encoded) # If we get here, it means that either formname was None # or invalid @@ -107,8 +108,12 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): inputs = form.xpath('descendant::textarea' '|descendant::select' - '|descendant::input[@type!="submit" and @type!="image" and @type!="reset"' - 'and ((@type!="checkbox" and @type!="radio") or @checked)]') + '|descendant::input[not(@type) or @type[' + ' not(re:test(., "^(?:submit|image|reset)$", "i"))' + ' and (../@checked or' + ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', + namespaces={ + "re": "http://exslt.org/regular-expressions"}) values = [(k, u'' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata] @@ -151,9 +156,13 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - clickables = [el for el in form.xpath('descendant::input[@type="submit"]' - '|descendant::button[@type="submit"]' - '|descendant::button[not(@type)]')] + clickables = [ + el for el in form.xpath( + 'descendant::*[(self::input or self::button)' + ' and re:test(@type, "^submit$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) + ] if not clickables: return diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 60fd855dd..f82b2de8d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,5 +1,6 @@ import cgi import unittest +import re import six from six.moves import xmlrpc_client as xmlrpclib @@ -305,6 +306,19 @@ class FormRequestTest(RequestTest): request = FormRequest.from_response(response, url='/relative') self.assertEqual(request.url, 'http://example.com/relative') + def test_from_response_case_insensitive(self): + response = _buildresponse( + """ + + + +
""") + req = self.request_class.from_response(response) + fs = _qs(req) + self.assertEqual(fs['clickable1'], ['clicked1']) + self.assertFalse('i1' in fs, fs) # xpath in _get_inputs() + self.assertFalse('clickable2' in fs, fs) # xpath in _get_clickable() + def test_from_response_submit_first_clickable(self): response = _buildresponse( """
@@ -662,10 +676,11 @@ class FormRequestTest(RequestTest): +
''') req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {'i1': ['i1v1'], 'i2': ['']}) + self.assertEqual(fs, {'i1': ['i1v1'], 'i2': [''], 'i4': ['i4v1']}) def test_from_response_input_hidden(self): res = _buildresponse( @@ -733,6 +748,18 @@ class FormRequestTest(RequestTest): self.assertRaises(ValueError, self.request_class.from_response, response, formxpath="//form/input[@name='abc']") + def test_from_response_unicode_xpath(self): + response = _buildresponse(b'
') + r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']") + fs = _qs(r) + self.assertEqual(fs, {}) + + xpath = u"//form[@name='\u03b1']" + encoded = xpath if six.PY3 else xpath.encode('unicode_escape') + self.assertRaisesRegexp(ValueError, re.escape(encoded), + self.request_class.from_response, + response, formxpath=xpath) + def test_from_response_button_submit(self): response = _buildresponse( """