From 881bf19e6804c3b82543f4f23d6ed60e1a4bb651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 17:08:13 +0200 Subject: [PATCH 1/7] FormRequest: test case-insensitive type attribute --- tests/test_http_request.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 60fd855dd..593698dd2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -305,6 +305,19 @@ class FormRequestTest(RequestTest): request = FormRequest.from_response(response, url='/relative') self.assertEqual(request.url, 'http://example.com/relative') + def test_from_response_case_insensitive(self): + response = _buildresponse( + """
+ + + +
""") + req = self.request_class.from_response(response) + fs = _qs(req) + self.assertEqual(fs['clickable1'], ['clicked1']) + self.assertFalse('i1' in fs, fs) # xpath in _get_inputs() + self.assertFalse('clickable2' in fs, fs) # xpath in _get_clickable() + def test_from_response_submit_first_clickable(self): response = _buildresponse( """
From 650acad2b73c1abafe8ff95808b4d77b65ef67cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 17:57:46 +0200 Subject: [PATCH 2/7] FormRequest: fix case-insensitive type attributes --- scrapy/http/request/form.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 4a9bd732e..ea4bfd564 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -107,8 +107,13 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): inputs = form.xpath('descendant::textarea' '|descendant::select' - '|descendant::input[@type!="submit" and @type!="image" and @type!="reset"' - 'and ((@type!="checkbox" and @type!="radio") or @checked)]') + '|descendant::input[@type[' + ' translate(., "SUBMIT", "submit") != "submit"' + ' and translate(., "IMAGE", "image") !="image"' + ' and translate(., "RESET", "reset") != "reset"' + ' and (../@checked or (' + ' translate(., "CHECKBOX", "checkbox") != "checkbox"' + ' and translate(., "RADIO", "radio") != "radio"))]]') values = [(k, u'' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata] @@ -151,9 +156,11 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - clickables = [el for el in form.xpath('descendant::input[@type="submit"]' - '|descendant::button[@type="submit"]' - '|descendant::button[not(@type)]')] + clickables = [ + el for el in form.xpath( + 'descendant::*[(self::input or self::button)' + ' and translate(@type, "SUBMIT", "submit") = "submit"]' + '|descendant::button[not(@type)]')] if not clickables: return From 7a438a51b783f099894212abdefc95ad591fce3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:03:54 +0200 Subject: [PATCH 3/7] FormRequest: test default type (is text) --- tests/test_http_request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 593698dd2..7964a6591 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -675,10 +675,11 @@ class FormRequestTest(RequestTest): + ''') req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {'i1': ['i1v1'], 'i2': ['']}) + self.assertEqual(fs, {'i1': ['i1v1'], 'i2': [''], 'i4': ['i4v1']}) def test_from_response_input_hidden(self): res = _buildresponse( From 2d25eab0df4c5c8bb1894cbc45cc15059d726c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:05:07 +0200 Subject: [PATCH 4/7] FormRequest: 's default type must be text --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ea4bfd564..8ff394602 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -107,7 +107,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): inputs = form.xpath('descendant::textarea' '|descendant::select' - '|descendant::input[@type[' + '|descendant::input[not(@type) or @type[' ' translate(., "SUBMIT", "submit") != "submit"' ' and translate(., "IMAGE", "image") !="image"' ' and translate(., "RESET", "reset") != "reset"' From 395ef805eb74c1c575d32ab0c52d406f55c7d276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:54:02 +0200 Subject: [PATCH 5/7] FormRequest: test unicode xpath expr & exception --- tests/test_http_request.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 7964a6591..f82b2de8d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,5 +1,6 @@ import cgi import unittest +import re import six from six.moves import xmlrpc_client as xmlrpclib @@ -747,6 +748,18 @@ class FormRequestTest(RequestTest): self.assertRaises(ValueError, self.request_class.from_response, response, formxpath="//form/input[@name='abc']") + def test_from_response_unicode_xpath(self): + response = _buildresponse(b'
') + r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']") + fs = _qs(r) + self.assertEqual(fs, {}) + + xpath = u"//form[@name='\u03b1']" + encoded = xpath if six.PY3 else xpath.encode('unicode_escape') + self.assertRaisesRegexp(ValueError, re.escape(encoded), + self.request_class.from_response, + response, formxpath=xpath) + def test_from_response_button_submit(self): response = _buildresponse( """
From 4f98be60be700e0c0c359f3cbaa1535ae9ece7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Fri, 13 Nov 2015 18:56:05 +0200 Subject: [PATCH 6/7] FormRequest: fix unicode xpath exception --- scrapy/http/request/form.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 8ff394602..948ad05c9 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -85,7 +85,8 @@ def _get_form(response, formname, formid, formnumber, formxpath): el = el.getparent() if el is None: break - raise ValueError('No element found with %s' % formxpath) + encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape') + raise ValueError('No element found with %s' % encoded) # If we get here, it means that either formname was None # or invalid From ebfdb9bb03fd876bf519535dd57ec0816e0c3c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Sat, 14 Nov 2015 23:24:07 +0200 Subject: [PATCH 7/7] readable xpath with exslt --- scrapy/http/request/form.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 948ad05c9..f623a5aa3 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -109,12 +109,11 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): inputs = form.xpath('descendant::textarea' '|descendant::select' '|descendant::input[not(@type) or @type[' - ' translate(., "SUBMIT", "submit") != "submit"' - ' and translate(., "IMAGE", "image") !="image"' - ' and translate(., "RESET", "reset") != "reset"' - ' and (../@checked or (' - ' translate(., "CHECKBOX", "checkbox") != "checkbox"' - ' and translate(., "RADIO", "radio") != "radio"))]]') + ' not(re:test(., "^(?:submit|image|reset)$", "i"))' + ' and (../@checked or' + ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', + namespaces={ + "re": "http://exslt.org/regular-expressions"}) values = [(k, u'' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata] @@ -160,8 +159,10 @@ def _get_clickable(clickdata, form): clickables = [ el for el in form.xpath( 'descendant::*[(self::input or self::button)' - ' and translate(@type, "SUBMIT", "submit") = "submit"]' - '|descendant::button[not(@type)]')] + ' and re:test(@type, "^submit$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) + ] if not clickables: return