From d5b7ebcfdcfd40d29990712b587893fcc6e84ce8 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 3 Mar 2018 18:17:49 +0530 Subject: [PATCH 1/6] Fixed bug FormRequest.from_response() clickdata ignores input[type=image] --- scrapy/http/request/form.py | 10 ++++++---- tests/test_http_request.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d9d178a3e..184ee2599 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,10 +170,12 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::button[not(@type)]', - namespaces={"re": "http://exslt.org/regular-expressions"}) + 'descendant::*[(self::input or self::button)' + ' and re:test(@type, "^submit$", "i")]' + '|descendant::*[(self::input or self::button)' + ' and re:test(@type, "^image$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: return diff --git a/tests/test_http_request.py b/tests/test_http_request.py index fca8ff411..73a74cd5d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -532,6 +532,16 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v']}) + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response(response, dont_click=True) + fs = _qs(req) + self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( From 65744c2199fc6a5bccfa11eec40a867c1401aee9 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 3 Mar 2018 20:07:50 +0530 Subject: [PATCH 2/6] Corrected Test --- tests/test_http_request.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 73a74cd5d..a042f03b6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -532,16 +532,6 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v']}) - - def test_from_response_clickdata_does_not_ignore_image(self): - response = _buildresponse( - """
- - -
""") - req = self.request_class.from_response(response, dont_click=True) - fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( @@ -554,6 +544,16 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response(response) + fs = _qs(req) + self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_multiple_clickdata(self): response = _buildresponse( From ff5f717f7a2aaf0a1a1101019485201427edd536 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 17 Mar 2018 18:17:48 +0530 Subject: [PATCH 3/6] Fixed formatting issues --- scrapy/http/request/form.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 184ee2599..22846ad77 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -168,15 +168,17 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ + print("form =", form.__dict__) clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::*[(self::input or self::button)' - ' and re:test(@type, "^image$", "i")]' - '|descendant::button[not(@type)]', - namespaces={"re": "http://exslt.org/regular-expressions"}) + 'descendant::*[(self::input or self::button)' + ' and re:test(@type, "^submit$", "i")]' + '|descendant::*[(self::input or self::button)' + ' and re:test(@type, "^image$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) ] + print("clickables =", clickables) if not clickables: return From e25e2afe174bbe70ecbf88ea684937890e8ba4d5 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 17 Mar 2018 18:20:14 +0530 Subject: [PATCH 4/6] Removed unnecessary print statements --- scrapy/http/request/form.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 22846ad77..238dd44b3 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -168,7 +168,6 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - print("form =", form.__dict__) clickables = [ el for el in form.xpath( 'descendant::*[(self::input or self::button)' @@ -178,7 +177,6 @@ def _get_clickable(clickdata, form): '|descendant::button[not(@type)]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] - print("clickables =", clickables) if not clickables: return From a5acc9373f8735e27c22de6fbe345fbed8f268c1 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Mon, 19 Mar 2018 18:19:39 +0530 Subject: [PATCH 5/6] Resolving Comments --- scrapy/http/request/form.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 238dd44b3..d033a830e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,11 +170,8 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::*[(self::input or self::button)' - ' and re:test(@type, "^image$", "i")]' - '|descendant::button[not(@type)]', + 'descendant::input[re.test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re.test(@type, "^submit$", "i")]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: From dd064413a46356940151ac3f9ccd8a45bca2cbd8 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Mon, 19 Mar 2018 19:28:41 +0530 Subject: [PATCH 6/6] corrected syntax error in XPath --- scrapy/http/request/form.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d033a830e..95b38e990 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,8 +170,8 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::input[re.test(@type, "^(submit|image)$", "i")]' - '|descendant::button[not(@type) or re.test(@type, "^submit$", "i")]', + 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: