From 5a96a16914ad6e4e54a7bd313802891f82a8dc08 Mon Sep 17 00:00:00 2001 From: Dharmesh Pandav Date: Mon, 6 Apr 2015 14:44:47 +0530 Subject: [PATCH 1/4] Update form.py to improve existing capability Add capability to search HTML Form using formid when using `FormRequest.from_response()` refrenced issue :https://github.com/scrapy/scrapy/issues/1136 --- scrapy/http/request/form.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a4695f1a2..cae56f229 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -31,10 +31,10 @@ class FormRequest(Request): self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr) @classmethod - def from_response(cls, response, formname=None, formnumber=0, formdata=None, + def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, clickdata=None, dont_click=False, formxpath=None, **kwargs): kwargs.setdefault('encoding', response.encoding) - form = _get_form(response, formname, formnumber, formxpath) + form = _get_form(response, formname, formid, formnumber, formxpath) formdata = _get_inputs(form, formdata, dont_click, clickdata, response) url = _get_form_url(form, kwargs.pop('url', None)) method = kwargs.pop('method', form.method) @@ -67,6 +67,11 @@ def _get_form(response, formname, formnumber, formxpath): if f: return f[0] + if formid is not None: + f = root.xpath('//form[@id="%s"]' % formid) + if f: + return f[0] + # Get form element from xpath, if not found, go up if formxpath is not None: nodes = root.xpath(formxpath) From 1eccd34a7032c3e50c56f61da34ef39d1aaf43ec Mon Sep 17 00:00:00 2001 From: mrpandav Date: Sun, 12 Apr 2015 11:11:28 +0530 Subject: [PATCH 2/4] adding feature improvement for selecting form using form-id, in addition to formname , formnumer before we go for xpath.. making it more idiomatic in nature --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index cae56f229..82cb5f5d4 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -54,7 +54,7 @@ def _urlencode(seq, enc): return urlencode(values, doseq=1) -def _get_form(response, formname, formnumber, formxpath): +def _get_form(response, formname, formid, formnumber, formxpath): """Find the form element """ from scrapy.selector.lxmldocument import LxmlDocument root = LxmlDocument(response, lxml.html.HTMLParser) From 020a32a3d9639bf948b8f84c8904764f99e0092b Mon Sep 17 00:00:00 2001 From: mrpandav Date: Wed, 15 Apr 2015 11:23:25 +0530 Subject: [PATCH 3/4] Adding tests for pull request #1137 - addition of new shortcut for html form election by formid attribute --- tests/test_http_request.py | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index c81eebfa6..89102016b 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -532,6 +532,60 @@ class FormRequestTest(RequestTest): self.assertRaises(IndexError, self.request_class.from_response, \ response, formname="form3", formnumber=2) + def test_from_response_formid_exists(self): + response = _buildresponse( + """
+ + +
+
+ + +
""") + r1 = self.request_class.from_response(response, formid="form2") + self.assertEqual(r1.method, 'POST') + fs = _qs(r1) + self.assertEqual(fs, {'four': ['4'], 'three': ['3']}) + + def test_from_response_formid_notexists_fallback_formname(self): + response = _buildresponse( + """
+ + +
+
+ + +
""") + r1 = self.request_class.from_response(response, formid="form3", formname="form2") + self.assertEqual(r1.method, 'POST') + fs = _qs(r1) + self.assertEqual(fs, {'four': ['4'], 'three': ['3']}) + + def test_from_response_formid_notexist(self): + response = _buildresponse( + """
+ +
+
+ +
""") + r1 = self.request_class.from_response(response, formid="form3") + self.assertEqual(r1.method, 'POST') + fs = _qs(r1) + self.assertEqual(fs, {'one': ['1']}) + + def test_from_response_formid_errors_formnumber(self): + response = _buildresponse( + """
+ +
+
+ +
""") + self.assertRaises(IndexError, self.request_class.from_response, \ + response, formid="form3", formnumber=2) + def test_from_response_select(self): res = _buildresponse( '''
From 954c8fcecbd0c5c43e371f58585a2ce18f0a6212 Mon Sep 17 00:00:00 2001 From: mrpandav Date: Wed, 15 Apr 2015 16:49:09 +0530 Subject: [PATCH 4/4] changes `test_from_response_formname_notexists_fallback_formid` unit test for pull request #1137 - addition of new shortcut for html form election by formid attribute --- tests/test_http_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 89102016b..01a2de118 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -547,7 +547,7 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs, {'four': ['4'], 'three': ['3']}) - def test_from_response_formid_notexists_fallback_formname(self): + def test_from_response_formname_notexists_fallback_formid(self): response = _buildresponse( """ @@ -557,7 +557,7 @@ class FormRequestTest(RequestTest): """) - r1 = self.request_class.from_response(response, formid="form3", formname="form2") + r1 = self.request_class.from_response(response, formname="form3", formid="form2") self.assertEqual(r1.method, 'POST') fs = _qs(r1) self.assertEqual(fs, {'four': ['4'], 'three': ['3']})