From 63e4355fba1ec90bec9cf751b8b6fc3483062377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 19 Apr 2012 14:10:55 -0300 Subject: [PATCH 1/5] find form input elements using one xpath #111 #121 --- scrapy/http/request/form.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 50533ea42..3b3fb4fca 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -32,10 +32,8 @@ class FormRequest(Request): @classmethod def from_response(cls, response, formname=None, formnumber=0, formdata=None, clickdata=None, dont_click=False, **kwargs): - from scrapy.selector.lxmldocument import LxmlDocument kwargs.setdefault('encoding', response.encoding) - root = LxmlDocument(response, lxml.html.HTMLParser) - form = _get_form(root, formname, formnumber, response) + form = _get_form(response, formname, formnumber) formdata = _get_inputs(form, formdata, dont_click, clickdata, response) url = form.action or form.base_url return cls(url, method=form.method, formdata=formdata, **kwargs) @@ -47,11 +45,10 @@ def _urlencode(seq, enc): for v in (vs if hasattr(vs, '__iter__') else [vs])] return urllib.urlencode(values, doseq=1) -def _get_form(root, formname, formnumber, response): - """ - Uses all the passed arguments to get the required form - element - """ +def _get_form(response, formname, formnumber): + """Find the form element """ + from scrapy.selector.lxmldocument import LxmlDocument + root = LxmlDocument(response, lxml.html.HTMLParser) if not root.forms: raise ValueError("No
element found in %s" % response) @@ -77,15 +74,18 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): except (ValueError, TypeError): raise ValueError('formdata should be a dict or iterable of tuples') - inputs = [(n, u'' if v is None else v) for n, v in form.fields.items() if n not in formdata] + inputs = form.xpath('descendant::input[@type!="submit"]|descendant::textarea|descendant::select') + values = [(k, u'' if v is None else v) \ + for k, v in ((e.name, e.value) for e in inputs) \ + if k not in formdata] if not dont_click: clickable = _get_clickable(clickdata, form) if clickable and clickable[0] not in formdata and not clickable[0] is None: - inputs.append(clickable) + values.append(clickable) - inputs.extend(formdata.iteritems()) - return inputs + values.extend(formdata.iteritems()) + return values def _get_clickable(clickdata, form): """ From 84d5f5ea53b77bee08743af7a9fb940687b46adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 19 Apr 2012 15:40:07 -0300 Subject: [PATCH 2/5] test and fix each form input type with border cases. #111 #121 --- scrapy/http/request/form.py | 17 ++++- scrapy/tests/test_http_request.py | 103 ++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 3 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 3b3fb4fca..05288944f 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -74,10 +74,13 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): except (ValueError, TypeError): raise ValueError('formdata should be a dict or iterable of tuples') - inputs = form.xpath('descendant::input[@type!="submit"]|descendant::textarea|descendant::select') + inputs = form.xpath('descendant::textarea' + '|descendant::select' + '|descendant::input[@type!="submit" ' + 'and ((@type!="checkbox" and @type!="radio") or @checked)]') values = [(k, u'' if v is None else v) \ - for k, v in ((e.name, e.value) for e in inputs) \ - if k not in formdata] + for k, v in ((e.name, _value(e)) for e in inputs) \ + if k and k not in formdata] if not dont_click: clickable = _get_clickable(clickdata, form) @@ -87,6 +90,14 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): values.extend(formdata.iteritems()) return values +def _value(ele): + v = ele.value + if v is None and ele.tag == 'select' and not ele.multiple: + o = ele.value_options + if o: + return o[0] + return v + def _get_clickable(clickdata, form): """ Returns the clickable element specified in clickdata, diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index 2110e73f4..f2e53f442 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -524,6 +524,109 @@ class FormRequestTest(RequestTest): self.assertEqual(fs['key7'], ['']) self.assertEqual(set(fs), set(['key1', 'key2', 'key3', 'key4', 'key5', 'key6', 'key7'])) + def test_from_response_select(self): + res = _buildresponse( + ''' + + + + + + ''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['i1v2'], 'i2': ['i2v1'], 'i4': ['i4v2', 'i4v3']}) + + def test_from_response_radio(self): + res = _buildresponse( + '''
+ + + + + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['iv2'], 'i2': ['on']}) + + def test_from_response_checkbox(self): + res = _buildresponse( + '''
+ + + + + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['iv2'], 'i2': ['on']}) + + def test_from_response_input_text(self): + res = _buildresponse( + '''
+ + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['iv1'], 'i2': ['']}) + + def test_from_response_input_hidden(self): + res = _buildresponse( + '''
+ + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['iv1'], 'i2': ['']}) + + def test_from_response_input_hidden(self): + res = _buildresponse( + '''
+ + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['iv1'], 'i2': ['']}) + +def _buildresponse(body, **kwargs): + kwargs.setdefault('body', body) + kwargs.setdefault('url', 'http://example.com') + kwargs.setdefault('encoding', 'utf-8') + return HtmlResponse(**kwargs) + +def _qs(req): + if req.method == 'POST': + qs = req.body + else: + qs = req.url.partition('?')[2] + return cgi.parse_qs(qs, True) + class XmlRpcRequestTest(RequestTest): request_class = XmlRpcRequest From 4340a13db5d5020e4d1f345ab28012ac75705ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 19 Apr 2012 16:17:34 -0300 Subject: [PATCH 3/5] More lxml FormRequest fixes. #111 #121 * test textarea elements * handle odd cases for select elements like chrome and FF browsers does * Remove test case already covered by per tag test cases --- scrapy/http/request/form.py | 11 ++++-- scrapy/tests/test_http_request.py | 58 ++++++++++++------------------- 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 05288944f..aa23eaec7 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -79,7 +79,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): '|descendant::input[@type!="submit" ' 'and ((@type!="checkbox" and @type!="radio") or @checked)]') values = [(k, u'' if v is None else v) \ - for k, v in ((e.name, _value(e)) for e in inputs) \ + for k, v in (_value(e) for e in inputs) \ if k and k not in formdata] if not dont_click: @@ -91,12 +91,17 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): return values def _value(ele): + n = ele.name v = ele.value + # Match browser behaviour on simple select tag without options selected + # Or for select tags wihout options if v is None and ele.tag == 'select' and not ele.multiple: o = ele.value_options if o: - return o[0] - return v + return n, o[0] + else: + return None, None + return n, v def _get_clickable(clickdata, form): """ diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index f2e53f442..92e2be484 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -496,34 +496,6 @@ class FormRequestTest(RequestTest): response = HtmlResponse("http://www.example.com/formname.html", body=respbody) self.assertRaises(IndexError, self.request_class.from_response, response, formname="form3", formnumber=2) - def test_from_response_missed_value(self): - respbody = """ -
- - - - - - + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['i1v'], 'i2': [''], 'i3': ['']}) def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) From 3b2458dbbbfdb3aeb038b3f37575a24128f6f8e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 19 Apr 2012 16:48:02 -0300 Subject: [PATCH 4/5] cleanup all FormRequest test cases. #111 #121 --- scrapy/tests/test_http_request.py | 434 +++++++++++++----------------- 1 file changed, 194 insertions(+), 240 deletions(-) diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index 92e2be484..025d49b28 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -206,295 +206,248 @@ class FormRequestTest(RequestTest): self.assertEqual(r3.body, 'colours=red&colours=blue&colours=green&price=%C2%A3+100') def test_from_response_post(self): - respbody = """ -
- - - -
- """ - response = HtmlResponse("http://www.example.com/this/list.html", body=respbody) - r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}, callback=lambda x: x) - self.assertEqual(r1.method, 'POST') - self.assertEqual(r1.headers['Content-type'], 'application/x-www-form-urlencoded') - fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"}) - self.assertEqual(r1.url, "http://www.example.com/this/post.php") - self.assertEqual(set([f.value for f in fs["test"]]), set(["val1", "val2"])) - self.assertEqual(set([f.value for f in fs["one"]]), set(["two", "three"])) - self.assertEqual(fs['test2'].value, 'xxx') - self.assertEqual(fs['six'].value, 'seven') + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html") + req = self.request_class.from_response(response, + formdata={'one': ['two', 'three'], 'six': 'seven'}) + self.assertEqual(req.method, 'POST') + self.assertEqual(req.headers['Content-type'], 'application/x-www-form-urlencoded') + self.assertEqual(req.url, "http://www.example.com/this/post.php") + fs = _qs(req) + self.assertEqual(set(fs["test"]), set(["val1", "val2"])) + self.assertEqual(set(fs["one"]), set(["two", "three"])) + self.assertEqual(fs['test2'], ['xxx']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_extra_headers(self): - respbody = """ -
- - - -
- """ - headers = {"Accept-Encoding": "gzip,deflate"} - response = HtmlResponse("http://www.example.com/this/list.html", body=respbody) - r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}, headers=headers, callback=lambda x: x) - self.assertEqual(r1.method, 'POST') - self.assertEqual(r1.headers['Content-type'], 'application/x-www-form-urlencoded') - self.assertEqual(r1.headers['Accept-Encoding'], 'gzip,deflate') + response = _buildresponse( + """
+ + + +
""") + req = self.request_class.from_response(response, + formdata={'one': ['two', 'three'], 'six': 'seven'}, + headers={"Accept-Encoding": "gzip,deflate"}) + self.assertEqual(req.method, 'POST') + self.assertEqual(req.headers['Content-type'], 'application/x-www-form-urlencoded') + self.assertEqual(req.headers['Accept-Encoding'], 'gzip,deflate') def test_from_response_get(self): - respbody = """ -
- - - -
- """ - response = HtmlResponse("http://www.example.com/this/list.html", body=respbody) - r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html") + r1 = self.request_class.from_response(response, + formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(r1.method, 'GET') self.assertEqual(urlparse(r1.url).hostname, "www.example.com") self.assertEqual(urlparse(r1.url).path, "/this/get.php") - urlargs = cgi.parse_qs(urlparse(r1.url).query) - self.assertEqual(set(urlargs['test']), set(['val1', 'val2'])) - self.assertEqual(set(urlargs['one']), set(['two', 'three'])) - self.assertEqual(urlargs['test2'], ['xxx']) - self.assertEqual(urlargs['six'], ['seven']) + fs = _qs(r1) + self.assertEqual(set(fs['test']), set(['val1', 'val2'])) + self.assertEqual(set(fs['one']), set(['two', 'three'])) + self.assertEqual(fs['test2'], ['xxx']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_override_params(self): - respbody = """ -
- - -
- """ - response = HtmlResponse("http://www.example.com/this/list.html", body=respbody) - r1 = self.request_class.from_response(response, formdata={'two': '2'}) - fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"}) - self.assertEqual(fs['one'].value, '1') - self.assertEqual(fs['two'].value, '2') + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response(response, formdata={'two': '2'}) + fs = _qs(req) + self.assertEqual(fs['one'], ['1']) + self.assertEqual(fs['two'], ['2']) def test_from_response_submit_first_clickable(self): - respbody = """ -
- - - - -
- """ - response = HtmlResponse("http://www.example.com/this/list.html", body=respbody) - r1 = self.request_class.from_response(response, formdata={'two': '2'}) - urlargs = cgi.parse_qs(urlparse(r1.url).query) - self.assertEqual(urlargs['clickable1'], ['clicked1']) - self.assertFalse('clickable2' in urlargs, urlargs) - self.assertEqual(urlargs['one'], ['1']) - self.assertEqual(urlargs['two'], ['2']) + response = _buildresponse( + """
+ + + + +
""") + req = self.request_class.from_response(response, formdata={'two': '2'}) + fs = _qs(req) + self.assertEqual(fs['clickable1'], ['clicked1']) + self.assertFalse('clickable2' in fs, fs) + self.assertEqual(fs['one'], ['1']) + self.assertEqual(fs['two'], ['2']) def test_from_response_submit_not_first_clickable(self): - respbody = """ -
- - - - -
- """ - response = HtmlResponse("http://www.example.com/this/list.html", body=respbody) - r1 = self.request_class.from_response(response, formdata={'two': '2'}, clickdata={'name': 'clickable2'}) - urlargs = cgi.parse_qs(urlparse(r1.url).query) - self.assertEqual(urlargs['clickable2'], ['clicked2']) - self.assertFalse('clickable1' in urlargs, urlargs) - self.assertEqual(urlargs['one'], ['1']) - self.assertEqual(urlargs['two'], ['2']) + response = _buildresponse( + """
+ + + + +
""") + req = self.request_class.from_response(response, formdata={'two': '2'}, \ + clickdata={'name': 'clickable2'}) + fs = _qs(req) + self.assertEqual(fs['clickable2'], ['clicked2']) + self.assertFalse('clickable1' in fs, fs) + self.assertEqual(fs['one'], ['1']) + self.assertEqual(fs['two'], ['2']) def test_from_response_multiple_clickdata(self): - respbody = """ -
- - - - -
- """ - response = HtmlResponse("http://www.example.com/this/list.html", body=respbody) - r1 = self.request_class.from_response(response, \ + response = _buildresponse( + """
+ + + + +
""") + req = self.request_class.from_response(response, \ clickdata={'name': 'clickable', 'value': 'clicked2'}) - urlargs = cgi.parse_qs(urlparse(r1.url).query) - self.assertEqual(urlargs['clickable'], ['clicked2']) - self.assertEqual(urlargs['one'], ['clicked1']) - self.assertEqual(urlargs['two'], ['clicked2']) + fs = _qs(req) + self.assertEqual(fs['clickable'], ['clicked2']) + self.assertEqual(fs['one'], ['clicked1']) + self.assertEqual(fs['two'], ['clicked2']) def test_from_response_unicode_clickdata(self): - body = u""" -
- - - - -
- """ - response = HtmlResponse("http://www.example.com", body=body, \ - encoding='utf-8') - r1 = self.request_class.from_response(response, \ + response = _buildresponse( + u"""
+ + + + +
""") + req = self.request_class.from_response(response, \ clickdata={'name': u'price in \u00a3'}) - urlargs = cgi.parse_qs(urlparse(r1.url).query) - self.assertTrue(urlargs[u'price in \u00a3'.encode('utf-8')]) - - def test_from_response_with_select(self): - body = u""" -
- - -
- """ - res = HtmlResponse("http://example.com", body=body, encoding='utf-8') - req = self.request_class.from_response(res) - urlargs = cgi.parse_qs(urlparse(req.url).query) - self.assertEqual(urlargs['inputname'], ['inputvalue']) + fs = _qs(req) + self.assertTrue(fs[u'price in \u00a3'.encode('utf-8')]) def test_from_response_multiple_forms_clickdata(self): - body = u""" -
- - -
-
- - -
- """ - res = HtmlResponse("http://example.com", body=body, encoding='utf-8') - req = self.request_class.from_response(res, formname='form2', \ + response = _buildresponse( + """
+ + +
+
+ + +
+ """) + req = self.request_class.from_response(response, formname='form2', \ clickdata={'name': 'clickable'}) - urlargs = cgi.parse_qs(urlparse(req.url).query) - self.assertEqual(urlargs['clickable'], ['clicked2']) - self.assertEqual(urlargs['field2'], ['value2']) - self.assertFalse('field1' in urlargs, urlargs) + fs = _qs(req) + self.assertEqual(fs['clickable'], ['clicked2']) + self.assertEqual(fs['field2'], ['value2']) + self.assertFalse('field1' in fs, fs) def test_from_response_override_clickable(self): - body = u'
' - res = HtmlResponse("http://example.com", body=body, encoding='utf-8') - req = self.request_class.from_response(res, \ - formdata={'clickme': 'two'}, \ - clickdata={'name': 'clickme'}) - urlargs = cgi.parse_qs(urlparse(req.url).query) - self.assertEqual(urlargs['clickme'], ['two']) + response = _buildresponse('''
''') + req = self.request_class.from_response(response, \ + formdata={'clickme': 'two'}, clickdata={'name': 'clickme'}) + fs = _qs(req) + self.assertEqual(fs['clickme'], ['two']) def test_from_response_dont_click(self): - respbody = """ -
- - - - -
- """ - response = HtmlResponse("http://www.example.com/this/list.html", body=respbody) + response = _buildresponse( + """
+ + + + +
""") r1 = self.request_class.from_response(response, dont_click=True) - urlargs = cgi.parse_qs(urlparse(r1.url).query) - self.assertFalse('clickable1' in urlargs, urlargs) - self.assertFalse('clickable2' in urlargs, urlargs) + fs = _qs(r1) + self.assertFalse('clickable1' in fs, fs) + self.assertFalse('clickable2' in fs, fs) def test_from_response_ambiguous_clickdata(self): - respbody = """ -
- - - - -
- """ - response = HtmlResponse("http://www.example.com/this/list.html", body=respbody) - self.assertRaises(ValueError, - self.request_class.from_response, - response, - clickdata={'type': 'submit'}) + response = _buildresponse( + """ +
+ + + + +
""") + self.assertRaises(ValueError, self.request_class.from_response, + response, clickdata={'type': 'submit'}) def test_from_response_non_matching_clickdata(self): - body = """ -
- -
- """ - res = HtmlResponse("http://example.com", body=body) - self.assertRaises(ValueError, - self.request_class.from_response, res, - clickdata={'nonexistent': 'notme'}) + response = _buildresponse( + """
+ +
""") + self.assertRaises(ValueError, self.request_class.from_response, + response, clickdata={'nonexistent': 'notme'}) def test_from_response_errors_noform(self): - respbody = """""" - response = HtmlResponse("http://www.example.com/lala.html", body=respbody) + response = _buildresponse("""""") self.assertRaises(ValueError, self.request_class.from_response, response) def test_from_response_errors_formnumber(self): - respbody = """ -
- - - -
- """ - response = HtmlResponse("http://www.example.com/lala.html", body=respbody) + response = _buildresponse( + """
+ + + +
""") self.assertRaises(IndexError, self.request_class.from_response, response, formnumber=1) def test_from_response_noformname(self): - respbody = """ -
- - -
- """ - response = HtmlResponse("http://www.example.com/formname.html", body=respbody) - r1 = self.request_class.from_response(response, formdata={'two':'3'}, callback=lambda x: x) + response = _buildresponse( + """
+ + +
""") + r1 = self.request_class.from_response(response, formdata={'two':'3'}) self.assertEqual(r1.method, 'POST') self.assertEqual(r1.headers['Content-type'], 'application/x-www-form-urlencoded') - fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"}) - self.assertEqual(fs['one'].value, '1') - self.assertEqual(fs['two'].value, '3') - + fs = _qs(r1) + self.assertEqual(fs, {'one': ['1'], 'two': ['3']}) def test_from_response_formname_exists(self): - respbody = """ -
- - -
-
- - -
- """ - response = HtmlResponse("http://www.example.com/formname.html", body=respbody) - r1 = self.request_class.from_response(response, formname="form2", callback=lambda x: x) + response = _buildresponse( + """
+ + +
+
+ + +
""") + r1 = self.request_class.from_response(response, formname="form2") self.assertEqual(r1.method, 'POST') - fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"}) - self.assertEqual(fs['three'].value, "3") - self.assertEqual(fs['four'].value, "4") + fs = _qs(r1) + self.assertEqual(fs, {'four': ['4'], 'three': ['3']}) def test_from_response_formname_notexist(self): - respbody = """ -
- -
-
- -
- """ - response = HtmlResponse("http://www.example.com/formname.html", body=respbody) - r1 = self.request_class.from_response(response, formname="form3", callback=lambda x: x) + response = _buildresponse( + """
+ +
+
+ +
""") + r1 = self.request_class.from_response(response, formname="form3") self.assertEqual(r1.method, 'POST') - fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"}) - self.assertEqual(fs['one'].value, "1") + fs = _qs(r1) + self.assertEqual(fs, {'one': ['1']}) def test_from_response_formname_errors_formnumber(self): - respbody = """ -
- -
-
- -
- """ - response = HtmlResponse("http://www.example.com/formname.html", body=respbody) - self.assertRaises(IndexError, self.request_class.from_response, response, formname="form3", formnumber=2) + response = _buildresponse( + """
+ +
+
+ +
""") + self.assertRaises(IndexError, self.request_class.from_response, \ + response, formname="form3", formnumber=2) def test_from_response_select(self): res = _buildresponse( @@ -613,6 +566,7 @@ def _qs(req): qs = req.url.partition('?')[2] return cgi.parse_qs(qs, True) + class XmlRpcRequestTest(RequestTest): request_class = XmlRpcRequest From 72485128cb29751b3b03f6a04e73e4cc50912910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 19 Apr 2012 16:58:01 -0300 Subject: [PATCH 5/5] Add a test case to cover input elements as not direct child of form element. #111 #121 --- scrapy/tests/test_http_request.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index 025d49b28..8cc091446 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -553,6 +553,28 @@ class FormRequestTest(RequestTest): fs = _qs(req) self.assertEqual(fs, {'i1': ['i1v'], 'i2': [''], 'i3': ['']}) + def test_from_response_descendants(self): + res = _buildresponse( + '''
+
+
+ + +
+ + + + +
+ +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(set(fs), set(['h2', 'i2', 'i1', 'i3', 'h1', 'i5', 'i4'])) + + def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) kwargs.setdefault('url', 'http://example.com')