diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 3b3fb4fca..05288944f 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -74,10 +74,13 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): except (ValueError, TypeError): raise ValueError('formdata should be a dict or iterable of tuples') - inputs = form.xpath('descendant::input[@type!="submit"]|descendant::textarea|descendant::select') + inputs = form.xpath('descendant::textarea' + '|descendant::select' + '|descendant::input[@type!="submit" ' + 'and ((@type!="checkbox" and @type!="radio") or @checked)]') values = [(k, u'' if v is None else v) \ - for k, v in ((e.name, e.value) for e in inputs) \ - if k not in formdata] + for k, v in ((e.name, _value(e)) for e in inputs) \ + if k and k not in formdata] if not dont_click: clickable = _get_clickable(clickdata, form) @@ -87,6 +90,14 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): values.extend(formdata.iteritems()) return values +def _value(ele): + v = ele.value + if v is None and ele.tag == 'select' and not ele.multiple: + o = ele.value_options + if o: + return o[0] + return v + def _get_clickable(clickdata, form): """ Returns the clickable element specified in clickdata, diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index 2110e73f4..f2e53f442 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -524,6 +524,109 @@ class FormRequestTest(RequestTest): self.assertEqual(fs['key7'], ['']) self.assertEqual(set(fs), set(['key1', 'key2', 'key3', 'key4', 'key5', 'key6', 'key7'])) + def test_from_response_select(self): + res = _buildresponse( + '''
+ + + + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['i1v2'], 'i2': ['i2v1'], 'i4': ['i4v2', 'i4v3']}) + + def test_from_response_radio(self): + res = _buildresponse( + '''
+ + + + + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['iv2'], 'i2': ['on']}) + + def test_from_response_checkbox(self): + res = _buildresponse( + '''
+ + + + + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['iv2'], 'i2': ['on']}) + + def test_from_response_input_text(self): + res = _buildresponse( + '''
+ + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['iv1'], 'i2': ['']}) + + def test_from_response_input_hidden(self): + res = _buildresponse( + '''
+ + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['iv1'], 'i2': ['']}) + + def test_from_response_input_hidden(self): + res = _buildresponse( + '''
+ + + +
''') + req = self.request_class.from_response(res) + fs = _qs(req) + self.assertEqual(fs, {'i1': ['iv1'], 'i2': ['']}) + +def _buildresponse(body, **kwargs): + kwargs.setdefault('body', body) + kwargs.setdefault('url', 'http://example.com') + kwargs.setdefault('encoding', 'utf-8') + return HtmlResponse(**kwargs) + +def _qs(req): + if req.method == 'POST': + qs = req.body + else: + qs = req.url.partition('?')[2] + return cgi.parse_qs(qs, True) + class XmlRpcRequestTest(RequestTest): request_class = XmlRpcRequest