diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 95b38e990..c2413b431 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -114,10 +114,12 @@ def _get_form(response, formname, formid, formnumber, formxpath): def _get_inputs(form, formdata, dont_click, clickdata, response): try: - formdata = dict(formdata or ()) + formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): raise ValueError('formdata should be a dict or iterable of tuples') + if not formdata: + formdata = () inputs = form.xpath('descendant::textarea' '|descendant::select' '|descendant::input[not(@type) or @type[' @@ -128,14 +130,17 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): "re": "http://exslt.org/regular-expressions"}) values = [(k, u'' if v is None else v) for k, v in (_value(e) for e in inputs) - if k and k not in formdata] + if k and k not in formdata_keys] if not dont_click: clickable = _get_clickable(clickdata, form) if clickable and clickable[0] not in formdata and not clickable[0] is None: values.append(clickable) - values.extend((k, v) for k, v in formdata.items() if v is not None) + if isinstance(formdata, dict): + formdata = formdata.items() + + values.extend((k, v) for k, v in formdata if v is not None) return values diff --git a/tests/test_http_request.py b/tests/test_http_request.py index fc89229c6..58326a384 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -406,6 +406,29 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[u'test2'], [u'xxx ยต']) self.assertEqual(fs[u'six'], [u'seven']) + def test_from_response_duplicate_form_key(self): + response = _buildresponse( + '
', + url='http://www.example.com') + req = self.request_class.from_response(response, + method='GET', + formdata=(('foo', 'bar'), ('foo', 'baz'))) + self.assertEqual(urlparse(req.url).hostname, 'www.example.com') + self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz') + + def test_from_response_override_duplicate_form_key(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response( + response, + formdata=(('two', '2'), ('two', '4'))) + fs = _qs(req) + self.assertEqual(fs[b'one'], [b'1']) + self.assertEqual(fs[b'two'], [b'2', b'4']) + def test_from_response_extra_headers(self): response = _buildresponse( """