diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index c3e112041..0c947565a 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union -from urllib.parse import urljoin, urlencode +from urllib.parse import urljoin, urlencode, urlsplit, urlunsplit from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement from parsel.selector import create_root_node @@ -34,12 +34,12 @@ class FormRequest(Request): if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata - querystr = _urlencode(items, self.encoding) + form_query_str = _urlencode(items, self.encoding) if self.method == 'POST': self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') - self._set_body(querystr) + self._set_body(form_query_str) else: - self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr) + self._set_url(urlunsplit(urlsplit(self.url)._replace(query=form_query_str))) @classmethod def from_response( diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 74579dfc4..b610087bd 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -379,6 +379,20 @@ class FormRequestTest(RequestTest): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') + def test_formdata_overrides_querystring(self): + data = (('a', 'one'), ('a', 'two'), ('b', '2')) + url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', + method='GET', formdata=data).url.split('#')[0] + fs = _qs(self.request_class(url, method='GET', formdata=data)) + self.assertEqual(set(fs[b'a']), {b'one', b'two'}) + self.assertEqual(fs[b'b'], [b'2']) + self.assertIsNone(fs.get(b'c')) + + data = {'a': '1', 'b': '2'} + fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'}