Merge pull request #3579 from maramsumanth/duplicate_key

[MRG+1] Issue #2919: Fix FormRequest.formdata with GET method duplicates same key in query string

scrapy/http/request/form.py

@@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union
-from urllib.parse import urljoin, urlencode
+from urllib.parse import urljoin, urlencode, urlsplit, urlunsplit
 
 from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement
 from parsel.selector import create_root_node
@@ -34,12 +34,12 @@ class FormRequest(Request):
 
         if formdata:
             items = formdata.items() if isinstance(formdata, dict) else formdata
-            querystr = _urlencode(items, self.encoding)
+            form_query_str = _urlencode(items, self.encoding)
             if self.method == 'POST':
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
-                self._set_body(querystr)
+                self._set_body(form_query_str)
             else:
-                self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
+                self._set_url(urlunsplit(urlsplit(self.url)._replace(query=form_query_str)))
 
     @classmethod
     def from_response(

tests/test_http_request.py

@@ -379,6 +379,20 @@ class FormRequestTest(RequestTest):
         r1 = self.request_class("http://www.example.com", formdata={})
         self.assertEqual(r1.body, b'')
 
+    def test_formdata_overrides_querystring(self):
+        data = (('a', 'one'), ('a', 'two'), ('b', '2'))
+        url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment',
+                                 method='GET', formdata=data).url.split('#')[0]
+        fs = _qs(self.request_class(url, method='GET', formdata=data))
+        self.assertEqual(set(fs[b'a']), {b'one', b'two'})
+        self.assertEqual(fs[b'b'], [b'2'])
+        self.assertIsNone(fs.get(b'c'))
+
+        data = {'a': '1', 'b': '2'}
+        fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
+
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)
         data = {b'one': b'two', b'price': b'\xc2\xa3 100'}